+
+
+
+
+
+
+
diff --git a/html-versions/3 - Images and Text.slides.html b/html-versions/3 - Images and Text.slides.html
new file mode 100644
index 0000000000000000000000000000000000000000..34bdf551dd9a17ebd739693a639723dabf9c8af5
--- /dev/null
+++ b/html-versions/3 - Images and Text.slides.html
@@ -0,0 +1,13603 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In this block¶
-
+
- Overview IIIF +
- Overview OCR formats +
+
+
+
+
+-
+
- Example: Create IIIF collection from SPARQL query result +
- Example: Download pre-downsized images for machine learning +
- Example: Download OCR text +
+
+
+
+
+Overview IIIF¶
+ +
+
+
+
+
+What is IIIF?¶
+
+
+
+
+
+-
+
- International Image Interoperability Framework (http://iiif.io/ - well written, worth a read) +
- Standardised method of describing and delivering images over the web +
- Community that develops APIs and implements them in Software +
+
+
+
+
+
Image courtesy of https://github.com/IIIF/training, CC-BY 4.0
+ +
+
+
+
+
+Why would I use this?¶
+
+
+
+
+
+If you want to display images¶
-
+
- If you want to use one of several nice viewers for images (zoom, rotate, fullscreen ootb) +
- If you want to include image data hosted elsewhere +
+
+
+
+
+If you want to process images¶
-
+
- If you want structured access to potentially huge sets of images +
- If you want included metadata +
- If you want to resize images before downloading +
+
+
+
+
+How would I use this?¶
+
+
+
+
+
+-
+
- You could access an image directly (Image API)
-
+
- Parameters can be changed in the URL +
- https://iiif.onb.ac.at/images/AKON/AK035_199/199/full/full/0/native.jpg +
+
+
+
+
+
+-
+
- You could get a manifest JSON (Presentation API)
-
+
- Contains images and metadata +
- https://iiif.onb.ac.at/presentation/AKON/AK035_199/manifest/ +
+
+
+
+
+
+-
+
- You could get a collection JSON (Presentation API)
-
+
- Contains manifests and possibly other collections +
- https://iiif.onb.ac.at/presentation/collection/pydays19 +
+
+
+
+
+
+Pics or didn't happen!¶
+
+
+
+
+
+-
+
- The ONB Labs viewers use IIIF: https://labs.onb.ac.at/en/dataset/akon/ +
+
+
+
+
+-
+
- Awesome IIIF-related resources : https://github.com/IIIF/awesome-iiif +
+
+
+
+
+
+
+
+
+
+
+
+-
+
- IIIF Presentation API
-
+
- returns JSON-LD structured documents that together describe the structure and layout of a digitized object or other collection of images and related content +
- https://iiif.io/api/presentation/2.1/ +
- https://iiif.onb.ac.at/presentation/ABO/+Z196807705/manifest/ +
+
+
+
+
+
+-
+
- IIIF Image API
-
+
- requesting and delivering images on the Web +
- https://iiif.io/api/image/2.1/ +
- Image Request URI Syntax
-
+
- {scheme}://{server}{/prefix}/{identifier}/{region}/{size}/{rotation}/{quality}.{format} +
- http://iiif.onb.ac.at/images/ABO/Z196807705/00000009/full/full/0/native.jpg +
- 90 degree rotation: http://iiif.onb.ac.at/images/ABO/Z196807705/00000009/full/full/90/native.jpg +
- "Articulus Quartus": https://iiif.onb.ac.at/images/ABO/Z196807705/00000009/pct:0,0,100,33/full/0/native.jpg +
+
+
+
+
+
+
+Overview OCR formats¶
+
+
+
+
+
+-
+
- ALTO (Analyzed Layout and Text Object)
-
+
- OCR (Optical Character Recognition) data representation format +
- XML Schema +
- https://github.com/altoxml +
- https://www.loc.gov/standards/alto/ +
+
+
+
+
+
+-
+
- 3 ALTO main elements
-
+
<Description>-
+
- metadata and general settings (e.g. measurement units) about the ALTO file +
+<Styles>-
+
- text and paragraph styles +
+<Layout>-
+
- content information +
- subdivided into
<Page>elements
+
+
+
+
+
+
+
+
+
+
+
+
+-
+
- hOCR
-
+
- alternative to ALTO +
- based on XHTML +
- not used in the ONB Labs +
+
+
+
+
+
+
+
+
+
diff --git a/html-versions/3.1 - IIIF Collection from SPARQL.html b/html-versions/3.1 - IIIF Collection from SPARQL.html
new file mode 100644
index 0000000000000000000000000000000000000000..78881a9e3cc2dfb1279df1f7cc66ccb5e2d53db0
--- /dev/null
+++ b/html-versions/3.1 - IIIF Collection from SPARQL.html
@@ -0,0 +1,13359 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In this block¶
-
+
- Overview IIIF +
- Overview OCR formats +
+
+
+
+
+
+-
+
- Example: Create IIIF collection from SPARQL query result +
- Example: Download pre-downsized images for machine learning +
- Example: Download OCR text +
+
+
+
+
+Overview IIIF¶
+ +
+
+
+
+
+What is IIIF?¶
+
+
+
+
+
+
+-
+
- International Image Interoperability Framework (http://iiif.io/ - well written, worth a read) +
- Standardised method of describing and delivering images over the web +
- Community that develops APIs and implements them in Software +
+
+
+
+
+
+
Image courtesy of https://github.com/IIIF/training, CC-BY 4.0
+ +
+
+
+
+
+Why would I use this?¶
+
+
+
+
+
+
+If you want to display images¶
-
+
- If you want to use one of several nice viewers for images (zoom, rotate, fullscreen ootb) +
- If you want to include image data hosted elsewhere +
+
+
+
+
+
+If you want to process images¶
-
+
- If you want structured access to potentially huge sets of images +
- If you want included metadata +
- If you want to resize images before downloading +
+
+
+
+
+How would I use this?¶
+
+
+
+
+
+
+-
+
- You could access an image directly (Image API)
-
+
- Parameters can be changed in the URL +
- https://iiif.onb.ac.at/images/AKON/AK035_199/199/full/full/0/native.jpg +
+
+
+
+
+
+
+-
+
- You could get a manifest JSON (Presentation API)
-
+
- Contains images and metadata +
- https://iiif.onb.ac.at/presentation/AKON/AK035_199/manifest/ +
+
+
+
+
+
+
+-
+
- You could get a collection JSON (Presentation API)
-
+
- Contains manifests and possibly other collections +
- https://iiif.onb.ac.at/presentation/collection/pydays19 +
+
+
+
+
+
+Pics or didn't happen!¶
+
+
+
+
+
+
+-
+
- The ONB Labs viewers use IIIF: https://labs.onb.ac.at/en/dataset/akon/ +
+
+
+
+
+
+-
+
- Awesome IIIF-related resources : https://github.com/IIIF/awesome-iiif +
+
+
+
+
+
+
+
+
+
+
+
+
+-
+
- IIIF Presentation API
-
+
- returns JSON-LD structured documents that together describe the structure and layout of a digitized object or other collection of images and related content +
- https://iiif.io/api/presentation/2.1/ +
- https://iiif.onb.ac.at/presentation/ABO/+Z196807705/manifest/ +
+
+
+
+
+
+-
+
- IIIF Image API
-
+
- requesting and delivering images on the Web +
- https://iiif.io/api/image/2.1/ +
- Image Request URI Syntax
-
+
- {scheme}://{server}{/prefix}/{identifier}/{region}/{size}/{rotation}/{quality}.{format} +
- http://iiif.onb.ac.at/images/ABO/Z196807705/00000009/full/full/0/native.jpg +
- 90 degree rotation: http://iiif.onb.ac.at/images/ABO/Z196807705/00000009/full/full/90/native.jpg +
- "Articulus Quartus": https://iiif.onb.ac.at/images/ABO/Z196807705/00000009/pct:0,0,100,33/full/0/native.jpg +
+
+
+
+
+
+
+Overview OCR formats¶
+
+
+
+
+
+-
+
- ALTO (Analyzed Layout and Text Object)
-
+
- OCR (Optical Character Recognition) data representation format +
- XML Schema +
- https://github.com/altoxml +
- https://www.loc.gov/standards/alto/ +
+
+
+
+
+
+-
+
- 3 ALTO main elements
-
+
<Description>-
+
- metadata and general settings (e.g. measurement units) about the ALTO file +
+<Styles>-
+
- text and paragraph styles +
+<Layout>-
+
- content information +
- subdivided into
<Page>elements
+
+
+
+
+
+
+
+
+
+
+
+
+-
+
- hOCR
-
+
- alternative to ALTO +
- based on XHTML +
- not used in the ONB Labs +
+
+
+
+
+
+
+
+
diff --git a/html-versions/3.2 - Images - Download pre-downsized images for machine learning.html b/html-versions/3.2 - Images - Download pre-downsized images for machine learning.html
new file mode 100644
index 0000000000000000000000000000000000000000..8aa7a55c700bb6d3d7b064b5271b702431ecb025
--- /dev/null
+++ b/html-versions/3.2 - Images - Download pre-downsized images for machine learning.html
@@ -0,0 +1,15023 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [1]:
+
+
+
+
+
+
+import requests
+import pandas as pd
+from SPARQLWrapper import SPARQLWrapper, JSON
+import json
+
+
+
+
+
+Set the SPARQL-Endpoint:
+-
+
- https://lod.onb.ac.at/sparql/anno for ANNO +
- https://lod.onb.ac.at/sparql/akon for AKON +
+
+
+
+
+In [2]:
+
+
+
+
+
+
+anno_lod_endpoint = "https://lod.onb.ac.at/sparql/anno"
+
+
+
+
+
+Methods to query the endpoint and build the dataframe:
+ +
+
+
+
+
+In [3]:
+
+
+
+
+
+
+def get_sparql_result(service, query):
+ sparql = SPARQLWrapper(service)
+ sparql.setQuery(query)
+ sparql.setReturnFormat(JSON)
+ return sparql.query()
+
+def get_sparql_dataframe(service, query):
+ result = get_sparql_result(service, query)
+
+ processed_results = result.convert()
+ cols = processed_results['head']['vars']
+
+ out = []
+ for row in processed_results['results']['bindings']:
+ item = []
+ for c in cols:
+ item.append(row.get(c, {}).get('value'))
+ out.append(item)
+
+ return pd.DataFrame(out, columns=cols)
+
+
+
+
+
+Select all newspapers and periodicals with subjectheading Statistik:
+ +
+
+
+
+
+In [4]:
+
+
+
+
+
+
+query = '''
+PREFIX dc: <http://purl.org/dc/elements/1.1/>
+PREFIX edm: <http://www.europeana.eu/schemas/edm/>
+PREFIX dcterms: <http://purl.org/dc/terms/>
+SELECT ?title ?subjectURI ?manifest
+WHERE {?subjectURI dc:subject <http://d-nb.info/gnd/4056995-0> .
+ ?subjectURI dc:title ?title .
+ ?subjectURI edm:isShownBy ?firstpage .
+ ?subjectURI edm:rights <http://creativecommons.org/publicdomain/mark/1.0/> .
+ ?firstpage dcterms:isReferencedBy ?manifest
+}'''
+
+
+
+
+
+Get list of IIIF Manifests URLs:
+ +
+
+
+
+
+In [5]:
+
+
+
+
+
+
+df = get_sparql_dataframe(anno_lod_endpoint, query)
+manifests = list(df['manifest'])
+manifests
+
+
+
+
+
+
+
+
+
+
+
+Out[5]:
+
+
+
+
+
+
+
+
+
+
+
+Function to create a SACHA Collection (https://iiif.onb.ac.at/api#_collectionspostjsonprocessor):
+ +
+
+
+
+
+In [6]:
+
+
+
+
+
+
+def create_collection(description, list_of_manifest_ids_or_ids):
+ j = {
+ "description": description,
+ "elements": list_of_manifest_ids_or_ids
+ }
+ creation_link = 'https://iiif.onb.ac.at/presentation/collection'
+ result = requests.post(creation_link, json=j)
+ if result.status_code == 201:
+ print('SUCCESS: Create collection {}'.format(result.json()['url']))
+ print('View collection in Mirador: https://iiif.onb.ac.at/view/collection/mirador/' + result.json()['url'].split('/').pop())
+ elif result.status_code == 400:
+ print('ERROR: Request error creating collection')
+ print(result.text)
+ elif result.status_code == 500:
+ print('ERROR: Server error creating collection')
+ print(result.text)
+ else:
+ print('ERROR: General error creating collection, HTTP status = {}'.format(result.status_code))
+
+
+
+
+
+Create the SACHA Collection:
+ +
+
+
+
+
+In [7]:
+
+
+
+
+
+
+create_collection("newspaper with subject heading Statistik", manifests)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/html-versions/3.3 - Text - Download OCR Text.html b/html-versions/3.3 - Text - Download OCR Text.html
new file mode 100644
index 0000000000000000000000000000000000000000..8ba241d6cf1bab2eefc679c9b0b19a60e8739fec
--- /dev/null
+++ b/html-versions/3.3 - Text - Download OCR Text.html
@@ -0,0 +1,14210 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+3.2 - Images - Download pre-downsized images for machine learning¶
I want to download a bunch of small images, already scaled down for my CNN
+https://labs.onb.ac.at/en/dataset/akon/
+https://labs.onb.ac.at/gitlab/labs-team/raw-metadata/raw/master/akon_postcards_public_domain.csv.bz2
+ + +
+
+
+
+
+Let's say you got a bunch of old timey scenery photographs. +And you want to extract all images containing mountains, why not. +And, because you can, you want an AI to do all the dirty work for you.
+What that has to do with this workshop?
+You can use the historic postcards from the ONB Labs as training data for your AI.
+ +
+
+
+
+
+Disclaimer: The AI-part is beyond the scope of this notebook, and would blow up the size of the venv considerably.
+If you want instructions on actually performing the training, take a look at
+ +One way to do it: Download a VGG16 network that's pre-trained on ImageNet, remove the last layer (the actual classifier), add your own output layer with 2 outputs ('mountain', 'no mountain') and train that one.
+Now back to the show.
+ +
+
+
+
+
+What do we have to do?
+-
+
- Download Metdata
-
+
- List of all available postcards +
- Info about the 'mountain-ness' of postcards +
+ - Create Download Links
-
+
- To fetch all images +
+ - Split Into Two Sets
-
+
- Mountain and non-mountain +
+ - Download Images +
+
+
+
+
+Download Metadata¶
+
+
+
+
+
+Download the metadata set from the ONB Labs
+ +
+
+
+
+
+In [1]:
+
+
+
+
+
+
+import pandas as pd
+
+# Let pandas show all available columns
+pd.set_option('display.max_columns', 50)
+# Pandas can read data directly from web links, even compressed files
+meta = pd.read_csv('https://labs.onb.ac.at/gitlab/labs-team/' \
+ 'raw-metadata/raw/master/akon_postcards_public_domain.csv.bz2', compression='bz2')
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [2]:
+
+
+
+
+
+
+meta.sample(6)
+
+
+
+
+
+
+
+
+
+
+
+Out[2]:
+
+
+
+
+
+
+
+
+
+
+Ok, we have metadata. And look, there's a column mountain:
+ +
+
+
+
+
+In [3]:
+
+
+
+
+
+
+meta.sample(5)[['akon_id', 'mountain']]
+
+
+
+
+
+
+
+
+
+
+
+Out[3]:
+
+
+
+
+
+
+
+
+
+
+Later, we'll split the dataset in two using the data in this column.
+ +
+
+
+
+
+Create Download Links¶
+
+
+
+
+
+The SACHA project provides an API for accessing digitized objects of the National Library via IIIF. +The online documentation for the API is here: https://iiif.onb.ac.at/api.
+We're especially interested in the possibility to serve manifests: https://iiif.onb.ac.at/api#_manifestrequestprocessor:
+ +GET /presentation/{projectName}/{id}/manifest
+
+
+
+
+
+GET /presentation/{projectName}/{id}/manifestThe projectName is AKON ('AnsichtsKarten ONline'), the id is the akon_id.
+
+
+
+
+Create Download Links for Manifests¶
+
+
+
+
+
+In [4]:
+
+
+
+
+
+
+def akon_id_to_manifest_link(akon_id):
+ return f'https://iiif.onb.ac.at/presentation/AKON/{akon_id}/manifest'
+
+
+
+
+
+In [5]:
+
+
+
+
+
+
+akon_id_to_manifest_link('AK024_176')
+
+
+
+
+
+
+
+
+
+
+
+Out[5]:
+
+
+
+
+
+
+
+
+
+
+
+Let's test the link
+ +
+
+
+
+
+In [6]:
+
+
+
+
+
+
+import requests
+
+r = requests.get(akon_id_to_manifest_link('AK024_176'))
+r.json()
+
+
+
+
+
+
+
+
+
+
+
+Out[6]:
+
+
+
+
+
+
+
+
+
+
+
+The manifest link seems to work. Let's add manifest links for all postcards to the dataframe:
+ +
+
+
+
+
+In [7]:
+
+
+
+
+
+
+meta['manifest_link'] = meta['akon_id'].apply(akon_id_to_manifest_link)
+
+
+
+
+
+In [8]:
+
+
+
+
+
+
+meta.sample(6)[['akon_id', 'manifest_link']]
+
+
+
+
+
+
+
+
+
+
+
+Out[8]:
+
+
+
+
+
+
+
+
+
+
+Collect Image Links¶
+
+
+
+
+
+Let's take a look at that manifest again:
+ +
+
+
+
+
+In [9]:
+
+
+
+
+
+
+r = requests.get(akon_id_to_manifest_link('AK024_176'))
+r.json()
+
+
+
+
+
+
+
+
+
+
+
+Out[9]:
+
+
+
+
+
+
+
+
+
+
+
+We need to collect all @ids from all resources from all images from all canvases.
That's tedious by hand. We'll use jsonpath-ng:
+
+
+
+
+In [10]:
+
+
+
+
+
+
+from jsonpath_ng import jsonpath, parse
+
+image_id_jp = parse('$.sequences[*].canvases[*].images[*].resource.@id')
+
+
+
+
+
+In [11]:
+
+
+
+
+
+
+[match.value for match in image_id_jp.find(r.json())]
+
+
+
+
+
+
+
+
+
+
+
+Out[11]:
+
+
+
+
+
+
+
+
+
+
+
+All of this in one function:
+ +
+
+
+
+
+In [12]:
+
+
+
+
+
+
+image_id_jp = parse('$.sequences[*].canvases[*].images[*].resource.@id')
+
+def image_links_for_manifest_link(manifest_link):
+ r = requests.get(manifest_link)
+ try:
+ json = r.json()
+ except:
+ # default to empty on exceptions - makes batch processing easier in pandas
+ json = {}
+ image_links = [match.value for match in image_id_jp.find(json)]
+ return image_links
+
+
+
+
+
+Let's test it:
+ +
+
+
+
+
+In [13]:
+
+
+
+
+
+
+random_akon_id = meta.sample().iloc[0]['akon_id']
+manifest_link = akon_id_to_manifest_link(random_akon_id)
+image_links_for_manifest_link(manifest_link)
+
+
+
+
+
+
+
+
+
+
+
+Out[13]:
+
+
+
+
+
+
+
+
+
+
+
+Looking good.
+ +
+
+
+
+
+Now let's add the image links to the dataframe...
+...actually, let's not do that now, because it takes a while (upwards of 10 minutes). Let's cheat instead, skip this step and load the resulting dataframe directly.
+ +
+
+
+
+
+In [14]:
+
+
+
+
+
+
+# %%time
+# meta['image_links'] = meta['manifest_link'].apply(image_links_for_manifest_link)
+
+
+
+
+
+In [15]:
+
+
+
+
+
+
+import json
+
+def load_json(s):
+ try:
+ return json.loads(s.replace("'", '"'))
+ except:
+ return []
+
+meta = pd.read_csv('postcards_with_image_links.csv.bz2', compression='bz2', converters={
+ 'image_links': load_json
+})
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [16]:
+
+
+
+
+
+
+meta.sample(10)
+
+
+
+
+
+
+
+
+
+
+
+Out[16]:
+
+
+
+
+
+
+
+
+
+
+Split Into Two Sets¶
+
+
+
+
+
+We'll split the dataframe into two: One with mountains, one without.
+ +
+
+
+
+
+In [17]:
+
+
+
+
+
+
+nomountain = meta[ meta['mountain'].isnull() ]
+mountain = meta[ ~ meta['mountain'].isnull() ]
+
+
+
+
+
+In [18]:
+
+
+
+
+
+
+len(meta), len(nomountain), len(mountain)
+
+
+
+
+
+
+
+
+
+
+
+Out[18]:
+
+
+
+
+
+
+
+
+
+
+
+Yeah, that adds up.
+ +
+
+
+
+
+Download¶
+
+
+
+
+
+Ok, so what's left to do?
+-
+
- Download all image data into two separate directories for training +
- Resize the images for the CNN used +
VGG16 and VGG19 expect 224x224 pixel RGB images.
+ +
+
+
+
+
+Luckily, IIIF allows us to request images already resized to our demands. That saves on bandwidth, time and code complexity.
+According to the standard we can use the size parameter to resize the image exactly to the dimensions we need.
The links, before and after, would be:
+https://iiif.onb.ac.at/images/AKON/AK024_176/176/full/full/0/native.jpg
https://iiif.onb.ac.at/images/AKON/AK024_176/176/full/224,224/0/native.jpg
+
+
+
+
+Let's try it:
+ +
+
+
+
+
+In [19]:
+
+
+
+
+
+
+r = requests.get('https://iiif.onb.ac.at/images/AKON/AK024_176/176/full/224,224/0/native.jpg')
+
+
+
+
+
+In [20]:
+
+
+
+
+
+
+from IPython.display import display, Image
+
+
+
+
+
+In [21]:
+
+
+
+
+
+
+display(Image(r.content))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+That looks about right.
+ +
+
+
+
+
+Download to file:
+ +
+
+
+
+
+In [22]:
+
+
+
+
+
+
+import shutil
+
+def download_to_file(url, filename):
+ with requests.get(url, stream=True) as r:
+ with open(filename, 'wb') as fh:
+ shutil.copyfileobj(r.raw, fh)
+
+def sized_link(iiif_url, size='224,224'):
+ frags = iiif_url.split('/')
+ frags[-3] = size
+ return '/'.join(frags)
+
+
+
+
+
+Test that:
+ +
+
+
+
+
+In [23]:
+
+
+
+
+
+
+link = sized_link('https://iiif.onb.ac.at/images/AKON/AK024_176/176/full/full/0/native.jpg')
+download_to_file(link, 'testimg.jpg')
+
+
+
+
+
+In [24]:
+
+
+
+
+
+
+with open('testimg.jpg', 'rb') as fh:
+ display(Image(fh.read()))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Create directories:
+ +
+
+
+
+
+In [25]:
+
+
+
+
+
+
+import os
+
+os.mkdir('./images')
+os.mkdir('./images/mountain')
+os.mkdir('./images/nomountain')
+
+
+
+
+
+Now let's download!
+ +
+
+
+
+
+In [26]:
+
+
+
+
+
+
+# For this demonstration we'll just take 10 images each
+for idx, row in mountain.sample(10).iterrows():
+ akon_id = row['akon_id']
+ for n, link in enumerate(row['image_links']):
+ small_image_link = sized_link(link)
+ file_name = f'./images/mountain/{akon_id}_{n}.jpg'
+ download_to_file(small_image_link, file_name)
+ print('.', end='')
+for idx, row in nomountain.sample(10).iterrows():
+ akon_id = row['akon_id']
+ for n, link in enumerate(row['image_links']):
+ small_image_link = sized_link(link)
+ file_name = f'./images/nomountain/{akon_id}_{n}.jpg'
+ download_to_file(small_image_link, file_name)
+ print('.', end='')
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+3.3 - Text - Download OCR Text¶
I need loads of text from old newspapers, preferably with loads of errors due to bad OCR.
+ + + +
+
+
+
+
+In order to get to this text, we have to
+-
+
- Find a newspaper issue we'd like to harvest +
- Download the IIIF manifest for this newspaper issue +
- Download the ALTO-XML files for this newspaper issue +
- Convert the ALTO-XML to TXT +
+
+
+
+
+Find a Newspaper Issue¶
+
+
+
+
+
+Let's take a look at the ONB Labs' historic newspapers
+ +
+
+
+
+
+In [1]:
+
+
+
+
+
+
+import pandas as pd
+
+meta = pd.read_csv('https://labs.onb.ac.at/gitlab/labs-team/raw-metadata/raw/master/anno_labs_issues.csv.bz2', compression='bz2')
+
+
+
+
+
+In [2]:
+
+
+
+
+
+
+meta.sample(10)
+
+
+
+
+
+
+
+
+
+
+
+Out[2]:
+
+
+
+
+
+
+
+
+
+
+Let's go with the Ost-Deutsche Post issue from the 30th of July 1863
+ +
+
+
+
+
+In [3]:
+
+
+
+
+
+
+manifest_id = 'ode18630730'
+
+
+
+
+
+Download the IIIF Manifest¶
+
+
+
+
+
+If we look at the SACHA API description, we see that the link for the IIIF manifest has to look like this:
+http://iiif.onb.ac.at/presentation/ANNO/ode18630730/manifest
+
+
+
+
+In [4]:
+
+
+
+
+
+
+import requests
+
+
+
+
+
+In [5]:
+
+
+
+
+
+
+r = requests.get(f'http://iiif.onb.ac.at/presentation/ANNO/{manifest_id}/manifest')
+
+
+
+
+
+In [6]:
+
+
+
+
+
+
+r.json()
+
+
+
+
+
+
+
+
+
+
+
+Out[6]:
+
+
+
+
+
+
+
+
+
+
+
+There's a lot of information in there. We need the info blocks with links to ALTO-XML resources.
+Let's use jsonpath-ng for that.
+ +
+
+
+
+
+In [7]:
+
+
+
+
+
+
+from jsonpath_ng import parse
+
+
+
+
+
+In [8]:
+
+
+
+
+
+
+def jp(http_response, parser):
+ return [match.value for match in parser.find(http_response.json())]
+
+
+
+
+
+In [9]:
+
+
+
+
+
+
+resource_parser = parse('$.sequences[*].canvases[*].otherContent[*].resources')
+
+
+
+
+
+In [10]:
+
+
+
+
+
+
+jp(r, resource_parser)
+
+
+
+
+
+
+
+
+
+
+
+Out[10]:
+
+
+
+
+
+
+
+
+
+
+
+Not quite there yet.
+ +
+
+
+
+
+In [11]:
+
+
+
+
+
+
+all_resources = parse('$.sequences[*].canvases[*].otherContent[*].resources[*].resource')
+
+
+
+
+
+In [12]:
+
+
+
+
+
+
+jp(r, all_resources)
+
+
+
+
+
+
+
+
+
+
+
+Out[12]:
+
+
+
+
+
+
+
+
+
+
+
+Filter just the ones with format application/xml+alto, and there only the @id:
+
+
+
+
+In [13]:
+
+
+
+
+
+
+ids = [d['@id'] for d in jp(r, all_resources) if d['format'] == 'application/xml+alto']
+
+
+
+
+
+In [14]:
+
+
+
+
+
+
+ids
+
+
+
+
+
+
+
+
+
+
+
+Out[14]:
+
+
+
+
+
+
+
+
+
+
+
+Download the ALTO Files¶
+
+
+
+
+
+In [15]:
+
+
+
+
+
+
+alto_storage = {}
+
+for xml_link in ids:
+ r = requests.get(xml_link)
+ if r.ok:
+ alto_storage[xml_link] = r.text
+
+
+
+
+
+In [16]:
+
+
+
+
+
+
+alto_storage
+
+
+
+
+
+
+
+
+
+
+
+Out[16]:
+
+
+
+
+
+
+
+
+
+
+
+In [17]:
+
+
+
+
+
+
+r
+
+
+
+
+
+
+
+
+
+
+
+Out[17]:
+
+
+
+
+
+
+
+
+
+
+
+In [18]:
+
+
+
+
+
+
+r.ok
+
+
+
+
+
+
+
+
+
+
+
+Out[18]:
+
+
+
+
+
+
+
+
+
+
+
+Uh oh.
+ +
+
+
+
+
+Convert the ALTO-XML to TXT¶
+
+
+
+
+
+In [19]:
+
+
+
+
+
+
+import alto_tools
+
+def alto_extract_text_lines(xml, xmlns):
+ text_lines = []
+ nsdict = {'alto': xmlns}
+ for lines in xml.iterfind('.//alto:TextLine', nsdict):
+ words = [line.attrib.get('CONTENT') for line in lines.findall('alto:String', nsdict)]
+ text_lines.append(' '.join(words))
+ return '\n'.join(text_lines)
+
+def alto_to_text(raw_alto_text):
+ alto, xml, xmlns = alto_tools.alto_parse(raw_alto_text)
+ return alto_extract_text_lines(xml, xmlns)
+
+
+
+
+
+In [20]:
+
+
+
+
+
+
+print(alto_to_text(f'http://iiif.onb.ac.at/presentation/ANNO/{manifest_id}/resource/00000002.xml'))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+