{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "5b24e324-6659-482d-8d82-39c1d604f0d3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import cv2 as cv\n",
    "import re\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.mixture import GaussianMixture\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import lxml.etree as et\n",
    "# from scipy.optimize import curve_fit\n",
    "# import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d6809d5-f111-455e-8add-7e3793a84266",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# filenames where grouping entries failed:\n",
    "failed_names_14376 = [\n",
    "    '14.376_00000049.jpeg',\n",
    "    '14.376_00000100.jpeg',\n",
    "    '14.376_00000104.jpeg',\n",
    "    '14.376_00000114.jpeg',\n",
    "    '14.376_00000127.jpeg',\n",
    "    '14.376_00000129.jpeg',\n",
    "    '14.376_00000133.jpeg',\n",
    "    '14.376_00000134.jpeg',\n",
    "    '14.376_00000182.jpeg',\n",
    "    '14.376_00000221.jpeg',\n",
    "    '14.376_00000226.jpeg',\n",
    "    '14.376_00000249.jpeg',\n",
    "    '14.376_00000270.jpeg',\n",
    "    '14.376_00000284.jpeg',\n",
    "    '14.376_00000297.jpeg',\n",
    "    '14.376_00000301.jpeg',\n",
    "    '14.376_00000324.jpeg',\n",
    "    '14.376_00000328.jpeg',\n",
    "    '14.376_00000338.jpeg',\n",
    "    '14.376_00000371.jpeg',\n",
    "    '14.376_00000381.jpeg',\n",
    "    '14.376_00000399.jpeg',\n",
    "    '14.376_00000414.jpeg',\n",
    "    '14.376_00000428.jpeg',\n",
    "    '14.376_00000438.jpeg',\n",
    "    '14.376_00000441.jpeg',\n",
    "    '14.376_00000443.jpeg'\n",
    "]\n",
    "\n",
    "failed_names_14377 = [\n",
    "    '14.377_00000017.jpeg',\n",
    "    '14.377_00000025.jpeg',\n",
    "    '14.377_00000054.jpeg',\n",
    "    '14.377_00000064.jpeg',\n",
    "    '14.377_00000078.jpeg',\n",
    "    '14.377_00000079.jpeg',\n",
    "    '14.377_00000093.jpeg',\n",
    "    '14.377_00000096.jpeg',\n",
    "    '14.377_00000097.jpeg',\n",
    "    '14.377_00000098.jpeg',\n",
    "    '14.377_00000112.jpeg',\n",
    "    '14.377_00000122.jpeg',\n",
    "    '14.377_00000129.jpeg',\n",
    "    '14.377_00000130.jpeg',\n",
    "    '14.377_00000131.jpeg',\n",
    "    '14.377_00000132.jpeg',\n",
    "    '14.377_00000133.jpeg',\n",
    "    '14.377_00000134.jpeg',\n",
    "    '14.377_00000135.jpeg',\n",
    "    '14.377_00000136.jpeg',\n",
    "    '14.377_00000137.jpeg',\n",
    "    '14.377_00000138.jpeg',\n",
    "    '14.377_00000139.jpeg',\n",
    "    '14.377_00000153.jpeg',\n",
    "    '14.377_00000155.jpeg',\n",
    "    '14.377_00000172.jpeg',\n",
    "    '14.377_00000179.jpeg',\n",
    "    '14.377_00000180.jpeg',\n",
    "    '14.377_00000181.jpeg',\n",
    "    '14.377_00000186.jpeg',\n",
    "    '14.377_00000199.jpeg',\n",
    "    '14.377_00000231.jpeg',\n",
    "    '14.377_00000259.jpeg',\n",
    "    '14.377_00000273.jpeg',\n",
    "    '14.377_00000281.jpeg',\n",
    "    '14.377_00000291.jpeg',\n",
    "    '14.377_00000292.jpeg',\n",
    "    '14.377_00000294.jpeg',\n",
    "    '14.377_00000295.jpeg',\n",
    "    '14.377_00000329.jpeg',\n",
    "    '14.377_00000366.jpeg',\n",
    "    '14.377_00000386.jpeg',\n",
    "    '14.377_00000387.jpeg',\n",
    "    '14.377_00000392.jpeg',\n",
    "    '14.377_00000407.jpeg',\n",
    "    '14.377_00000433.jpeg',\n",
    "]\n",
    "\n",
    "failed_names_14378 = [\n",
    "    '14.378_00000015.jpeg',\n",
    "    '14.378_00000059.jpeg',\n",
    "    '14.378_00000064.jpeg',\n",
    "    '14.378_00000076.jpeg',\n",
    "    '14.378_00000079.jpeg',\n",
    "    '14.378_00000091.jpeg',\n",
    "    '14.378_00000101.jpeg',\n",
    "    '14.378_00000111.jpeg',\n",
    "    '14.378_00000125.jpeg',\n",
    "    '14.378_00000139.jpeg',\n",
    "    '14.378_00000147.jpeg',\n",
    "    '14.378_00000151.jpeg',\n",
    "    '14.378_00000152.jpeg',\n",
    "    '14.378_00000153.jpeg',\n",
    "    '14.378_00000155.jpeg',\n",
    "    '14.378_00000169.jpeg',\n",
    "    '14.378_00000171.jpeg',\n",
    "    '14.378_00000173.jpeg',\n",
    "    '14.378_00000205.jpeg',\n",
    "    '14.378_00000210.jpeg',\n",
    "    '14.378_00000246.jpeg',\n",
    "    '14.378_00000251.jpeg',\n",
    "    '14.378_00000258.jpeg',\n",
    "    '14.378_00000261.jpeg',\n",
    "    '14.378_00000266.jpeg',\n",
    "    '14.378_00000274.jpeg',\n",
    "    '14.378_00000282.jpeg',\n",
    "    '14.378_00000283.jpeg',\n",
    "    '14.378_00000310.jpeg',\n",
    "    '14.378_00000322.jpeg',\n",
    "    '14.378_00000377.jpeg',\n",
    "    '14.378_00000388.jpeg',\n",
    "    '14.378_00000392.jpeg',\n",
    "    '14.378_00000393.jpeg',\n",
    "    '14.378_00000399.jpeg',\n",
    "    '14.378_00000408.jpeg',\n",
    "    '14.378_00000410.jpeg',\n",
    "    '14.378_00000433.jpeg',\n",
    "    '14.378_00000452.jpeg',\n",
    "    '14.378_00000461.jpeg',\n",
    "    '14.378_00000462.jpeg',\n",
    "    '14.378_00000465.jpeg',\n",
    "    '14.378_00000466.jpeg',\n",
    "    '14.378_00000498.jpeg',\n",
    "    '14.378_00000513.jpeg',\n",
    "    '14.378_00000529.jpeg',\n",
    "    '14.378_00000579.jpeg',\n",
    "]\n",
    "\n",
    "joint_failed_names = failed_names_14376 + failed_names_14377 + failed_names_14378"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba26e3ae-d2d9-4876-94a9-af1a5f29f8f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Subcategories which are not yet recognized\n",
    "missing_subcats = [\n",
    "    'Historia Iudaïca, Assyria, Medorum, Persica vetus,'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7535107-5d1e-43d9-b0ed-077c306a73c1",
   "metadata": {},
   "source": [
    "# Klassenstruktur für eine Seite und ihre Untereinheiten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "id": "97ce5633-934c-4fbb-a71f-7e4710ff4211",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def prepare_string(s):\n",
    "    new = s.lower()\n",
    "    new = re.sub(r'[àáâãäå]', 'a', new)\n",
    "    new = re.sub(r'[èéêë]', 'e', new)\n",
    "    new = re.sub(r'[ìíîï]', 'i', new)\n",
    "    new = re.sub(r'[òóôõö]', 'o', new)\n",
    "    new = re.sub(r'[ùúûü]', 'u', new)\n",
    "    new = re.sub('\\.|,|;|\\s', '', new)\n",
    "    return new\n",
    "\n",
    "class Page:\n",
    "    def __init__(self, page_xml_string):\n",
    "        self.XMLSources = XMLSources(page_xml_string)\n",
    "        self.isMainMatter = self.check_main_matter()\n",
    "        self.Header = Header(self.XMLSources, self.isMainMatter)\n",
    "        self.Entries = Entries(self.Header, self.XMLSources, self.isMainMatter)\n",
    "        self.RemainingLines = RemainingLines(self.Header, self.Entries, self.XMLSources)\n",
    "    \n",
    "    def __str__(self):\n",
    "        return f'Filename: {self.XMLSources.PageXML.find_all(\"Page\")[0].attrs[\"imageFilename\"]}\\n' \\\n",
    "            f'{self.Header}'\n",
    "    \n",
    "    def check_main_matter(self):\n",
    "        filename = self.get_filename()\n",
    "        num = int(filename[-8:-5])\n",
    "        if '14.376' in filename:\n",
    "            return num >= 23\n",
    "        else: # 14.377 oder 14.378\n",
    "            return num >= 11\n",
    "    \n",
    "    def get_filename(self):\n",
    "        return self.XMLSources.PageXML.find(\"Page\").attrs[\"imageFilename\"]\n",
    "\n",
    "class Line:\n",
    "    def __init__(self, xml_line):\n",
    "        self.ID = xml_line.attrs['id']\n",
    "        polygon_string = xml_line.Coords.attrs['points']\n",
    "        polygon_tuples = polygon_string.split(' ')\n",
    "        polygon = [tup.split(',') for tup in polygon_tuples]\n",
    "        self.BoundingPolygon = np.array(polygon, dtype='int')\n",
    "        baseline_string = xml_line.Baseline.attrs['points']\n",
    "        baseline_tuples = baseline_string.split(' ')\n",
    "        baseline = [tup.split(',') for tup in baseline_tuples]\n",
    "        self.Baseline = np.array(baseline, dtype='int')\n",
    "        self.Text = xml_line.find_all('TextEquiv')[-1].Unicode.text\n",
    "        self.Tags = xml_line.attrs['custom']\n",
    "    \n",
    "    def __str__(self):\n",
    "        # return f'ID: {self.ID}, text: {self.Text}\\n'\n",
    "        return self.Text\n",
    "    \n",
    "    # def get_polygon_bb(self):\n",
    "    #     pass\n",
    "\n",
    "class Header:\n",
    "    def __init__(self, XMLSources, is_main_matter):\n",
    "        self.XMLSources = XMLSources\n",
    "        self.PageNumber = self.get_page_number()\n",
    "        self.Format = self.get_format()\n",
    "        if is_main_matter:\n",
    "            self.Category = self.get_category()\n",
    "            self.Subcategory = self.get_subcategory()\n",
    "        else:\n",
    "            self.Category = [[], []]\n",
    "            self.Subcategory = [[], []]\n",
    "        self.Section = self.get_section()\n",
    "        self.MarginNote = self.get_margin_note()\n",
    "        self.isValid = self.check_validity()\n",
    "    \n",
    "    def __str__(self):\n",
    "        # header_string = f'Has valid header: {self.isValid}\\n'\n",
    "        header_string = ''\n",
    "        if self.isValid:\n",
    "            header_string += f'Page number(s): {\"\".join(self.PageNumber[0])}\\n' \\\n",
    "            f'Format: {\" \".join(self.Format[0])}\\n' \\\n",
    "            f'Category: {\" \".join(self.Category[0])}\\n' \\\n",
    "            f'Subcategory: {\" \".join(self.Subcategory[0])}'\n",
    "        return header_string\n",
    "\n",
    "    def get_all_ids(self):\n",
    "        lines = []\n",
    "        lines += self.PageNumber[1] + self.Format[1] + self.Category[1] + self.Subcategory[1] + self.Section[1] + self.MarginNote[1]\n",
    "        ids = [l.ID for l in lines]\n",
    "        return ids\n",
    "    \n",
    "    def print_pagenumber(self):\n",
    "        page_numbers = self.PageNumber[0]\n",
    "        if len(page_numbers) > 0:\n",
    "            return page_numbers[0]\n",
    "        else:\n",
    "            return ''\n",
    "    \n",
    "    def print_category(self):\n",
    "        categories = self.Category[0]\n",
    "        if len(categories) > 0:\n",
    "            return categories[0]\n",
    "        else:\n",
    "            return ''\n",
    "    \n",
    "    def print_subcategory(self):\n",
    "        subcategories = self.Subcategory[0]\n",
    "        if len(subcategories) > 0:\n",
    "            return subcategories[0]\n",
    "        else:\n",
    "            return ''\n",
    "    \n",
    "    def print_format(self):\n",
    "        formats = self.Format[0]\n",
    "        if len(formats) > 0:\n",
    "            return formats[0]\n",
    "        else:\n",
    "            return ''\n",
    "    \n",
    "    def check_validity(self):\n",
    "        hasPageNumber = self.PageNumber[0]\n",
    "        # hasFormat = self.Format[0]\n",
    "        hasCategory = self.Category[0]\n",
    "        # hasSubcategory = self.Subcategory[0]\n",
    "        # check if a header line has more than one tag\n",
    "        # ids = self.get_all_ids()\n",
    "        # id_set = set(ids)\n",
    "        # if len(ids) != len(id_set):\n",
    "        #     print('more than one tag for header lines')\n",
    "        categories, category_lines = self.Category\n",
    "        subcategory_line_IDs = [subcat_l.ID for subcat_l in self.Subcategory[1]]\n",
    "        for cat, cat_l in zip(categories, category_lines):\n",
    "            if cat_l.ID in subcategory_line_IDs:\n",
    "                category_lines.remove(cat_l)\n",
    "                categories.remove(cat)\n",
    "        self.Category = categories, category_lines\n",
    "        return bool(hasPageNumber and hasCategory)\n",
    "    \n",
    "    def get_width_height(self):\n",
    "        page = self.XMLSources.PageXML.find_all('Page')[0]\n",
    "        w, h = int(page.attrs['imageWidth']), int(page.attrs['imageHeight'])\n",
    "        return w, h\n",
    "\n",
    "    def get_page_number(self):\n",
    "        page_regex = '^[IVX]+\\Z|^\\d+\\**\\Z|^\\d+⁎*\\Z|^[a-h]\\Z|^\\*+\\Z'\n",
    "        textlines = self.XMLSources.PageXML.find_all('TextLine')\n",
    "        textline_objects = [Line(line) for line in textlines]\n",
    "        sorted_textlines = sorted(textline_objects, key=lambda x: x.Baseline[0][0])\n",
    "        w, h = self.get_width_height()\n",
    "        w_cut, h_cut = 0.75, 0.1\n",
    "        page_numbers = []\n",
    "        page_number_lines = []\n",
    "        for line in sorted_textlines:\n",
    "            avg_x, avg_y = np.mean(line.Baseline, axis=0)\n",
    "            text = line.Text\n",
    "            prep_text = re.sub('\\.|\\s', '', text)\n",
    "            m = re.match(page_regex, prep_text)\n",
    "            if m and (avg_y < h_cut * h) and ((avg_x > w_cut * w) or (avg_x < (1 - w_cut) * w)):\n",
    "                page_numbers.append(m.group(0))\n",
    "                page_number_lines.append(line)\n",
    "        \n",
    "        return page_numbers, page_number_lines\n",
    "    \n",
    "    def get_format(self):\n",
    "        prep_formats = ['infolio', 'inquarto', 'inoctavo', '&minforma']\n",
    "        true_formats = ['Folio', 'Quarto', 'Octavo und kleiner', 'Octavo und kleiner']\n",
    "        textlines = self.XMLSources.PageXML.find_all('TextLine')\n",
    "        w, h = self.get_width_height()\n",
    "        w_cut, h_cut = 0.75, 0.2\n",
    "        formats = []\n",
    "        format_lines = []\n",
    "        for line in textlines:\n",
    "            l = Line(line)\n",
    "            avg_x, avg_y = np.mean(l.Baseline, axis=0)\n",
    "            text = l.Text\n",
    "            cand_format = prepare_string(text)\n",
    "            if (cand_format in prep_formats) and (avg_y < h_cut * h) and ((avg_x > w_cut * w) or (avg_x < (1 - w_cut) * w)):\n",
    "                formats.append(true_formats[prep_formats.index(cand_format)])\n",
    "                format_lines.append(l)\n",
    "        return formats, format_lines\n",
    "    \n",
    "    def get_category(self):\n",
    "        cat_fp = 'XML_Aufbereitung_Kategorien/categories.json'\n",
    "        norm_cat_fp = 'XML_Aufbereitung_Kategorien/normed_categories.json'\n",
    "        with open(cat_fp) as fh:\n",
    "            true_categories = json.load(fh)\n",
    "        with open(norm_cat_fp) as fh:\n",
    "            normed_categories = json.load(fh)\n",
    "        \n",
    "        textlines = self.XMLSources.PageXML.find_all('TextLine')\n",
    "        # w, h = self.get_width_height()\n",
    "        # w_cut, h_cut = 0.1, 0.1\n",
    "        categories = []\n",
    "        category_lines = []\n",
    "        for line in textlines:\n",
    "            l = Line(line)\n",
    "            # avg_x, avg_y = np.mean(l.Baseline, axis=0)\n",
    "            text = l.Text\n",
    "            # TODO: implement similarity comparison and location check\n",
    "            prepared_string = prepare_string(text)\n",
    "            prep_text = re.sub('\\d', '', prepared_string)\n",
    "            # if a 4-digit number was removed from the text we're probably looking at a line from an entry\n",
    "            if len(prepared_string) - len(prep_text) > 3:\n",
    "                continue\n",
    "            prep_text_in_cats = [prep_text in prepare_string(cat) for cat in true_categories]\n",
    "            # code snippet for location check:\n",
    "            # ... and (avg_y < h_cut * h) and (abs(avg_x - w/2) < w_cut * w)\n",
    "            if sum(prep_text_in_cats) and len(prep_text) > 3:\n",
    "                categories.append(normed_categories[prep_text_in_cats.index(True)])\n",
    "                # categories.append(text)\n",
    "                category_lines.append(l)\n",
    "        return categories, category_lines\n",
    "    \n",
    "    def get_subcategory(self):\n",
    "        cat_fp = 'XML_Aufbereitung_Kategorien/categories.json'\n",
    "        subcat_fp = 'XML_Aufbereitung_Kategorien/subcategories.json'\n",
    "        normed_subcat_fp = 'XML_Aufbereitung_Kategorien/normed_subcategories.json'\n",
    "        with open(cat_fp) as fh:\n",
    "            true_categories = json.load(fh)\n",
    "        with open(subcat_fp) as fh:\n",
    "            true_subcategories = json.load(fh)\n",
    "        with open(normed_subcat_fp) as fh:\n",
    "            normed_subcategories = json.load(fh)\n",
    "        \n",
    "        prep_cats = [prepare_string(cat) for cat in true_categories]\n",
    "        textlines = self.XMLSources.PageXML.find_all('TextLine')\n",
    "        subcategories = []\n",
    "        subcategory_lines = []\n",
    "        w, h = self.get_width_height()\n",
    "        w_cut, h_cut = 0.2, 0.2\n",
    "        for line in textlines:\n",
    "            l = Line(line)\n",
    "            text = l.Text\n",
    "            avg_x, avg_y = np.mean(l.Baseline, axis=0)\n",
    "            prepared_string = prepare_string(text)\n",
    "            prep_text = re.sub('\\d', '', prepared_string)\n",
    "            # if a 4-digit number was removed from the text we're probably looking at a line from an entry\n",
    "            if len(prepared_string) - len(prep_text) > 3:\n",
    "                continue\n",
    "            # if there is no subcategory\n",
    "            # if prep_text == 'codicesmanuscripti' \\\n",
    "            #     or prep_text == 'imaginesvariæincisæ' \\\n",
    "            #     or prep_text =='effigierumincisarcollectio' \\\n",
    "            #     or prep_text == 'imaginumdelineatarcollectio':\n",
    "            #     subcategories.append(text)\n",
    "            #     subcategory_lines.append(l)\n",
    "            # if category skip this line\n",
    "            if prep_text in prep_cats:\n",
    "                continue\n",
    "            prep_text_in_subcat = [prep_text in prepare_string(subcat) for subcat in true_subcategories]\n",
    "            # TODO: implement similarity comparison and location check\n",
    "            \n",
    "            if sum(prep_text_in_subcat) and len(prep_text) > 3 and (avg_y < h_cut * h) and (abs(avg_x - w/2) < w_cut * w):\n",
    "                subcategories.append(normed_subcategories[prep_text_in_subcat.index(True)])\n",
    "                subcategory_lines.append(l)\n",
    "        return subcategories, subcategory_lines\n",
    "    \n",
    "    def get_section(self):\n",
    "        imagines_sections = ['Ecole de Florence', 'Ecole Romaine', 'Ecole Venitienne', 'Ecole de Bologne', 'Ecole de Gennes & de Naples', 'Ecole de Lombardie']\n",
    "        prep_sections = [prepare_string(sec) for sec in imagines_sections]\n",
    "        textlines = self.XMLSources.PageXML.find_all('TextLine')\n",
    "        sections = []\n",
    "        section_lines = []\n",
    "        for line in textlines:\n",
    "            l = Line(line)\n",
    "            text = l.Text\n",
    "            prepared_string = prepare_string(text)\n",
    "            if prepared_string in prep_sections:\n",
    "                sections.append(imagines_sections[prep_sections.index(prepared_string)])\n",
    "                section_lines.append(l)\n",
    "        return sections, section_lines\n",
    "    \n",
    "    def get_margin_note(self):\n",
    "        maitre_words = ['Maitres', \"d'Italie\", \"d'Allemagne\", '& des', 'Paysbas', 'de', 'France']\n",
    "        prep_maitre_words = [prepare_string(w) for w in maitre_words]\n",
    "        w, h = self.get_width_height()\n",
    "        w_cut, h_cut = 0.75, 0.2\n",
    "        textlines = self.XMLSources.PageXML.find_all('TextLine')\n",
    "        notes = []\n",
    "        note_lines = []\n",
    "        \n",
    "        for line in textlines:\n",
    "            l = Line(line)\n",
    "            text = prepare_string(l.Text)\n",
    "            avg_x, avg_y = np.mean(l.Baseline, axis=0)\n",
    "            if (text in prep_maitre_words) and (avg_y < h_cut * h) and ((avg_x > w_cut * w) or (avg_x < (1 - w_cut) * w)):\n",
    "                notes.append(maitre_words[prep_maitre_words.index(text)])\n",
    "                note_lines.append(l)\n",
    "        return notes, note_lines\n",
    "\n",
    "class Entries:\n",
    "    def __init__(self, Header, XMLSources, is_main_matter):\n",
    "        self.Entries = self.group_entries(Header, XMLSources, is_main_matter)\n",
    "        self.Centers = self.calculate_centers()\n",
    "    \n",
    "    def __str__(self):\n",
    "        entry_string = '\\n'\n",
    "        for entry in self.Entries:\n",
    "            # entry_string += f'Entry of length {len(entry)}:\\n'\n",
    "            for line in entry:\n",
    "                if entry.index(line) > 0:\n",
    "                    entry_string += '\\t'\n",
    "                entry_string += f'{line}\\n'\n",
    "            entry_string += '\\n'\n",
    "        return entry_string\n",
    "    \n",
    "    def get_all_ids(self):\n",
    "        lines = []\n",
    "        for entry in self.Entries:\n",
    "            for line in entry:\n",
    "                lines.append(line)\n",
    "        IDs = [l.ID for l in lines]\n",
    "        return IDs\n",
    "    \n",
    "    def group_entries(self, Header, XMLSources, is_main_matter):\n",
    "        entries = []\n",
    "        manual_groups = []\n",
    "        if Header.isValid and is_main_matter:\n",
    "            all_lines = [Line(line) for line in XMLSources.PageXML.find_all('TextLine')]\n",
    "            header_ids = Header.get_all_ids()\n",
    "            no_header_lines = list(filter(lambda x: x.ID not in header_ids, all_lines))\n",
    "            sorted_lines = sorted(no_header_lines, key=lambda x: x.Baseline[0][1])\n",
    "            w, h = Header.get_width_height()\n",
    "            filter_lines = list(filter(lambda x: x.Baseline[0][0] < 0.4 * w, sorted_lines))\n",
    "            \n",
    "            # check for manual layout information\n",
    "            if XMLSources.PageXML.find(\"Page\").attrs[\"imageFilename\"] in joint_failed_names:\n",
    "                tag_format_re = re.compile('(\\w+) (\\{.+?\\})')\n",
    "                tag_length_re = re.compile('length:(\\d+?);')\n",
    "\n",
    "                new_filter_lines = []\n",
    "                for line in filter_lines:\n",
    "                    tags = tag_format_re.findall(line.Tags)\n",
    "                    has_add = False\n",
    "                    has_contd = False\n",
    "                    for tag_values in tags:\n",
    "                        tag_length_str_lis = tag_length_re.findall(tag_values[1])\n",
    "                        if len(tag_length_str_lis) > 0:\n",
    "                            tag_length = int(tag_length_str_lis[0])\n",
    "                            line_length = len(line.Text)\n",
    "                            cut_off = 0.8\n",
    "                            if tag_values[0] == 'add' and tag_length >= cut_off * line_length:\n",
    "                                has_add = True\n",
    "                            if tag_values[0] == 'continued':\n",
    "                                has_contd = True\n",
    "                    if has_add:\n",
    "                        if not has_contd:\n",
    "                            # new entry\n",
    "                            manual_groups.append([line])\n",
    "                        else:\n",
    "                            # append to entry from line above\n",
    "                            if len(manual_groups) > 0:\n",
    "                                manual_groups[-1].append(line)\n",
    "                            else:\n",
    "                                new_filter_lines.append(line)\n",
    "                    else:\n",
    "                        new_filter_lines.append(line)\n",
    "                filter_lines = new_filter_lines\n",
    "\n",
    "            if not filter_lines:\n",
    "                return manual_groups\n",
    "            bl_centers_x = [np.mean(l.Baseline, axis=0)[0] for l in filter_lines]\n",
    "            bl_left_x = [l.Baseline[0][0] for l in filter_lines]\n",
    "            # print(bl_centers_x)\n",
    "            bl_left_x_mean = np.mean(bl_left_x)\n",
    "            # print('bl_left_x_mean:', bl_left_x_mean)\n",
    "            left_outliers_removed = []\n",
    "            for line, center, left in zip(filter_lines, bl_centers_x, bl_left_x):\n",
    "                if (center > bl_left_x_mean + 50 and left < bl_left_x_mean + 700):\n",
    "                    left_outliers_removed.append(line)\n",
    "            filter_coord = [line.Baseline[0][0] for line in left_outliers_removed]\n",
    "            if not filter_coord:\n",
    "                return manual_groups\n",
    "            if len(filter_coord) == 1:\n",
    "                return sorted(manual_groups + [left_outliers_removed], key=lambda x: x[0].Baseline[0][1])\n",
    "            X = np.array(filter_coord)[:, np.newaxis]\n",
    "            # for line in left_outliers_removed:\n",
    "            #     print(line)\n",
    "            # print(X)\n",
    "            gm = GaussianMixture(n_components=2, random_state=42).fit(X)\n",
    "            preds = gm.predict(X)\n",
    "            # print(gm.means_)\n",
    "            # print(preds)\n",
    "            beg_marker = int(not (gm.means_[0, 0] < gm.means_[1, 0]))\n",
    "            # print(beg_marker)\n",
    "            # print('Means:', gm.means_[0][0], gm.means_[1][0])\n",
    "            # print('Covariances:', gm.covariances_[0, 0, 0], gm.covariances_[1, 0, 0])\n",
    "            for line, pred in zip(left_outliers_removed, preds):\n",
    "                if pred == beg_marker or left_outliers_removed.index(line) == 0:\n",
    "                    entries.append([line])\n",
    "                else:\n",
    "                    entries[-1].append(line)\n",
    "            \n",
    "        if len(manual_groups) > 0:\n",
    "            entries += manual_groups\n",
    "            entries = sorted(entries, key=lambda x: x[0].Baseline[0][1])\n",
    "        return entries\n",
    "    \n",
    "    def calculate_centers(self):\n",
    "        centers = []\n",
    "        for lines in self.Entries:\n",
    "            line_centers = [np.mean(l.Baseline, axis=0) for l in lines]\n",
    "            x, y = np.mean(line_centers, axis=0)\n",
    "            centers.append((x, y))\n",
    "        return centers\n",
    "\n",
    "class RemainingLines:\n",
    "    def __init__(self, Header, Entries, XMLSources):\n",
    "        self.Lines = self.get_remaining_lines(Header, Entries, XMLSources)\n",
    "    \n",
    "    def __str__(self):\n",
    "        rem_string = 'Remaining Lines:\\n'\n",
    "        for line in self.Lines:\n",
    "            rem_string += f'{line}\\n'\n",
    "        return rem_string\n",
    "    \n",
    "    def get_remaining_lines(self, Header, Entries, XMLSources):\n",
    "        all_lines = [Line(line) for line in XMLSources.PageXML.find_all('TextLine')]\n",
    "        header_ids = Header.get_all_ids()\n",
    "        no_header_lines = list(filter(lambda x: x.ID not in header_ids, all_lines))\n",
    "        entry_ids = Entries.get_all_ids()\n",
    "        no_entry_lines = list(filter(lambda x: x.ID not in entry_ids, no_header_lines))\n",
    "        return no_entry_lines\n",
    "\n",
    "class XMLSources:\n",
    "    def __init__(self, page_string):\n",
    "        with open(page_string, 'r') as page_xml:\n",
    "            content = page_xml.readlines()\n",
    "            content = \"\".join(content)\n",
    "            bs_content = bs(content, \"lxml-xml\")\n",
    "            self.PageXML = bs_content\n",
    "        # with open(tei_string, 'r') as tei_xml:\n",
    "        #     content = tei_xml.readlines()\n",
    "        #     content = \"\".join(content)\n",
    "        #     bs_content = bs(content, \"lxml-xml\")\n",
    "        #     self.TEIXML = bs_content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "id": "90696c0e-91ea-4968-89fe-06700535a4e9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.378_00000539.jpeg\n",
      "1359\n",
      "\n",
      "Imagines Incisæ\n",
      "Opera Integra Pictorum, Sculptorum & Cælatorum\n",
      "(['Ecole de Lombardie', 'Ecole de Bologne'], [<__main__.Line object at 0x7f78fdd918a0>, <__main__.Line object at 0x7f78fdd92170>])\n",
      "(['Maitres', \"d'Italie\"], [<__main__.Line object at 0x7f78fdd90f70>, <__main__.Line object at 0x7f78fdd91810>])\n",
      "\n",
      "XXXI 1 Vol. contenant les Oeuvres de\n",
      "\tMartin Rosa de Sebenigo en Dalmatie Graveur\n",
      "\n",
      "XXXII 1 Vol. contenant les Oeuvres de\n",
      "\tAntoine Correge Peintre disciple d'André Mantegne\n",
      "\n",
      "XXXIII 1 Vol. contenant les Oeuvres de\n",
      "\tFrançois Mazzuoli dit le Parmesan. Peintre. n. XXXIII.\n",
      "\t.........Raccolta di vari dissegni, posti in Stampa da An. M. Zanetti.\n",
      "\n",
      "XXXIV 1 Vol. contenant les Oeuvres de\n",
      "\tEnea Vico Parmesan Peintre.\n",
      "\n",
      "XXXV 1 Vol. contenant les Oeuvres de\n",
      "\tIules Bonasone de Bologne Peintre & Graveur.\n",
      "\n",
      "XXXVI 7 Vol. contenant les Oeuvres de\n",
      "\tLouis, Annibal, & Augustin Carrache Peintres.\n",
      "\n",
      "XXXVII 2 Vol. contenant les Oeuvres de\n",
      "\tGuido Rheni l'un des plus illustres Peintres de l'Ecole\n",
      "\tdes Carrache, qui a gravé plusieurs de ses Ouvrages.\n",
      "\tSimon Cantarini dit le Pesarese\n",
      "\tLe Sirani; & quelques autres disciples du Guide.\n",
      "\n",
      "XXXVIII 1 Vol. contenant les Oeuvres de\n",
      "\tDominique Zampieri dit le Dominiquain Peintre.\n",
      "\n",
      "XXXVIIII 1 Vol. contenant les Oeuvres de\n",
      "\tFrançois Albane Peintre de l'Ecole des Carrache\n",
      "\n",
      "\n",
      "Remaining Lines:\n",
      "n. 968.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# ex_page = sorted(pages_14378, key=lambda x: x.get_filename())[305]\n",
    "# ex_page = Page('../../digital-edition/Step0_Export/Cod. 14.378/page/14.378_00000316.xml')\n",
    "# ex_page_main = Page('../../digital-edition/Step0_Export/Cod. 14.376/page/14.376_00000025.xml')\n",
    "ex_page_front = Page('../../digital-edition/Step0_Export/Cod. 14.378/page/14.378_00000539.xml')\n",
    "print(ex_page_front.get_filename())\n",
    "print(ex_page_front.Header.print_pagenumber())\n",
    "print(ex_page_front.Header.print_format())\n",
    "print(ex_page_front.Header.print_category())\n",
    "print(ex_page_front.Header.print_subcategory())\n",
    "print(ex_page_front.Header.Section)\n",
    "print(ex_page_front.Header.MarginNote)\n",
    "print(ex_page_front.Entries)\n",
    "print(ex_page_front.RemainingLines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "25e8978d-c62f-4b4c-88e1-be9f2c2b80c2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "page_dir_14376 = pathlib.Path('../../digital-edition/Step0_Export/Cod. 14.376/page/')\n",
    "pages_14376 = []\n",
    "\n",
    "for xml in page_dir_14376.glob('*.xml'):\n",
    "    try:\n",
    "        pages_14376.append(Page(xml))\n",
    "    except Exception as e:\n",
    "        print(xml)\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "57168e21-394a-44ed-b8e1-632a7ad76863",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "page_dir_14377 = pathlib.Path('../../digital-edition/Step0_Export/Cod. 14.377/page/')\n",
    "pages_14377 = []\n",
    "\n",
    "for xml in page_dir_14377.glob('*.xml'):\n",
    "    try:\n",
    "        pages_14377.append(Page(xml))\n",
    "    except Exception as e:\n",
    "        print(xml)\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "id": "edc2700e-7590-4f7f-a773-b6363552b473",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "page_dir_14378 = pathlib.Path('../../digital-edition/Step0_Export/Cod. 14.378/page/')\n",
    "pages_14378 = []\n",
    "\n",
    "for xml in page_dir_14378.glob('*.xml'):\n",
    "    try:\n",
    "        pages_14378.append(Page(xml))\n",
    "    except Exception as e:\n",
    "        print(xml)\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "0f1f3087-81c9-451c-81bb-2f2e59d29d96",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total: 451 pages\n",
      "Valid header: 430 pages\n",
      "Invalid header: 21 pages\n"
     ]
    }
   ],
   "source": [
    "invalid_pages_14376 = list(filter(lambda x: not x.Header.isValid, pages_14376))\n",
    "print('Total:', len(pages_14376), 'pages')\n",
    "print('Valid header:', len(pages_14376) - len(invalid_pages_14376), 'pages')\n",
    "print('Invalid header:', len(invalid_pages_14376), 'pages')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "46836c45-0926-456b-bf94-a1ac07462e21",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total: 450 pages\n",
      "Valid header: 440 pages\n",
      "Invalid header: 10 pages\n"
     ]
    }
   ],
   "source": [
    "invalid_pages_14377 = list(filter(lambda x: not x.Header.isValid, pages_14377))\n",
    "print('Total:', len(pages_14377), 'pages')\n",
    "print('Valid header:', len(pages_14377) - len(invalid_pages_14377), 'pages')\n",
    "print('Invalid header:', len(invalid_pages_14377), 'pages')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "id": "5e3b7597-72b0-4e06-a6fc-be069dfa9b7a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total: 565 pages\n",
      "Valid header: 556 pages\n",
      "Invalid header: 9 pages\n"
     ]
    }
   ],
   "source": [
    "invalid_pages_14378 = list(filter(lambda x: not x.Header.isValid, pages_14378))\n",
    "print('Total:', len(pages_14378), 'pages')\n",
    "print('Valid header:', len(pages_14378) - len(invalid_pages_14378), 'pages')\n",
    "print('Invalid header:', len(invalid_pages_14378), 'pages')\n",
    "sorted_pages_14378 = sorted(pages_14378, key=lambda x: x.get_filename())\n",
    "sorted_invalid_pages_14378 = sorted(invalid_pages_14378, key=lambda x: x.get_filename())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "f18d74ec-63ac-4950-a1d8-89ab3c42849e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.378_00000009.jpeg\n",
      "14.378_00000280.jpeg\n",
      "14.378_00000328.jpeg\n"
     ]
    }
   ],
   "source": [
    "def has_empty_header(page):\n",
    "    p_num = page.Header.PageNumber[0]\n",
    "    p_format = page.Header.Format[0]\n",
    "    p_cat = page.Header.Category[0]\n",
    "    p_subcat = page.Header.Subcategory[0]\n",
    "    return p_num + p_format + p_cat + p_subcat == []\n",
    "\n",
    "empty_headers = []\n",
    "for page in invalid_pages_14378:\n",
    "    if has_empty_header(page):\n",
    "        empty_headers.append(page)\n",
    "\n",
    "for page in empty_headers:\n",
    "    print(page.XMLSources.PageXML.find_all('Page')[0].attrs['imageFilename'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "c92c8f90-ac4e-49eb-9d76-786ee0fc6ee3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.378_00000009.jpeg\n",
      "14.378_00000011.jpeg\n",
      "14.378_00000279.jpeg\n",
      "14.378_00000280.jpeg\n",
      "14.378_00000317.jpeg\n",
      "14.378_00000327.jpeg\n",
      "14.378_00000328.jpeg\n",
      "14.378_00000487.jpeg\n",
      "14.378_00000531.jpeg\n"
     ]
    }
   ],
   "source": [
    "for page in sorted_invalid_pages_14378:\n",
    "    print(page.get_filename())\n",
    "    # print(page.Header.Format[0])\n",
    "    # if not page.Header.Format[0]:\n",
    "    #     print(page.get_filename())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "id": "1b9ba554-0903-4367-b966-86ae213d59c5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "345\n",
      "123\n",
      "456\n",
      "456\n"
     ]
    }
   ],
   "source": [
    "def extract_number(page_number_array):\n",
    "    pg_nums = page_number_array[0]\n",
    "    for num in pg_nums:\n",
    "        num_str = re.sub('\\*+|⁎+|X+', '', num)\n",
    "        if not num_str:\n",
    "            continue\n",
    "        parsed_num = int(num_str)\n",
    "    return parsed_num\n",
    "\n",
    "print(extract_number([['345', '*'], ['asdf', 'qwer']]))\n",
    "print(extract_number([['**', '123'], ['asdf', 'qwer']]))\n",
    "print(extract_number([['456**'], ['asdf']]))\n",
    "print(extract_number([['456⁎⁎'], ['asdf']]))\n",
    "sorted_valid_pages_14376 = sorted(set(pages_14376) - set(invalid_pages_14376), key=lambda x: x.get_filename())\n",
    "sorted_valid_pages_14377 = sorted(set(pages_14377) - set(invalid_pages_14377), key=lambda x: x.get_filename())\n",
    "sorted_valid_pages_14378 = sorted(set(pages_14378) - set(invalid_pages_14378), key=lambda x: x.get_filename())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "id": "9556eaa9-ba80-4960-a5f6-d351dc59fcab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# for page in sorted_valid_pages_14378:\n",
    "#     rem_page_str = str(page.RemainingLines)\n",
    "#     if rem_page_str != 'Remaining Lines:\\n':\n",
    "#         print(page.get_filename())\n",
    "#         print(rem_page_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "id": "dddc2174-31a8-4165-80b7-ea519ccefb2e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for page in sorted_valid_pages_14378:\n",
    "    if len(page.Header.Format[0]) > 2 or len(page.Header.PageNumber[0]) > 2 or len(page.Header.Category[0]) > 1 or len(page.Header.Subcategory[0]) > 3:\n",
    "        print(page.XMLSources.PageXML.find_all('Page')[0].attrs['imageFilename'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "id": "5fdfbb51-c781-48b2-81a8-887872b06b17",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filename: 14.376_00000049.jpeg\n",
      "Page number(s): 25\n",
      "Format: Octavo und kleiner Octavo und kleiner\n",
      "Category: Theologia\n",
      "Subcategory: Critici Sacri\n"
     ]
    }
   ],
   "source": [
    "print(list(filter(lambda x: x.get_filename() == failed_names_14376[0], pages_14376))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "id": "313f429d-25af-4dd1-b2b2-ee48d515b619",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120\n"
     ]
    }
   ],
   "source": [
    "print(len(failed_names_14376 + failed_names_14377 + failed_names_14378))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "85aacdce-6686-46f7-9200-c77a87ac2569",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# x = 300\n",
    "\n",
    "# for page in sorted_valid_pages_14378[x:x+50]:\n",
    "#     print(page)\n",
    "#     print(page.Entries)\n",
    "#     print(page.RemainingLines)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfd319f6-268c-415a-95bb-bbbca979dfda",
   "metadata": {},
   "source": [
    "# Counting number of entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "id": "b08d2889-3dd3-4cb4-8cb5-2a297ee336c1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2781 3161 3461 9403\n"
     ]
    }
   ],
   "source": [
    "count_14376 = 0\n",
    "count_14377 = 0\n",
    "count_14378 = 0\n",
    "\n",
    "for page in sorted_valid_pages_14376:\n",
    "    entries = page.Entries.Entries\n",
    "    count_14376 += len(entries)\n",
    "\n",
    "for page in sorted_valid_pages_14377:\n",
    "    entries = page.Entries.Entries\n",
    "    count_14377 += len(entries)\n",
    "    \n",
    "for page in sorted_valid_pages_14378:\n",
    "    entries = page.Entries.Entries\n",
    "    count_14378 += len(entries)\n",
    "    \n",
    "print(count_14376, count_14377, count_14378, count_14376 + count_14377 + count_14378)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "id": "b8f69ff4-5fa5-4c30-96d9-0ebeff325463",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Im ersten Band fehlen 26 Seiten\n",
      "Im zweiten Band fehlen 17 Seiten\n",
      "Im dritten Band fehlen 22 Seiten\n"
     ]
    }
   ],
   "source": [
    "list_1 = list(pathlib.Path('../../digital-edition/Step0_Export/Cod. 14.376/page/').glob('*.xml'))\n",
    "list_2 = list(pathlib.Path('../../digital-edition/Step0_Export/Cod. 14.377/page/').glob('*.xml'))\n",
    "list_3 = list(pathlib.Path('../../digital-edition/Step0_Export/Cod. 14.378/page/').glob('*.xml'))\n",
    "missing_1 = 477 - len(list_1)\n",
    "missing_2 = 467 - len(list_2)\n",
    "missing_3 = 587 - len(list_3)\n",
    "\n",
    "print(f'Im ersten Band fehlen {missing_1} Seiten')\n",
    "print(f'Im zweiten Band fehlen {missing_2} Seiten')\n",
    "print(f'Im dritten Band fehlen {missing_3} Seiten')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4ddedf9-1e08-408a-983f-69603e07068c",
   "metadata": {},
   "source": [
    "# Processing and enriching TEI-XML file\n",
    "## Add list of changes to TEI header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "id": "984dfe0e-b5d0-4764-b038-9fee8f7a13fb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step0_Export/Cod. 14.376_tei.xml', 'r') as tei_xml_input:\n",
    "    content = tei_xml_input.read()\n",
    "    tei_1 = bs(content, \"lxml-xml\")\n",
    "\n",
    "with open('../../digital-edition/Step0_Export/Cod. 14.377_tei.xml', 'r') as tei_xml_input:\n",
    "    content = tei_xml_input.read()\n",
    "    tei_2 = bs(content, \"lxml-xml\")\n",
    "\n",
    "with open('../../digital-edition/Step0_Export/Cod. 14.378_tei.xml', 'r') as tei_xml_input:\n",
    "    content = tei_xml_input.read()\n",
    "    tei_3 = bs(content, \"lxml-xml\")\n",
    "\n",
    "dod_ids = [51202, 51184, 51219]\n",
    "cod_prefixes = {51202: '14.376', 51184: '14.377', 51219: '14.378'}\n",
    "tei = {51202: tei_1, 51184: tei_2, 51219: tei_3}\n",
    "\n",
    "for key in tei:\n",
    "    revisionDesc = bs('<revisionDesc><listChange>'\n",
    "                      '<change when=\"2022-12-12\" who=\"#AT\">Import of facsimiles to Transkribus</change>'\n",
    "                      '<change from=\"2022-12-13\" to=\"2023-01-26\" who=\"#AT #SM #MK\">Manual transcription of pages to be used as Ground truth for training process in Transkribus</change>'\n",
    "                      '<change from=\"2023-02-20\" to=\"2023-02-28\" who=\"#SM\">Automated transcription of the first three volumes Cod. 14.376-14.378</change>'\n",
    "                      '<change from=\"2023-03-01\" to=\"2023-12-13\" who=\"#AT #SM #MK #GF #PE #TG #AR\">Correction of automated transcripts in Transkribus, adding custom tags (add, corr, bleistift, roetel, continued, comment)</change>'\n",
    "                      '<change when=\"2023-12-18\" who=\"#SM\">Export from Transkribus to TEI-XML</change>'\n",
    "                  '</listChange></revisionDesc>', 'lxml-xml')\n",
    "\n",
    "    other_parts = bs('''<body><!-- Three missing parts to be added later: More information under https://tei-c.org/release/doc/tei-p5-doc/en/html/HD.html -->\n",
    "            <encodingDesc>\n",
    "                <!--  describes the relationship between an electronic text and its source or sources. It allows for detailed description of whether (or how) the text was normalized during transcription, how the encoder resolved ambiguities in the source, what levels of encoding or analysis were applied, and similar matters -->\n",
    "            </encodingDesc>\n",
    "            <profileDesc>\n",
    "                <!-- contains classificatory and contextual information about the text, such as its subject matter, the situation in which it was produced, the individuals described by or participating in producing it, and so forth. Such a text profile is of particular use in highly structured composite texts such as corpora or language collections, where it is often highly desirable to enforce a controlled descriptive vocabulary or to perform retrievals from a body of text in terms of text type or origin. The text profile may however be of use in any form of automatic text processing. -->\n",
    "            </profileDesc>\n",
    "            <xenoData>\n",
    "                <!-- is a container element that allows easy inclusion of metadata from non-TEI schemes (i.e., other than elements in the TEI namespace). For example, the MARC record for the encoded document might be included using MARCXML or MODS. A simple set of metadata for harvesting might be included encoded in Dublin Core. -->\n",
    "            </xenoData></body>\n",
    "        ''', 'lxml-xml')\n",
    "    tei[key].teiHeader.append(revisionDesc)\n",
    "    for child in other_parts.body.children:\n",
    "        tei[key].revisionDesc.insert_before(child)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5e115c2-59c2-4b90-81bf-f4962859a812",
   "metadata": {},
   "source": [
    "## Insert missing pages to TEI-XML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "id": "76c0e9a7-5f78-4831-afcc-475ae0f8b1c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for dod_id in dod_ids:\n",
    "    cod_prefix = cod_prefixes[dod_id]\n",
    "    req = requests.get(f'http://iiif.onb.ac.at/presentation/DOD/{dod_id}/manifest')\n",
    "    manifest = json.loads(req.content)\n",
    "    canvases = manifest['sequences'][0]['canvases']\n",
    "    image_shapes = list(map(lambda x: (x['height'], x['width']), canvases))\n",
    "    first_facsimile = tei[dod_id].facsimile\n",
    "    first_pb = tei[dod_id].pb\n",
    "    insert_before = True\n",
    "    for i in range(len(image_shapes)):\n",
    "        url = first_facsimile.surface.graphic.attrs['url']\n",
    "        filename = f'{cod_prefix}_{(i+1):0>8}.jpeg'\n",
    "        if url == filename:\n",
    "            old_facs_num = first_facsimile.attrs['xml:id']\n",
    "            new_facs_num = f'facs_{i+1}'\n",
    "            zone_children = first_facsimile.surface.find_all('zone')\n",
    "            for zone in zone_children:\n",
    "                zone.attrs['xml:id'] = zone.attrs['xml:id'].replace(old_facs_num, new_facs_num)\n",
    "            # if there are multiple text regions per page\n",
    "            first_p_in_pb = first_pb.find_next_sibling()\n",
    "            while ((first_p_in_pb is not None) and (first_p_in_pb.name == 'p')):\n",
    "                first_p_in_pb.attrs['facs'] = first_p_in_pb.attrs['facs'].replace(old_facs_num, new_facs_num)\n",
    "                lb_children = first_p_in_pb.find_all('lb')\n",
    "                for lb in lb_children:\n",
    "                    lb.attrs['facs'] = lb.attrs['facs'].replace(old_facs_num, new_facs_num)\n",
    "                first_p_in_pb = first_p_in_pb.find_next_sibling()\n",
    "            \n",
    "            first_pb.attrs['facs'] = first_pb.attrs['facs'].replace(old_facs_num, new_facs_num)\n",
    "            first_pb.attrs['n'] = i+1\n",
    "            first_facsimile.attrs['xml:id'] = new_facs_num\n",
    "            \n",
    "            next_facs = first_facsimile.find_next_sibling('facsimile')\n",
    "            next_pb = first_pb.find_next_sibling('pb')\n",
    "            if next_facs is not None: \n",
    "                first_facsimile = next_facs\n",
    "                first_pb = next_pb\n",
    "            else:  # reached the end of the existing facsimiles, now insert after\n",
    "                insert_before = False\n",
    "        else:\n",
    "            new_facs = bs(f'<facsimile xml:id=\"facs_{i+1}\"><surface lrx=\"{image_shapes[i][1]}\" lry=\"{image_shapes[i][0]}\" ulx=\"0\" uly=\"0\"><graphic height=\"{image_shapes[i][0]}px\" url=\"{filename}\" width=\"{image_shapes[i][1]}px\"/></surface></facsimile>', 'lxml-xml')\n",
    "            new_pb = bs(f'<pb facs=\"#facs_{i+1}\" n=\"{i+1}\"/>', 'lxml-xml')\n",
    "            if insert_before:\n",
    "                first_facsimile.insert_before(new_facs)\n",
    "                first_pb.insert_before(new_pb)\n",
    "            else:\n",
    "                # try to find next p tag and insert after\n",
    "                next_p = first_pb.find_next_sibling('p')\n",
    "                if next_p is not None:\n",
    "                    next_p.insert_after(new_pb)\n",
    "                else:\n",
    "                    first_pb.insert_after(new_pb)\n",
    "                first_facsimile.insert_after(new_facs)\n",
    "                first_facsimile = first_facsimile.find_next_sibling('facsimile')\n",
    "                first_pb = first_pb.find_next_sibling('pb')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26ad9d51-1dd0-4039-8442-9d7e7f514095",
   "metadata": {},
   "source": [
    "## Convert from linebreaks to line tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "id": "fc85c99f-ce88-4fc8-ad16-ba587b302bea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def convert_lb_into_l(p):\n",
    "    first_lb = p.lb\n",
    "    while(first_lb is not None):\n",
    "        content = []\n",
    "        el = first_lb.next_element\n",
    "        while (el is not None and el.name != 'lb'):\n",
    "            next_el = el.next_sibling\n",
    "            content.append(el.extract())\n",
    "            el = next_el\n",
    "        new_l = bs().new_tag('l')\n",
    "        new_l.attrs['facs'] = first_lb.attrs['facs']\n",
    "        new_l.attrs['n'] = first_lb.attrs['n']\n",
    "        for el in content:\n",
    "            new_l.append(el)\n",
    "        first_lb.replace_with(new_l)\n",
    "        first_lb = p.lb\n",
    "\n",
    "for dod_id in dod_ids:\n",
    "    for p in tei[dod_id].find_all('p'):\n",
    "        convert_lb_into_l(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e3c44d-005e-4136-85ec-e344bda5687a",
   "metadata": {},
   "source": [
    "# Join separate text regions (multiple p tags) into one, remove meaningless region name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "id": "1f377a85-a0cc-420b-9306-afa9bc273721",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def delete_from_second_underscore(string):\n",
    "    und_pos = string.find('_', 6)\n",
    "    if und_pos != -1:\n",
    "        return string[:und_pos]\n",
    "    else: \n",
    "        return string\n",
    "\n",
    "# print(delete_from_second_underscore('#facs_110_TextRegion...'))\n",
    "# print(delete_from_second_underscore('#facs_110'))\n",
    "\n",
    "for key in tei:\n",
    "    all_p_tags = tei[key].find_all('p')\n",
    "    \n",
    "    for p in all_p_tags:\n",
    "        p['facs'] = delete_from_second_underscore(p['facs'])\n",
    "    \n",
    "    same_facs_p_tags = []\n",
    "    for p in all_p_tags:\n",
    "        facs_num = p['facs']\n",
    "        duplicate_cands = list(filter(lambda x: x['facs'] == facs_num, all_p_tags))\n",
    "        if len(duplicate_cands) > 1:\n",
    "            same_facs_p_tags.append(p)\n",
    "    \n",
    "    fac_groups = {}\n",
    "    for p_tag in same_facs_p_tags:\n",
    "        tag_num = p_tag['facs']\n",
    "        if tag_num not in fac_groups.keys():\n",
    "            fac_groups[tag_num] = [p_tag]\n",
    "        else:\n",
    "            fac_groups[tag_num].append(p_tag)\n",
    "\n",
    "    for num in fac_groups:\n",
    "        main_p = fac_groups[num][0]\n",
    "        remaining_ps = fac_groups[num][1:]\n",
    "        for p in remaining_ps:\n",
    "            p.extract()\n",
    "            main_p.extend(list(p.children))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23c483ea-cb67-49c8-8e00-d03822bc5537",
   "metadata": {},
   "source": [
    "## Add additional tags from processing\n",
    "\n",
    "Easy: Category, Subcategory, Page number, Format specification. Tag names have to be normalized.\n",
    "\n",
    "Harder: Grouping of entries, together with remaining lines per page and sorting of lines from the top."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "id": "a6bf33c1-4f47-4f07-9d91-f35505edfead",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def insert_tags(page, tei_xml):\n",
    "    num = int(page.get_filename()[-8:-5])\n",
    "    pb = tei_xml.find('pb', facs=f'#facs_{num}')\n",
    "    p = pb.find_next_sibling('p')\n",
    "    head_div = bs().new_tag('div')\n",
    "    head_div['type'] = 'head'\n",
    "    page_data = {\n",
    "        'category': page.Header.Category,\n",
    "        'subcategory': page.Header.Subcategory,\n",
    "        'pagenumber': page.Header.PageNumber,\n",
    "        'format': page.Header.Format,\n",
    "        'margin': page.Header.MarginNote\n",
    "    }\n",
    "    order = {\n",
    "        'category': 0,\n",
    "        'subcategory': 1,\n",
    "        'pagenumber': 2,\n",
    "        'format': 3,\n",
    "        'margin': 4\n",
    "    }\n",
    "    for l in p.find_all('l'):\n",
    "        l_id = l['facs'].replace(f'#facs_{num}_', '')\n",
    "        for tag in page_data:\n",
    "            data = page_data[tag]\n",
    "            for text, line in zip(data[0], data[1]):\n",
    "                if l_id == line.ID:\n",
    "                    ab_tag = head_div.find('ab', type=tag)\n",
    "                    if ab_tag is None:\n",
    "                        ab_tag = bs().new_tag('ab')\n",
    "                        ab_tag['type'] = tag\n",
    "                        if tag in ['category', 'subcategory', 'format']:\n",
    "                            ab_tag['value'] = text\n",
    "                        if tag == 'margin':\n",
    "                            ab_tag['value'] = ' '.join(data[0])\n",
    "                        head_div.insert(order[tag], ab_tag)\n",
    "                    l.extract()\n",
    "                    ab_tag.append(l)\n",
    "    if len(head_div.contents) > 0:\n",
    "        p.insert(0, head_div)\n",
    "\n",
    "# ex_page = pages_14378[0]\n",
    "# insert_tags(ex_page, tei_3)\n",
    "for page in pages_14376:\n",
    "    insert_tags(page, tei_1)\n",
    "for page in pages_14377:\n",
    "    insert_tags(page, tei_2)\n",
    "for page in pages_14378:\n",
    "    insert_tags(page, tei_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04e2e6c5-eb06-499a-85d8-fc66be5f4e31",
   "metadata": {},
   "source": [
    "# Join entries and remaining lines, then add them in correct reading order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "id": "b00f9b85-08cd-4d0a-bb7e-b32548ec19da",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def group_entries_and_remaining(page, tei_xml):\n",
    "    num = int(page.get_filename()[-8:-5])\n",
    "    pb = tei_xml.find('pb', facs=f'#facs_{num}')\n",
    "    p = pb.find_next_sibling('p')\n",
    "    entries = page.Entries.Entries\n",
    "    header_section = page.Header.Section[1]\n",
    "    sections = [[section] for section in header_section]\n",
    "    sorted_entries_sections = sorted(entries + sections, key=lambda x: x[0].Baseline[0][1])\n",
    "    centers = page.Entries.Centers\n",
    "    rem_lines = sorted(page.RemainingLines.Lines, key=lambda x: x.Baseline[0][1])\n",
    "    grouped = [[entry, []] for entry in sorted_entries_sections]\n",
    "    if len(centers) > 0:\n",
    "        for line in rem_lines:\n",
    "            rl_y = line.Baseline[0][1]\n",
    "            distances = [np.linalg.norm(rl_y - c[1]) for c in centers]\n",
    "            index_of_closest_entry = np.argmin(distances)\n",
    "            grouped[index_of_closest_entry][1].append(line)\n",
    "\n",
    "        for group in grouped:\n",
    "            text_lines, rem_lines = group\n",
    "            entry_div = bs().new_tag('div')\n",
    "            entry_div['type'] = 'entry'\n",
    "            ab_text = bs().new_tag('ab')\n",
    "            ab_text['type'] = 'text'\n",
    "            for line in text_lines:\n",
    "                tei_line = p.find(facs=f'#facs_{num}_{line.ID}')\n",
    "                if tei_line is not None:\n",
    "                    tei_line.extract()\n",
    "                    ab_text.append(tei_line)\n",
    "            entry_div.append(ab_text)\n",
    "            if len(rem_lines) > 0:\n",
    "                ab_rem = bs().new_tag('ab')\n",
    "                ab_rem['type'] = 'remaining'\n",
    "                for line in rem_lines:\n",
    "                    tei_line = p.find(facs=f'#facs_{num}_{line.ID}')\n",
    "                    if tei_line is not None:\n",
    "                        tei_line.extract()\n",
    "                        ab_rem.append(tei_line)\n",
    "                entry_div.append(ab_rem)\n",
    "            p.append(entry_div)\n",
    "\n",
    "# ex_page = Page('../../digital-edition/Step0_Export/Cod. 14.378/page/14.378_00000539.xml')\n",
    "# group_entries_and_remaining(ex_page, tei_3)\n",
    "for page in pages_14376:\n",
    "    group_entries_and_remaining(page, tei_1)\n",
    "for page in pages_14377:\n",
    "    group_entries_and_remaining(page, tei_2)\n",
    "for page in pages_14378:\n",
    "    group_entries_and_remaining(page, tei_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb3dbd30-714b-4110-ac07-2172a84d5628",
   "metadata": {},
   "source": [
    "# Save processed/enriched files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "id": "bf5481a2-cbb4-4dac-ad82-ee5e5cd7df18",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from bs4.formatter import XMLFormatter\n",
    "formatter = XMLFormatter(indent=4)\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.376_tei.xml', 'w') as tei_xml_output:\n",
    "    tei_xml_output.write(tei_1.prettify(formatter=formatter))\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.377_tei.xml', 'w') as tei_xml_output:\n",
    "    tei_xml_output.write(tei_2.prettify(formatter=formatter))\n",
    "    \n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.378_tei.xml', 'w') as tei_xml_output:\n",
    "    tei_xml_output.write(tei_3.prettify(formatter=formatter))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93e4322c-7965-42c4-b434-4260e7b6bf00",
   "metadata": {},
   "source": [
    "# Export entries for string matching with modern catalogue data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "id": "8b8c8a07-98b4-453a-95d0-1c70257d1f2f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "entry_list = [['volume', 'page number', 'category', 'subcategory', 'format', 'handwritten page number', 'entry_ID', 'entry']]\n",
    "pages_dict = {'14.376': pages_14376,\n",
    "              '14.377': pages_14377,\n",
    "              '14.378': pages_14378\n",
    "             }\n",
    "\n",
    "for vol in pages_dict:\n",
    "    for page in pages_dict[vol]:\n",
    "        entry_num = 0\n",
    "        for entry in page.Entries.Entries:\n",
    "            page_number = int(page.get_filename()[-8:-5]) \n",
    "            entry_texts = [line.Text for line in entry]\n",
    "            entry_string = ' '.join(entry_texts)\n",
    "            entry_ID = f'{vol}_{page_number:03}_{entry_num:02}'\n",
    "            data = [vol, page_number, page.Header.print_category(), page.Header.print_subcategory(), page.Header.print_format(), page.Header.print_pagenumber(), entry_ID, entry_string]\n",
    "            entry_list.append(data)\n",
    "            entry_num += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "id": "a994ef96-c7b8-41c3-89ab-bd80455ed3b8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9403\n",
      "[['volume', 'page number', 'category', 'subcategory', 'format', 'handwritten page number', 'entry_ID', 'entry'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_00', '1 Biblia Sacra Polyglotta curis Cardinalis Ximenij¬ edita, cum Vocabulario Hebraïco & Chaldaïco veteris Testamenti. 56 Vol. Compluti. 1515. & annis Seqq. Guill. de Brocario. n. 84. Integra biblia sunt V. volumini= bus comprehensa. deest Vocabularium, quod pro sexto numeratur.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_01', '2 Biblia Sacra Polyglotta Philippi II. Regis Cathol. pietate, ac # Studio ad Sacro- Sanctæ Ecclesiæ usum edita, cum Apparatu & Lexico. 8 Vol. Antuerp. 1572. Plantin. n. 9.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_02', '3 Biblia Sacra Polyglotta Studio & curâ Briani Waltoni edita. 6 Vol. chartâ Magnâ. Londini. 1637. Roycroft. accedit n. 36. Lexicon Heptaglotton ad eadem Biblia Polyglotta autore Edmundo Castello. 2 Vol. chartâ M. Londini. 1669. Roycroft. N. 36.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_03', '4 Biblia Sacra Latina Moguntina dicta, prima omnium editio in Membranis. 2 Vol. Moguntiæ. 1462.¬ Ioan. Fust, & Petr. Schoiffer de Gerneshem. n. 2088. II.O.5.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_04', '5 Biblia Sacra Latina Moguntina, editio altera 2 Vol. chartâ Magnâ. Moguntiæ. 1472. Petr. Schoiffer. II.O.7. de Gerneshem. n. 2089.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_05', '6 Biblia Sacra Latina cum glossa & Comment. 4 Vol. chartâ Magnâ. Editio perantiqua sine loco & II.O.1. anno n. 2086.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_06', '7 Biblia Sacra latina vulgatæ Editionis. 2 Vol. chartâ III.B.3. Magnâ. 1474. absque loco Editionis n. 2090.'], ['14.376', 25, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '1', '14.376_025_07', '8 Biblia Sacra Latina cum Evangelistarum Canonibus III.B.7. & concordantiis. chartâ M. Coloniæ. 1479. de Homborch. n. 2093.'], ['14.376', 26, 'Theologia', 'Textus & Versiones Sacræ Scripturæ', 'Folio', '2', '14.376_026_00', '9. Biblia Sacra Latina. Venetiis. 1481. Leonard Wild de Ratisbonâ n. 2302. III. D. 11.']]\n"
     ]
    }
   ],
   "source": [
    "print(len(entry_list) - 1)\n",
    "print(entry_list[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "id": "7390e95e-6fc7-4f2b-884b-37166064214c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "entry_df = pd.DataFrame(entry_list[1:], columns=entry_list[0])\n",
    "entry_df.sort_values('entry_ID', inplace=True, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "3a779456-4810-4428-85d7-b0c5277717d8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "entry_df.to_excel('../Daten/Katalogabgleich/Einträge.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98bf1e20-d09c-41d5-8bc3-725a851f6ab3",
   "metadata": {},
   "source": [
    "# Add matches from handwritten catalog to modern catalog into the TEI_XMLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "0d8fa911-2986-4a4d-af5b-2adf3bddada3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "entry_df = pd.read_excel('../Daten/Katalogabgleich/Einträge.xlsx', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "0a86cfbe-33b3-4a3a-b384-270b5228359a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BE_df = pd.read_excel('../Daten/Vorhersagen/WIP_final_BE_4.xlsx', index_col=0)\n",
    "\n",
    "# drop signature duplicates?!\n",
    "to_drop = [2292, 13801, 18647]\n",
    "\n",
    "BE_df.drop(to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0c055e8-59c4-41c7-b0d7-f709c2734e5a",
   "metadata": {},
   "source": [
    "## Download all available IIIF manifests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "2e24b3fe-2908-4d88-8df1-6045815bd4ae",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e6779a4cc604b988daac74fa308d6c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21356 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "urls = BE_df[['IIIF Manifest', 'Barcode']].dropna(subset=['IIIF Manifest'])\n",
    "\n",
    "for i, url in tqdm(urls.iterrows(), total=len(urls)):\n",
    "    filename = url['Barcode'] + '.json'\n",
    "    man = requests.get(url['IIIF Manifest']).content\n",
    "    with open(f'data/iiif_manifests/{filename}', 'wb') as fh:\n",
    "        fh.write(man)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "fdff3c2f-0326-412f-8347-da864f306322",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21356\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "mans = glob.glob('data/iiif_manifests/*.json')\n",
    "\n",
    "print(len(mans))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "a0c928c7-8bc2-4e89-997b-34be5a9d2a54",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d335c8424de4d169a44b9e1aeeb0079",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/22871 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_AC_num(BE_row):\n",
    "    bc = BE_row['Barcode']\n",
    "    fn = f'data/iiif_manifests/{bc}.json'\n",
    "    try:\n",
    "        with open(fn, 'r') as fh:\n",
    "            man = fh.read()\n",
    "        metadata = json.loads(man)['metadata']\n",
    "        ac = ''\n",
    "        for dic in metadata:\n",
    "            if dic['label'] == 'IDNR':\n",
    "                ac = dic['value']\n",
    "        return ac\n",
    "    except FileNotFoundError as e:\n",
    "        return ''\n",
    "\n",
    "for i, BE_row in tqdm(BE_df.iterrows(), total=len(BE_df)):\n",
    "    BE_df.at[i, 'AC Nummer'] = get_AC_num(BE_row)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71d603b3-5ead-4ecd-9039-961325a60973",
   "metadata": {},
   "source": [
    "## Obtain AC number from ASTOR repository via signature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "28fc3270-5d3e-4b20-a149-37d433afc197",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BE_no_AC = BE_df[BE_df['AC Nummer'] == '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "7fd80561-9e5d-498c-b2f9-428e293fd3a2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16d72df27a6548f6a148f3dd4158d2f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1515 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "ASTOR_API_KEY = os.getenv('ASTOR_API_KEY')\n",
    "\n",
    "for i, BE_row in tqdm(BE_no_AC.iterrows(), total=len(BE_no_AC)):\n",
    "    sig = BE_row['Signatur']\n",
    "    sig_esc = sig.replace('.', '\\.').replace('(', '\\(').replace(')', '\\)')\n",
    "    sig_sru = json.loads(requests.get(f'https://astor.onb.ac.at/discovery/internal/search?query=signature:{sig_esc}&from=1&rows=200&apikey={ASTOR_API_KEY}').content)\n",
    "    if 'documents' in sig_sru.keys():\n",
    "        docs = sig_sru['documents']\n",
    "        for doc in docs:\n",
    "            if doc['signature'] == sig:\n",
    "                BE_df.at[i, 'AC Nummer'] = doc['idnr']\n",
    "                break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30500e79-5227-4c21-9825-365bf6992c2a",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Obtain AC number from catalagoue via signature and SRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "c293013f-b44d-44ec-9529-c40a2863d9f3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fa10904058e425c9670c9d5f09b1713",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from lxml import etree\n",
    "\n",
    "ns = {\n",
    "    'srw': 'http://www.loc.gov/zing/srw/',\n",
    "    'marc': 'http://www.loc.gov/MARC21/slim'\n",
    "}\n",
    "\n",
    "lang_data = pd.read_csv('data/iso-639-3.tab', sep='\\t')\n",
    "\n",
    "def english_language_from_code(lang_code):\n",
    "    find_by_Id = lang_data[lang_data['Id'] == lang_code]\n",
    "    find_by_Part2b = lang_data[lang_data['Part2b'] == lang_code]\n",
    "    if len(find_by_Id):\n",
    "        name = find_by_Id['Ref_Name'].values[0]\n",
    "    elif len(find_by_Part2b):\n",
    "        name = find_by_Part2b['Ref_Name'].values[0]\n",
    "    else:\n",
    "        name = ''\n",
    "    return name\n",
    "\n",
    "def extract_catalog_data_from_signature(sig):\n",
    "    metadata_lis = []\n",
    "    sru = f'https://obv-at-oenb.alma.exlibrisgroup.com/view/sru/43ACC_ONB?version=1.2&query=alma.accessionNumber%3D%22{sig}%22&operation=searchRetrieve'\n",
    "    sru_request = requests.get(sru)\n",
    "    marcxml = sru_request.content\n",
    "    tree = etree.fromstring(marcxml)\n",
    "    records = tree.xpath('.//marc:record', namespaces=ns)\n",
    "    for rec in records:\n",
    "        metadata = {}\n",
    "        marc_paths = {\n",
    "            'Titel': './/marc:datafield[@tag=\"245\"]/marc:subfield[@code=\"a\"]',\n",
    "            'Autor': './/marc:datafield[@tag=\"100\"]/marc:subfield[@code=\"a\"]',\n",
    "            'Mitwirkender': './/marc:datafield[@tag=\"700\"]/marc:subfield[@code=\"a\"]',\n",
    "            'Signatur': './/marc:datafield[@tag=\"AVA\"]/marc:subfield[@code=\"d\"]',\n",
    "            'Veröffentlichungsdatum': './/marc:datafield[@tag=\"264\"]/marc:subfield[@code=\"c\"]',\n",
    "            'Veröffentlichungsort': './/marc:datafield[@tag=\"264\"]/marc:subfield[@code=\"a\"]',\n",
    "            'Sprache': './/marc:datafield[@tag=\"041\"]/marc:subfield[@code=\"a\"]',\n",
    "            'AC Nummer': './/marc:controlfield[@tag=\"009\"]'\n",
    "        }\n",
    "\n",
    "        for key, path in marc_paths.items():\n",
    "            values = [elm.text for elm in rec.xpath(path, namespaces=ns)]\n",
    "            if key == 'Sprache':\n",
    "                values = [english_language_from_code(val) for val in values]\n",
    "\n",
    "            metadata[key] = '; '.join(values)\n",
    "            # metadata['Signatur'] = sig\n",
    "        metadata_lis.append(metadata)\n",
    "    return metadata_lis\n",
    "\n",
    "for i, BE_row in tqdm(BE_no_AC.iterrows(), total=len(BE_no_AC)):\n",
    "    if BE_row['AC Nummer'] != '':\n",
    "        continue\n",
    "    sig = BE_row['Signatur']\n",
    "    sig_esc = sig.replace('.', '\\.').replace('(', '\\(').replace(')', '\\)').replace('-', '\\-').replace(',', '\\,')\n",
    "    metadata_lis = extract_catalog_data_from_signature(sig_esc)\n",
    "    for dic in metadata_lis:\n",
    "        if '; ' in sig:\n",
    "            siglis = dic['Signatur'].split('; ')\n",
    "        else:\n",
    "            siglis = [sig]\n",
    "        for s in siglis:\n",
    "            if sig == s:\n",
    "                BE_df.at[i, 'AC Nummer'] = dic['AC Nummer']\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be259674-5257-43e4-ba19-cbd12b3bfc29",
   "metadata": {},
   "outputs": [],
   "source": [
    "useless_entries = [2074, 4200, 5976]\n",
    "\n",
    "BE_df.drop(useless_entries, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "805140d6-8b72-40ac-b297-e0ff24bc74a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BE_df.at[11481, 'AC Nummer'] = 'AC10374727'\n",
    "BE_df.at[12810, 'AC Nummer'] = 'AC11979464'\n",
    "BE_df.at[19399, 'AC Nummer'] = 'AC10075950'\n",
    "BE_df.at[19785, 'AC Nummer'] = 'AC10103649'\n",
    "BE_df.at[19958, 'AC Nummer'] = 'AC10058904'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "4a8d583c-f5a2-4626-a133-c48f80f0a2fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Signatur</th>\n",
       "      <th>Barcode</th>\n",
       "      <th>Titel</th>\n",
       "      <th>Autor</th>\n",
       "      <th>Mitwirkender</th>\n",
       "      <th>Anfang Veröffentlichungsdatum</th>\n",
       "      <th>Ende Veröffentlichungsdatum</th>\n",
       "      <th>Veröffentlichungsdatum</th>\n",
       "      <th>Veröffentlichungsort</th>\n",
       "      <th>Veröffentlichungsort (normiert)</th>\n",
       "      <th>...</th>\n",
       "      <th>Formatangabe</th>\n",
       "      <th>hs. Katalogseite Handschrift</th>\n",
       "      <th>hs. Katalogeintrag ID</th>\n",
       "      <th>hs. Katalogeintrag</th>\n",
       "      <th>hs. Katalog Image URL</th>\n",
       "      <th>dup_title</th>\n",
       "      <th>copy_from</th>\n",
       "      <th>Einfache Klassifizierung</th>\n",
       "      <th>Komplexe Klassifizierung</th>\n",
       "      <th>AC Nummer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>657</th>\n",
       "      <td>BE.1.N.75.(Teil.1)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Prvi [Drugi] del Novoga Teslamenta [!], vatom ...</td>\n",
       "      <td>Trubar, Primož 1508-1586 [Bearb.]</td>\n",
       "      <td>Ungnad, Hans von; Maximilian II. Heiliges Römi...</td>\n",
       "      <td>1563.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1562-1563</td>\n",
       "      <td>V Tubingi [Urach]</td>\n",
       "      <td>Tübingen</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12124</th>\n",
       "      <td>BE.4.S.81</td>\n",
       "      <td>B1572545</td>\n",
       "      <td>Description des Monumens Musulmans du Cabinet ...</td>\n",
       "      <td>Reinaud, Joseph Toussaint 1795-1867</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1828.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1828</td>\n",
       "      <td>Paris</td>\n",
       "      <td>Paris</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16795</th>\n",
       "      <td>BE.7.E.7.(2)</td>\n",
       "      <td>B1782971</td>\n",
       "      <td>Appendix ad historiam literariam Gulielmi cave...</td>\n",
       "      <td>Wharton, Henricus</td>\n",
       "      <td>Gere, Robertus</td>\n",
       "      <td>1743.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1743</td>\n",
       "      <td>Oxonii</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17957</th>\n",
       "      <td>BE.7.T.64</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Journal Universel, Ou Mémoires Pour servir à l...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1743.0</td>\n",
       "      <td>1748.0</td>\n",
       "      <td>1743-1748</td>\n",
       "      <td>La Haye</td>\n",
       "      <td>Den Haag</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18105</th>\n",
       "      <td>BE.7.V.58.(Adl)</td>\n",
       "      <td>B1380793</td>\n",
       "      <td>Leben und letzte Stunden Christinä von Munk we...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1757.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1757</td>\n",
       "      <td>Kopenhagen usw.</td>\n",
       "      <td>Kopenhagen</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19065</th>\n",
       "      <td>BE.8.K.58.(Vol.Tab.,1)</td>\n",
       "      <td>B1633235</td>\n",
       "      <td>Geschichte der Griechischen Litteratur, von de...</td>\n",
       "      <td>Schoell, Friedrich 1766-1833</td>\n",
       "      <td>Schoell, Maximilien Samson Frederic; Pinder, M...</td>\n",
       "      <td>1828.0</td>\n",
       "      <td>1830.0</td>\n",
       "      <td>1828-1830</td>\n",
       "      <td>Berlin</td>\n",
       "      <td>Berlin</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19066</th>\n",
       "      <td>BE.8.K.58.(Vol.Tab.,2)</td>\n",
       "      <td>3461960-50</td>\n",
       "      <td>Geschichte der Griechischen Litteratur, von de...</td>\n",
       "      <td>Schoell, Friedrich 1766-1833</td>\n",
       "      <td>Schoell, Maximilien Samson Frederic; Pinder, M...</td>\n",
       "      <td>1828.0</td>\n",
       "      <td>1830.0</td>\n",
       "      <td>1828-1830</td>\n",
       "      <td>Berlin</td>\n",
       "      <td>Berlin</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20641</th>\n",
       "      <td>BE.9.C.4.(Vol.2-5)</td>\n",
       "      <td>B1657002</td>\n",
       "      <td>Fauna Japonica Sive Descriptio animalium, quae...</td>\n",
       "      <td>Siebold, Philipp Franz &lt;&lt;von&gt;&gt; 1796-1866</td>\n",
       "      <td>Temminck, Coenraad Jacob; Schlegel, Hermann; H...</td>\n",
       "      <td>1833.0</td>\n",
       "      <td>1850.0</td>\n",
       "      <td>1833-1850</td>\n",
       "      <td>Lugduni Batavorum</td>\n",
       "      <td>Leiden</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Signatur     Barcode   \n",
       "657        BE.1.N.75.(Teil.1)         NaN  \\\n",
       "12124               BE.4.S.81    B1572545   \n",
       "16795            BE.7.E.7.(2)    B1782971   \n",
       "17957               BE.7.T.64         NaN   \n",
       "18105         BE.7.V.58.(Adl)    B1380793   \n",
       "19065  BE.8.K.58.(Vol.Tab.,1)    B1633235   \n",
       "19066  BE.8.K.58.(Vol.Tab.,2)  3461960-50   \n",
       "20641      BE.9.C.4.(Vol.2-5)    B1657002   \n",
       "\n",
       "                                                   Titel   \n",
       "657    Prvi [Drugi] del Novoga Teslamenta [!], vatom ...  \\\n",
       "12124  Description des Monumens Musulmans du Cabinet ...   \n",
       "16795  Appendix ad historiam literariam Gulielmi cave...   \n",
       "17957  Journal Universel, Ou Mémoires Pour servir à l...   \n",
       "18105  Leben und letzte Stunden Christinä von Munk we...   \n",
       "19065  Geschichte der Griechischen Litteratur, von de...   \n",
       "19066  Geschichte der Griechischen Litteratur, von de...   \n",
       "20641  Fauna Japonica Sive Descriptio animalium, quae...   \n",
       "\n",
       "                                          Autor   \n",
       "657           Trubar, Primož 1508-1586 [Bearb.]  \\\n",
       "12124       Reinaud, Joseph Toussaint 1795-1867   \n",
       "16795                         Wharton, Henricus   \n",
       "17957                                       NaN   \n",
       "18105                                       NaN   \n",
       "19065              Schoell, Friedrich 1766-1833   \n",
       "19066              Schoell, Friedrich 1766-1833   \n",
       "20641  Siebold, Philipp Franz <<von>> 1796-1866   \n",
       "\n",
       "                                            Mitwirkender   \n",
       "657    Ungnad, Hans von; Maximilian II. Heiliges Römi...  \\\n",
       "12124                                                NaN   \n",
       "16795                                     Gere, Robertus   \n",
       "17957                                                NaN   \n",
       "18105                                                NaN   \n",
       "19065  Schoell, Maximilien Samson Frederic; Pinder, M...   \n",
       "19066  Schoell, Maximilien Samson Frederic; Pinder, M...   \n",
       "20641  Temminck, Coenraad Jacob; Schlegel, Hermann; H...   \n",
       "\n",
       "       Anfang Veröffentlichungsdatum  Ende Veröffentlichungsdatum   \n",
       "657                           1563.0                          NaN  \\\n",
       "12124                         1828.0                          NaN   \n",
       "16795                         1743.0                          NaN   \n",
       "17957                         1743.0                       1748.0   \n",
       "18105                         1757.0                          NaN   \n",
       "19065                         1828.0                       1830.0   \n",
       "19066                         1828.0                       1830.0   \n",
       "20641                         1833.0                       1850.0   \n",
       "\n",
       "      Veröffentlichungsdatum Veröffentlichungsort   \n",
       "657                1562-1563    V Tubingi [Urach]  \\\n",
       "12124                   1828                Paris   \n",
       "16795                   1743               Oxonii   \n",
       "17957              1743-1748              La Haye   \n",
       "18105                   1757      Kopenhagen usw.   \n",
       "19065              1828-1830               Berlin   \n",
       "19066              1828-1830               Berlin   \n",
       "20641              1833-1850    Lugduni Batavorum   \n",
       "\n",
       "      Veröffentlichungsort (normiert)  ... Formatangabe   \n",
       "657                          Tübingen  ...          NaN  \\\n",
       "12124                           Paris  ...          NaN   \n",
       "16795                          Oxford  ...          NaN   \n",
       "17957                        Den Haag  ...          NaN   \n",
       "18105                      Kopenhagen  ...          NaN   \n",
       "19065                          Berlin  ...          NaN   \n",
       "19066                          Berlin  ...          NaN   \n",
       "20641                          Leiden  ...          NaN   \n",
       "\n",
       "      hs. Katalogseite Handschrift hs. Katalogeintrag ID  hs. Katalogeintrag   \n",
       "657                            NaN                   NaN                 NaN  \\\n",
       "12124                          NaN                   NaN                 NaN   \n",
       "16795                          NaN                   NaN                 NaN   \n",
       "17957                          NaN                   NaN                 NaN   \n",
       "18105                          NaN                   NaN                 NaN   \n",
       "19065                          NaN                   NaN                 NaN   \n",
       "19066                          NaN                   NaN                 NaN   \n",
       "20641                          NaN                   NaN                 NaN   \n",
       "\n",
       "       hs. Katalog Image URL  dup_title  copy_from Einfache Klassifizierung   \n",
       "657                      NaN        1.0       -1.0                    False  \\\n",
       "12124                    NaN        0.0       -1.0                    False   \n",
       "16795                    NaN        0.0       -1.0                    False   \n",
       "17957                    NaN        1.0       -1.0                    False   \n",
       "18105                    NaN        0.0       -1.0                    False   \n",
       "19065                    NaN        1.0       -1.0                    False   \n",
       "19066                    NaN        1.0       -1.0                    False   \n",
       "20641                    NaN        1.0       -1.0                    False   \n",
       "\n",
       "       Komplexe Klassifizierung  AC Nummer  \n",
       "657                           1             \n",
       "12124                         0             \n",
       "16795                         0             \n",
       "17957                         0             \n",
       "18105                         0             \n",
       "19065                         0             \n",
       "19066                         0             \n",
       "20641                         0             \n",
       "\n",
       "[8 rows x 38 columns]"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BE_df[BE_df['AC Nummer'] == '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "d73467a8-a7d2-44ad-8537-b281beaea2fa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AC Nummer</th>\n",
       "      <th>hs. Katalogeintrag ID</th>\n",
       "      <th>hs. Katalog Konfidenz</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23376</th>\n",
       "      <td>AC09684749</td>\n",
       "      <td>14.378_443_03</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23383</th>\n",
       "      <td>AC09684749</td>\n",
       "      <td>14.378_443_03</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23384</th>\n",
       "      <td>AC09684749</td>\n",
       "      <td>14.378_443_03</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23385</th>\n",
       "      <td>AC09684749</td>\n",
       "      <td>14.378_443_03</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23386</th>\n",
       "      <td>AC09684749</td>\n",
       "      <td>14.378_443_03</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22693</th>\n",
       "      <td>AC10057410</td>\n",
       "      <td>14.376_423_10</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23282</th>\n",
       "      <td>AC10097073</td>\n",
       "      <td>14.377_059_01</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22650</th>\n",
       "      <td>AC09771765</td>\n",
       "      <td>14.376_341_05</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22651</th>\n",
       "      <td>AC09836828</td>\n",
       "      <td>14.378_367_03</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23439</th>\n",
       "      <td>AC10221894</td>\n",
       "      <td>14.376_340_01</td>\n",
       "      <td>sicher</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>801 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        AC Nummer hs. Katalogeintrag ID hs. Katalog Konfidenz\n",
       "23376  AC09684749         14.378_443_03                sicher\n",
       "23383  AC09684749         14.378_443_03                sicher\n",
       "23384  AC09684749         14.378_443_03                sicher\n",
       "23385  AC09684749         14.378_443_03                sicher\n",
       "23386  AC09684749         14.378_443_03                sicher\n",
       "...           ...                   ...                   ...\n",
       "22693  AC10057410         14.376_423_10                sicher\n",
       "23282  AC10097073         14.377_059_01                sicher\n",
       "22650  AC09771765         14.376_341_05                sicher\n",
       "22651  AC09836828         14.378_367_03                sicher\n",
       "23439  AC10221894         14.376_340_01                sicher\n",
       "\n",
       "[801 rows x 3 columns]"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches_sig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f61076e-8384-4080-9003-06b9ea774fe6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "matches_sig = BE_df[['AC Nummer', 'hs. Katalogeintrag ID', 'hs. Katalog Konfidenz']].dropna(subset=['hs. Katalogeintrag ID'])\n",
    "matches_sig = matches_sig.loc[has_hw_matches['input_id']]\n",
    "\n",
    "for i, m in matches_sig.iterrows():\n",
    "    ac_num = m['AC Nummer']\n",
    "    if ac_num == '':\n",
    "        continue\n",
    "    hs_id = m['hs. Katalogeintrag ID']\n",
    "    hs_cert = m['hs. Katalog Konfidenz']\n",
    "    \n",
    "    if ' | ' in hs_id:\n",
    "        hs_id = hs_id.split(' | ')\n",
    "        hs_cert = hs_cert.split(' | ')\n",
    "    else:\n",
    "        hs_id = [hs_id]\n",
    "        hs_cert = [hs_cert]\n",
    "    \n",
    "    for hs_i, hs_c in zip(hs_id, hs_cert):\n",
    "        ind = entry_df[entry_df['entry_ID'] == hs_i].index.values[0]\n",
    "        \n",
    "        if (entry_df.at[ind, 'AC number'] == -1) or (entry_df.at[ind, 'AC number'] is None):\n",
    "            entry_df.at[ind, 'AC number'] = [ac_num]\n",
    "            entry_df.at[ind, 'AC cert'] = [hs_c]\n",
    "        else:\n",
    "            entry_df.at[ind, 'AC number'] += [ac_num]\n",
    "            entry_df.at[ind, 'AC cert'] += [hs_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "f83d1d13-5dee-4bf3-a06d-ed58b643adfa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BE_df.to_excel('data/wip_BE_data/BE_df_WIP.xlsx')\n",
    "entry_df.to_excel('data/wip_BE_data/entry_df_WIP.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "a21f6db7-19c0-4bf4-a185-d94c647750a2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BE_df = pd.read_excel('data/wip_BE_data/BE_df_WIP.xlsx', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "3aa2e989-255e-42d9-aa9f-986772b478cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "entry_df = pd.read_excel('data/wip_BE_data/entry_df_WIP.xlsx', index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e94b63a-228e-44ae-a8b4-9770ae5fba36",
   "metadata": {},
   "source": [
    "## Finalize BE data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "e7223020-e040-4601-823d-19f219373e00",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "no_BE_matches = pd.read_excel('../Daten/Katalogabgleich/Ausgefüllt/Ausgefüllt_no_BE.xlsx')\n",
    "has_hw_matches = no_BE_matches.dropna(subset='control')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "69ee2de7-a0e1-4f4c-b154-99931abb9765",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "826\n"
     ]
    }
   ],
   "source": [
    "def get_matches_from_entry(s):\n",
    "    certain_match = re.compile('\\d{1,4}\\Z|\\d{1,4},')\n",
    "    certs = certain_match.findall(s)\n",
    "    uncertain_match = re.compile('\\d{1,4}\\?')\n",
    "    uncerts = uncertain_match.findall(s)\n",
    "    clean_certs = [s.replace(',', '') for s in certs]\n",
    "    clean_uncerts = [s.replace('?', '') for s in uncerts]\n",
    "    return clean_certs, clean_uncerts\n",
    "\n",
    "dod_id = {\n",
    "    '14.376': 51202, \n",
    "    '14.377': 51184,\n",
    "    '14.378': 51219\n",
    "}\n",
    "\n",
    "repl = 0\n",
    "\n",
    "for i, entry in has_hw_matches.iterrows():\n",
    "    BE_id = entry['input_id']\n",
    "    man_match = str(entry['control'])\n",
    "    certain, uncertain = get_matches_from_entry(man_match)\n",
    "    match_dict = {\n",
    "        'hs. Katalog': 1,\n",
    "        'hs. Katalog Konfidenz': [],\n",
    "        'hs. Katalogband': [],\n",
    "        'hs. Katalogseite Digitalisat': [],\n",
    "        'Wissensklasse': [],\n",
    "        'Wissensunterklasse': [],\n",
    "        'Formatangabe': [],\n",
    "        'hs. Katalogseite Handschrift': [],\n",
    "        'hs. Katalogeintrag ID': [],\n",
    "        'hs. Katalogeintrag': [],\n",
    "        'hs. Katalog Image URL': []\n",
    "    }\n",
    "    for m in certain:\n",
    "        match_dict['hs. Katalog Konfidenz'] += ['sicher']\n",
    "    for m in uncertain:\n",
    "        match_dict['hs. Katalog Konfidenz'] += ['unsicher']\n",
    "    \n",
    "    all_matches = certain + uncertain\n",
    "    repl += len(all_matches)\n",
    "    \n",
    "    for m in all_matches:\n",
    "        hw_entry = entry_df.loc[int(m)]\n",
    "        match_dict['hs. Katalogband'] += [str(hw_entry['volume'])]\n",
    "        match_dict['hs. Katalogseite Digitalisat'] += [str(hw_entry['page number'])]\n",
    "        match_dict['Wissensklasse'] += [hw_entry['category']]\n",
    "        match_dict['Wissensunterklasse'] += [hw_entry['subcategory'] if not pd.isna(hw_entry['subcategory']) else '']\n",
    "        match_dict['Formatangabe'] += [hw_entry['format'] if not pd.isna(hw_entry['format']) else '']\n",
    "        match_dict['hs. Katalogseite Handschrift'] += [hw_entry['handwritten page number']]\n",
    "        match_dict['hs. Katalogeintrag ID'] += [hw_entry['entry_ID']]\n",
    "        match_dict['hs. Katalogeintrag'] += [hw_entry['entry']]\n",
    "        match_dict['hs. Katalog Image URL'] += [f\"https://iiif.onb.ac.at/images/DOD/{dod_id[str(hw_entry['volume'])]}/{hw_entry['page number']:08}.jp2/full/full/0/native.jpg\"]\n",
    "    \n",
    "    for key, val in match_dict.items():\n",
    "        if key != 'hs. Katalog':\n",
    "            val = ' | '.join(val)\n",
    "        BE_df.at[BE_id, key] = val\n",
    "\n",
    "print(repl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "10e58748-acb8-48c7-b4c7-a1953f2a70a4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_BE_df = BE_df.drop(['dup_title', 'copy_from'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "ee4e5056-d53d-4288-ba06-41af7e6cd3d4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_BE_df.to_excel('../Daten/Vorhersagen/BE_final.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "32fc7d42-2093-4a54-9003-00b4a03e9558",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volume</th>\n",
       "      <th>page number</th>\n",
       "      <th>category</th>\n",
       "      <th>subcategory</th>\n",
       "      <th>format</th>\n",
       "      <th>handwritten page number</th>\n",
       "      <th>entry_ID</th>\n",
       "      <th>entry</th>\n",
       "      <th>AC numbers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.376</td>\n",
       "      <td>25</td>\n",
       "      <td>Theologia</td>\n",
       "      <td>Textus &amp; Versiones Sacræ Scripturæ</td>\n",
       "      <td>Folio</td>\n",
       "      <td>1</td>\n",
       "      <td>14.376_025_00</td>\n",
       "      <td>1 Biblia Sacra Polyglotta curis Cardinalis Xim...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14.376</td>\n",
       "      <td>25</td>\n",
       "      <td>Theologia</td>\n",
       "      <td>Textus &amp; Versiones Sacræ Scripturæ</td>\n",
       "      <td>Folio</td>\n",
       "      <td>1</td>\n",
       "      <td>14.376_025_01</td>\n",
       "      <td>2 Biblia Sacra Polyglotta Philippi II. Regis C...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.376</td>\n",
       "      <td>25</td>\n",
       "      <td>Theologia</td>\n",
       "      <td>Textus &amp; Versiones Sacræ Scripturæ</td>\n",
       "      <td>Folio</td>\n",
       "      <td>1</td>\n",
       "      <td>14.376_025_02</td>\n",
       "      <td>3 Biblia Sacra Polyglotta Studio &amp; curâ Briani...</td>\n",
       "      <td>[(AC09762517, sicher)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14.376</td>\n",
       "      <td>25</td>\n",
       "      <td>Theologia</td>\n",
       "      <td>Textus &amp; Versiones Sacræ Scripturæ</td>\n",
       "      <td>Folio</td>\n",
       "      <td>1</td>\n",
       "      <td>14.376_025_03</td>\n",
       "      <td>4 Biblia Sacra Latina Moguntina dicta, prima o...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14.376</td>\n",
       "      <td>25</td>\n",
       "      <td>Theologia</td>\n",
       "      <td>Textus &amp; Versiones Sacræ Scripturæ</td>\n",
       "      <td>Folio</td>\n",
       "      <td>1</td>\n",
       "      <td>14.376_025_04</td>\n",
       "      <td>5 Biblia Sacra Latina Moguntina, editio altera...</td>\n",
       "      <td>[(AC07081950, sicher)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9398</th>\n",
       "      <td>14.378</td>\n",
       "      <td>582</td>\n",
       "      <td>Imaginum Delineatarum Collectio</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1402</td>\n",
       "      <td>14.378_582_00</td>\n",
       "      <td>CCCXXXI Vn Portefeüilles contenant des Dessein...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9399</th>\n",
       "      <td>14.378</td>\n",
       "      <td>582</td>\n",
       "      <td>Imaginum Delineatarum Collectio</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1402</td>\n",
       "      <td>14.378_582_01</td>\n",
       "      <td>CCCXXXII  Vn Recueil des Portraits peints en m...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9400</th>\n",
       "      <td>14.378</td>\n",
       "      <td>582</td>\n",
       "      <td>Imaginum Delineatarum Collectio</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1402</td>\n",
       "      <td>14.378_582_02</td>\n",
       "      <td>CCCXXXIII  Dix Vol. de Plantes peintes en mini...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9401</th>\n",
       "      <td>14.378</td>\n",
       "      <td>582</td>\n",
       "      <td>Imaginum Delineatarum Collectio</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1402</td>\n",
       "      <td>14.378_582_03</td>\n",
       "      <td>CCCXXXIV  Cinq Vol. d'Oiseaux peints en Miniat...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9402</th>\n",
       "      <td>14.378</td>\n",
       "      <td>582</td>\n",
       "      <td>Imaginum Delineatarum Collectio</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1402</td>\n",
       "      <td>14.378_582_04</td>\n",
       "      <td>CCCXXXV Divers Portraits, Ceremonies, Marches ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9403 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      volume  page number                         category   \n",
       "0     14.376           25                        Theologia  \\\n",
       "1     14.376           25                        Theologia   \n",
       "2     14.376           25                        Theologia   \n",
       "3     14.376           25                        Theologia   \n",
       "4     14.376           25                        Theologia   \n",
       "...      ...          ...                              ...   \n",
       "9398  14.378          582  Imaginum Delineatarum Collectio   \n",
       "9399  14.378          582  Imaginum Delineatarum Collectio   \n",
       "9400  14.378          582  Imaginum Delineatarum Collectio   \n",
       "9401  14.378          582  Imaginum Delineatarum Collectio   \n",
       "9402  14.378          582  Imaginum Delineatarum Collectio   \n",
       "\n",
       "                             subcategory format handwritten page number   \n",
       "0     Textus & Versiones Sacræ Scripturæ  Folio                       1  \\\n",
       "1     Textus & Versiones Sacræ Scripturæ  Folio                       1   \n",
       "2     Textus & Versiones Sacræ Scripturæ  Folio                       1   \n",
       "3     Textus & Versiones Sacræ Scripturæ  Folio                       1   \n",
       "4     Textus & Versiones Sacræ Scripturæ  Folio                       1   \n",
       "...                                  ...    ...                     ...   \n",
       "9398                                 NaN    NaN                    1402   \n",
       "9399                                 NaN    NaN                    1402   \n",
       "9400                                 NaN    NaN                    1402   \n",
       "9401                                 NaN    NaN                    1402   \n",
       "9402                                 NaN    NaN                    1402   \n",
       "\n",
       "           entry_ID                                              entry   \n",
       "0     14.376_025_00  1 Biblia Sacra Polyglotta curis Cardinalis Xim...  \\\n",
       "1     14.376_025_01  2 Biblia Sacra Polyglotta Philippi II. Regis C...   \n",
       "2     14.376_025_02  3 Biblia Sacra Polyglotta Studio & curâ Briani...   \n",
       "3     14.376_025_03  4 Biblia Sacra Latina Moguntina dicta, prima o...   \n",
       "4     14.376_025_04  5 Biblia Sacra Latina Moguntina, editio altera...   \n",
       "...             ...                                                ...   \n",
       "9398  14.378_582_00  CCCXXXI Vn Portefeüilles contenant des Dessein...   \n",
       "9399  14.378_582_01  CCCXXXII  Vn Recueil des Portraits peints en m...   \n",
       "9400  14.378_582_02  CCCXXXIII  Dix Vol. de Plantes peintes en mini...   \n",
       "9401  14.378_582_03  CCCXXXIV  Cinq Vol. d'Oiseaux peints en Miniat...   \n",
       "9402  14.378_582_04  CCCXXXV Divers Portraits, Ceremonies, Marches ...   \n",
       "\n",
       "                  AC numbers  \n",
       "0                        NaN  \n",
       "1                        NaN  \n",
       "2     [(AC09762517, sicher)]  \n",
       "3                        NaN  \n",
       "4     [(AC07081950, sicher)]  \n",
       "...                      ...  \n",
       "9398                     NaN  \n",
       "9399                     NaN  \n",
       "9400                     NaN  \n",
       "9401                     NaN  \n",
       "9402                     NaN  \n",
       "\n",
       "[9403 rows x 9 columns]"
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_entry_df = entry_df.drop(['AC number', 'AC cert', 'AC comb', 'AC comb set len'], axis=1)\n",
    "final_entry_df.rename(columns={'AC comb set': 'AC numbers'}, inplace=True)\n",
    "final_entry_df['AC numbers'].replace(to_replace=-1, value=np.nan, inplace=True)\n",
    "final_entry_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "83c0b002-6cf5-4677-8b84-a39d8815daa1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_entry_df.to_excel('../Daten/Vorhersagen/hw_entries_final.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "0746de2c-3343-48b1-b921-4215f594ab79",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "entry_df.at[8510, 'AC number'] = \"['AC07010383']\"\n",
    "entry_df.at[8510, 'AC cert'] = \"['sicher']\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "7a448905-e284-44b9-bdb1-38710466b341",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1\n",
      "['AC09762517']\n"
     ]
    }
   ],
   "source": [
    "def read_list_from_str(s):\n",
    "    if s != -1:\n",
    "        lis = eval(s)\n",
    "        return lis\n",
    "    else:\n",
    "        return s\n",
    "\n",
    "print(read_list_from_str(\"-1\"))\n",
    "print(read_list_from_str(\"['AC09762517']\"))\n",
    "\n",
    "entry_df['AC number'] = entry_df['AC number'].apply(lambda x: read_list_from_str(x))\n",
    "entry_df['AC cert'] = entry_df['AC cert'].apply(lambda x: read_list_from_str(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27eda982-6931-4c31-addb-912bbe743cdf",
   "metadata": {},
   "source": [
    "## Add `<idno>` tags for n signature using simple string replacement (ignoring nested occurrences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "02ccd717-a6e4-4754-b617-485b16b73ca7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.376_tei.xml', 'r') as tei_xml_input:\n",
    "    f1 = tei_xml_input.readlines()\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.377_tei.xml', 'r') as tei_xml_input:\n",
    "    f2 = tei_xml_input.readlines()\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.378_tei.xml', 'r') as tei_xml_input:\n",
    "    f3 = tei_xml_input.readlines()\n",
    "\n",
    "raw_files = [f1, f2, f3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "2792c546-aafc-4224-97a3-4f8db5fc585c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for file in raw_files:\n",
    "    for i, line in enumerate(file):\n",
    "        n_re = re.compile(' ([nN]\\.? ?\\d{1,4})')\n",
    "        match = re.search(n_re, line)\n",
    "        if match:\n",
    "            n_tag = f'<idno type=\"n_signature\">{match[1]}</idno>'\n",
    "            file[i] = line.replace(match[1], n_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "c4baa4ed-65de-4c20-99a3-275fb49ed6e9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.376_tei.xml', 'w') as out:\n",
    "    out.writelines(f1)\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.377_tei.xml', 'w') as out:\n",
    "    out.writelines(f2)\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.378_tei.xml', 'w') as out:\n",
    "    out.writelines(f3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ac728dd-4fe8-4d8d-9049-5891b556a19e",
   "metadata": {},
   "source": [
    "## Parse files as XML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "id": "ae4d5368-bda2-4cdf-a170-f7e0eda103c3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.376_tei.xml', 'r') as tei_xml_input:\n",
    "    content = tei_xml_input.read()\n",
    "    tei_1 = bs(content, \"lxml-xml\")\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.377_tei.xml', 'r') as tei_xml_input:\n",
    "    content = tei_xml_input.read()\n",
    "    tei_2 = bs(content, \"lxml-xml\")\n",
    "\n",
    "with open('../../digital-edition/Step1_Aufbereitung/Cod. 14.378_tei.xml', 'r') as tei_xml_input:\n",
    "    content = tei_xml_input.read()\n",
    "    tei_3 = bs(content, \"lxml-xml\")\n",
    "\n",
    "dod_ids = [51202, 51184, 51219]\n",
    "cod_prefixes = {51202: '14.376', 51184: '14.377', 51219: '14.378'}\n",
    "tei = {51202: tei_1, 51184: tei_2, 51219: tei_3}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d59c6a3a-e06b-4beb-a273-9594f8473cbf",
   "metadata": {},
   "source": [
    "## Add `<div type=\"knowledge_class\"></div>` tags and modify pagenumber tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "id": "b73227d4-a0f1-4113-ad2c-093507aaf3a4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    headers = tei[key].find_all(\"div\", type=\"head\")\n",
    "    for head in headers:\n",
    "        cat = head.find(type=\"category\")\n",
    "        subcat = head.find(type=\"subcategory\")\n",
    "        if cat is not None:\n",
    "            cat['subtype'] = cat['value']\n",
    "            del cat['value']\n",
    "            cat = cat.extract()\n",
    "            cat_lis = [cat]\n",
    "            if subcat is not None:\n",
    "                subcat['subtype'] = subcat['value']\n",
    "                del subcat['value']\n",
    "                subcat = subcat.extract()\n",
    "                cat_lis.append(subcat)\n",
    "            new_div = bs('''<div type=\"knowledge_class\"></div>''', \"lxml-xml\")\n",
    "            new_div.div.extend(cat_lis)\n",
    "            head.insert(1, new_div)\n",
    "\n",
    "        form = head.find(type=\"format\")\n",
    "        if form is not None:\n",
    "            form['subtype'] = form['value']\n",
    "            del form['value']\n",
    "        pnum = head.find(type=\"pagenumber\")\n",
    "        if pnum is not None:\n",
    "            pnum['type'] = 'pageNum'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46340680-bc79-4bcb-a4ab-d17c9883ba86",
   "metadata": {},
   "source": [
    "## Add links to modern catalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "id": "ebb4894b-76db-4146-b62a-86c30f2f8609",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def zip_without_m1(row):\n",
    "    if row[0] == -1:\n",
    "        return -1\n",
    "    else:\n",
    "        return list(zip(row[0], row[1]))\n",
    "\n",
    "entry_df['AC comb'] = entry_df[['AC number', 'AC cert']].apply(lambda x: zip_without_m1(x), axis=1)\n",
    "entry_df['AC comb set'] = entry_df['AC comb'].apply(lambda x: list(set(x)) if x != -1 else -1)\n",
    "entry_df['AC comb set len'] = entry_df['AC comb set'].apply(lambda x: len(x) if x != -1 else -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "id": "ba37d08d-f064-4ba0-aaf5-d33c9893bb1c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "links = entry_df[entry_df['AC number'] != -1]\n",
    "p1 = tei_1.find_all('pb')\n",
    "p2 = tei_2.find_all('pb')\n",
    "p3 = tei_3.find_all('pb')\n",
    "\n",
    "ps = {\n",
    "    \"14.376\": p1,\n",
    "    \"14.377\": p2,\n",
    "    \"14.378\": p3\n",
    "}\n",
    "\n",
    "for i, row in links.iterrows():\n",
    "    entry = ps[str(row['volume'])][row['page number'] - 1].find_next('p').find_all(type='entry')[int(row['entry_ID'][-2:])]\n",
    "    ref_lis = []\n",
    "    for ref in row['AC comb set']:\n",
    "        ref_tag = bs(f'''<ref cert=\"{'high' if ref[1] == 'sicher' else 'low'}\" target=\"https://data.onb.ac.at/rec/{ref[0]}\" type=\"catalog\"/>''', \"lxml-xml\")\n",
    "        ref_lis.append(ref_tag)\n",
    "    entry.extend(ref_lis)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "deadc616-1dc6-4d7e-9392-5a92923026fc",
   "metadata": {},
   "source": [
    "## Remove `choice`, `expan` and `abbr` tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "id": "629a49b3-75c7-4e3d-b3aa-5c6165c32e06",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    tags = tei[key].find_all('choice') + tei[key].find_all('expan') + tei[key].find_all('abbr')\n",
    "    for tag in tags:\n",
    "        tag.unwrap()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5c447ce-40c1-4f5b-b8f5-41d5d4bb5558",
   "metadata": {},
   "source": [
    "## Move `note` tags to `ref` tags if there is no corresponding `ref` already"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "ec90629a-8afb-4a65-b6bb-782830fd8bcf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                                    Falsche Jahresangabe\n",
      "                                \n",
      "\n",
      "                                        Guinee, altes Synonym für Pfund?\n",
      "                                    \n",
      "\n",
      "                                Falsche Nummerierung (139 wäre richtig)\n",
      "                            \n"
     ]
    }
   ],
   "source": [
    "for key in tei:\n",
    "    notes = tei[key].find_all('note')\n",
    "    for note in notes:\n",
    "        re_ac = re.compile('AC\\d{8}')\n",
    "        match_ac = re.search(re_ac, str(note.string))\n",
    "        if not match_ac:\n",
    "            print(note.string)\n",
    "        else:\n",
    "            note_ac = match_ac[0]\n",
    "            entry_div = note.find_previous('div', type='entry')\n",
    "            refs = entry_div.find_all('ref')\n",
    "            if refs is not None:\n",
    "                is_new_ac = True\n",
    "                for ref in refs:\n",
    "                    ref_ac = ref['target'].split('/')[-1]\n",
    "                    if note_ac == ref_ac:\n",
    "                        is_new_ac = False\n",
    "                        break\n",
    "                if is_new_ac:\n",
    "                    new_ref_tag = bs(f'''<ref cert=\"high\" target=\"https://data.onb.ac.at/rec/{note_ac}\" type=\"catalog\"/>''', \"lxml-xml\")\n",
    "                    entry_div.append(new_ref_tag)\n",
    "            note.decompose()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa8a809c-adb7-44c0-9c8b-4a762bbeace3",
   "metadata": {},
   "source": [
    "## Add information to `<idno>` tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "id": "bd52a621-e46c-49d8-a5c5-a243ee85fba4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    idnos = tei[key].find_all('idno')\n",
    "    for id_tag in idnos:\n",
    "        num = re.search('\\d{1,4}', id_tag.string)[0]\n",
    "        form = id_tag.find_previous(type='format')['subtype']\n",
    "        id_tag['subtype'] = f'{form}, {num}'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d15e8da2-da31-4068-8aae-320f5534c523",
   "metadata": {},
   "source": [
    "## Change `roetel`, `bleistift` and `add` tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "id": "a4a7bc48-ec82-4b67-b0dd-055d27dd008e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    adds = tei[key].find_all('add')\n",
    "    for tag in adds:\n",
    "        tag['hand'] = 'other'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "id": "710a147b-ea34-47ca-a381-5b477d8f12ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    bls = tei[key].find_all('bleistift')\n",
    "    for tag in bls:\n",
    "        tag.name = 'add'\n",
    "        tag['hand'] = 'other'\n",
    "        tag['rend'] = 'bleistift'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "e7e81cbf-e473-4ae4-b846-2aea6b2e5c81",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    roet = tei[key].find_all('roetel')\n",
    "    for tag in roet:\n",
    "        tag.name = 'add'\n",
    "        tag['hand'] = 'other'\n",
    "        tag['rend'] = 'roetel'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ef5fcf1-74fe-4c55-95a8-a2a8ffe4e265",
   "metadata": {},
   "source": [
    "## Remove `<continued/>` tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "id": "de097669-9180-4510-a9ac-d215433a7fca",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    contd = tei[key].find_all('continued')\n",
    "    for tag in contd:\n",
    "        tag.decompose()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b699159-7ac2-4ced-9cb0-4a5e7e10df34",
   "metadata": {},
   "source": [
    "## Combine nested `<add>` tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "4687407c-d205-4aab-8c86-fafa73b997b9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    adds = tei[key].find_all('add')\n",
    "    for tag in adds:\n",
    "        for child in tag.children:\n",
    "            if child.name == 'add':\n",
    "                if (('rend' not in tag.attrs) and ('rend' in child.attrs)):\n",
    "                    tag.unwrap()\n",
    "                if (('rend' in tag.attrs) and ('rend' not in child.attrs)):\n",
    "                    child.unwrap()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4a47880-5ecf-41e1-b2ea-624bfb301c99",
   "metadata": {},
   "source": [
    "## Change `<num>` tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "id": "32240f6b-1ecc-47d1-8f2d-0cd14f899b3d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    num_lis = tei[key].find_all('num')\n",
    "    for tag in num_lis:\n",
    "        num = re.search('\\d{1,4}', tag.string)[0]\n",
    "        form = tag.find_previous(type='format')['subtype']\n",
    "        tag.name = 'idno'\n",
    "        tag['type'] = 'n_signature'\n",
    "        tag['subtype'] = f'{form}, {num}'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a34042b-f93a-4072-884f-66fef48d6a30",
   "metadata": {},
   "source": [
    "## List all tags in the documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "id": "55b131a0-58aa-4eb1-ba47-92b3bb46ffaa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\n",
      "['TEI', 'ab', 'add', 'bibl', 'body', 'change', 'corr', 'div', 'encodingDesc', 'facsimile', 'fileDesc', 'graphic', 'hi', 'idno', 'l', 'listChange', 'note', 'p', 'pb', 'profileDesc', 'publicationStmt', 'publisher', 'ref', 'revisionDesc', 'sic', 'sourceDesc', 'surface', 'teiHeader', 'text', 'title', 'titleStmt', 'unclear', 'xenoData', 'zone']\n"
     ]
    }
   ],
   "source": [
    "all_tags = set()\n",
    "for key in tei:\n",
    "    tag_lis = tei[key].find_all()\n",
    "    for tag in tag_lis:\n",
    "        all_tags.add(tag.name)\n",
    "\n",
    "print(len(all_tags))\n",
    "print(sorted(list(all_tags)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85b00fab-7faa-43c4-a294-b0e5febc8024",
   "metadata": {},
   "source": [
    "## Add description of changes for second step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "a50d2147-4ee5-4d6b-a950-b4aca1f5e5ca",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tei:\n",
    "    change = bs('''<change from=\"2024-01-18\" to=\"2024-10-10\" who=\"#SM\">\n",
    "Layout analysis of pages to obtain entries. Create header for all pages with knowledge classes, page number, and format specification if applicable. Added matching data for entries connecting them to the modern catalog. Add tags for n signature.\n",
    "</change>\n",
    "''', \"lxml-xml\")\n",
    "    listChange = tei[key].listChange\n",
    "    listChange.append(change)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed89defa-7231-42e7-bab8-2b235eee5b24",
   "metadata": {},
   "source": [
    "## Export TEI_XML files to Step 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "id": "ee793bf1-5a43-4715-8b2b-b5780d6fd8b3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step2_Matching/Cod. 14.376_tei.xml', 'w') as tei_xml_output:\n",
    "    tei_xml_output.write(tei_1.prettify(formatter='minimal'))\n",
    "with open('../../digital-edition/Step2_Matching/Cod. 14.377_tei.xml', 'w') as tei_xml_output:\n",
    "    tei_xml_output.write(tei_2.prettify(formatter='minimal'))\n",
    "with open('../../digital-edition/Step2_Matching/Cod. 14.378_tei.xml', 'w') as tei_xml_output:\n",
    "    tei_xml_output.write(tei_3.prettify(formatter='minimal'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f92f803b-a4d6-473a-955a-d8c174fc72ad",
   "metadata": {},
   "source": [
    "## Fix additional line breaks originating from stripped tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "id": "092825b9-9755-4658-8fd5-2183926ed981",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step2_Matching/Cod. 14.376_tei.xml', 'rb') as tei_xml:\n",
    "    xml_1 = tei_xml.read()\n",
    "with open('../../digital-edition/Step2_Matching/Cod. 14.377_tei.xml', 'rb') as tei_xml:\n",
    "    xml_2 = tei_xml.read()\n",
    "with open('../../digital-edition/Step2_Matching/Cod. 14.378_tei.xml', 'rb') as tei_xml:\n",
    "    xml_3 = tei_xml.read()\n",
    "\n",
    "xmls = [xml_1, xml_2, xml_3]\n",
    "\n",
    "str_xsl = '''<xsl:stylesheet version=\"1.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n",
    "    <xsl:output indent=\"yes\"/>\n",
    "    <xsl:strip-space elements=\"*\"/>\n",
    "\n",
    "    <!-- IDENTITY TRANSFORM -->\n",
    "    <xsl:template match=\"@*|node()\">\n",
    "      <xsl:copy>\n",
    "        <xsl:apply-templates select=\"@*|node()\"/>\n",
    "      </xsl:copy>\n",
    "    </xsl:template>\n",
    "\n",
    "    <!-- RUN normalize-space() ON ALL TEXT NODES -->\n",
    "    <xsl:template match=\"text()\">\n",
    "        <xsl:copy-of select=\"normalize-space()\"/>\n",
    "    </xsl:template>            \n",
    "</xsl:stylesheet>\n",
    "'''\n",
    "\n",
    "for i, xml in enumerate(xmls):\n",
    "    str_xml = xml\n",
    "    doc = et.fromstring(str_xml)\n",
    "    style = et.fromstring(str_xsl)\n",
    "    transformer = et.XSLT(style)\n",
    "    result = transformer(doc)\n",
    "    str_res = str(result)\n",
    "    # str_res = re.sub('(?<=[>\\S])<idno', ' <idno', str_res)\n",
    "    str_res = re.sub('\"n_signature\">', '\"n_signature\"> ', str_res)\n",
    "    # str_res = re.sub('(?<=[>\\S])<hi', ' <hi', str_res)\n",
    "    with open(f'../../digital-edition/Step3_Header/Cod. 14.37{i+6}_tei.xml', 'w') as f:\n",
    "         f.write(str_res)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "636031ca-c748-478d-8a26-6175a9fd004e",
   "metadata": {},
   "source": [
    "## Add `<idno type=\"PID\">o:bed.cod-14.3XX</idno>` and `<msDesc>` to header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "id": "63477b16-cda4-4eea-a27d-4ec0c5ceea6c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('../../digital-edition/Step3_Header/Cod. 14.376_tei.xml', 'rb') as tei_xml:\n",
    "    xml_1 = tei_xml.read()\n",
    "with open('../../digital-edition/Step3_Header/Cod. 14.377_tei.xml', 'rb') as tei_xml:\n",
    "    xml_2 = tei_xml.read()\n",
    "with open('../../digital-edition/Step3_Header/Cod. 14.378_tei.xml', 'rb') as tei_xml:\n",
    "    xml_3 = tei_xml.read()\n",
    "\n",
    "xmls = [xml_1, xml_2, xml_3]\n",
    "\n",
    "ns = {\n",
    "    'xml': 'http://www.tei-c.org/ns/1.0'\n",
    "}\n",
    "\n",
    "msDescIds = [\n",
    "    {\n",
    "        'AC': 'AC13948858',\n",
    "        'REPO': '131ABCFD',\n",
    "        'DOD': '51202'\n",
    "    },\n",
    "    {\n",
    "        'AC': 'AC13956022',\n",
    "        'REPO': '131ABD07',\n",
    "        'DOD': '51184'\n",
    "    },\n",
    "    {\n",
    "        'AC': 'AC13956023',\n",
    "        'REPO': '131ABD79',\n",
    "        'DOD': '51219'\n",
    "    }\n",
    "]\n",
    "\n",
    "for i, xml in enumerate(xmls):\n",
    "    doc = et.XML(xml)\n",
    "    pubStmt = doc.find('.//xml:publicationStmt', namespaces=ns)\n",
    "    pubStmt.append(et.XML(f'<idno type=\"PID\">o>bed.cod-14.37{i+6}</idno>', base_url=ns['xml']))\n",
    "    \n",
    "    title = doc.find('.//xml:title[@type=\"main\"]', namespaces=ns)\n",
    "    title.text = f'Catalogus Librorum Bibliothecae [...] Principis Eugenii [...] Tomus {\"I\"*(i+1)}'\n",
    "    \n",
    "    msDesc = et.XML(f'''\n",
    "    <msDesc>\n",
    "      <msIdentifier>\n",
    "        <settlement>Wien</settlement>\n",
    "        <repository>Österreichische Nationalbibliothek</repository>\n",
    "        <collection type=\"main\">Sammlung von Handschriften und alten Drucken</collection>\n",
    "        <idno type=\"sn\">Signatur: Cod. 1437{i+6} HAN MAG</idno>\n",
    "        <altIdentifier>\n",
    "          <idno type=\"AC\">{msDescIds[i]['AC']}</idno>\n",
    "        </altIdentifier>\n",
    "        <altIdentifier>\n",
    "          <idno type=\"REPO\">{msDescIds[i]['REPO']}</idno>\n",
    "        </altIdentifier>\n",
    "        <altIdentifier>\n",
    "          <idno type=\"DOD\">{msDescIds[i]['DOD']}</idno>\n",
    "        </altIdentifier>\n",
    "      </msIdentifier>\n",
    "    </msDesc>\n",
    "''', base_url=ns['xml'])\n",
    "    sourceDesc = doc.find('.//xml:sourceDesc', namespaces=ns)\n",
    "    sourceDesc.append(msDesc)\n",
    "    \n",
    "    with open(f'../../digital-edition/Step3_Header/Cod. 14.37{i+6}_tei.xml', 'wb') as f:\n",
    "        f.write(et.tostring(doc, encoding='utf-8', xml_declaration=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c43a4852-4c90-4ba3-bf14-70e1688c0d74",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
