README!

.idea/other.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PySciProjectComponent">
+    <option name="PY_SCI_VIEW_SUGGESTED" value="true" />
+  </component>
+</project>
\ No newline at end of file

README.MD

+# Alma Data Extractor
+
+Takes a list of ac-numbers, makes requests to sru/alma/önb, extracts data from the returned marc:xml, brings it in tabular form and writes it to an excel.
+
+## Concept
+
+The program is mostly written async, so that it can process the data, while watiting for new requests.
+
+The main controller is `travelogues_extraction.controller.main.FromAlmaOutputToExcel`. 
+
+1. It takes an path to an excel and generates ac numbers from a column in that excel.
+2. Makes requests to sru/önb and parses it with lxml, and take the first marc:record
+3. Runs a list of "Extractors" through each record, that populates th output dataframe
+
+
+### Change or Add to the software
+
+You will need a solid knowledge in python and xpath.
+
+#### 1. AC-Number-Generator
+
+If you would like to change the way the ac-numbers are generated, see here `travelogues_extraction/getrecords/acnumber_extractor.py`
+
+#### 2. Requests
+
+`travelogues_extraction.getrecords.session.RecordRetriever`
+
+You can change the url, the requests, the xpath for the records.
+
+#### 3. Extractors
+
+
+All Extractors inherit from `travelogues_extraction.dataextractors.abstract.AbstractDataExtractor`.
+
+There are a bunch of more detailed Abstract extractors:
+
+- `AbstractSingleDataExtractor`
+- `AbstractXpathDirectlyToColumn`
+- `AbstractXpathJoinDirectlyToColumn`
+- `AbstractMultifield`
+- `AbstractParentAsSecondCast`
+
+and all extractors in `travelogues_extraction/dataextractors/dataextractors` inherit from them
+
+A lot of the classes actually only have properties and no methods and it would be possible to just give this properties
+their parent classes in the constructor. However a lot them do have custom methods. The controller only needs to rely on 
+the interface of the top `AbstractDataExtractor` 
+
+
+##### 3.1. Order, Include / Exclude Extractors
+
+The columns in the dataframe will be generated in the order of the classes in `travelogues_extraction/controller/main.py:33`
+and will take the names of the columns in `travelogues_extraction.dataextractors.abstract.AbstractDataExtractor.get_columns_names_I_work_on` in their order.
+
+You can change the order of, include, exclude extractors there.
+
+##### 3.2. Write an data extractor
+
+Inherit from `travelogues_extraction.dataextractors.abstract.AbstractDataExtractor`, implement a method write, that 
+takes a `travelogues_extraction.getrecords.session.RecordRetriever.Record` as input and writes to 
+`self.target_dataframe.at[record.ac_number, self.column] = 'your data'. You will find the lxml parsed xml representation
+ in the `lxmlelement` the property of the record.
+ 
+ There are a few parent classes you can use:
+ 
+ 
+####### `AbstractXpathDirectlyToColumn`
+ 
+ Child classes define the column a xpath object and this class writes the first found text to the target dataframe. 
+ Example: `travelogues_extraction.dataextractors.dataextractors.index.MMSID`
+ 
+####### `AbstractXpathDirectlyToColumn`
+
+Child classes define the column a xpath object and this class joins the text results with property `join_string`
+
+###### `AbstractMultifield`
+
+Looks up data in the record with `primary_xml_path: lxmletree.XPath`, than it uses each of `xpath_isgnd_tuples`
+for data generation. `XpathIsGnd` consists of a xpath object, and tells if this is a datum containing gnd data.
+If so it wil render the output as uris. The output will be rendered as a string with one of the `join_string_*_level` 
+properties.
+
+Example: `travelogues_extraction.dataextractors.dataextractors.combinedsubfields.VerfasserGND`
+
+###### `AbstractParentAsSecondCast`
+
+The last looks in other records, if it does not find them in the current record. It takes `parent_ac_xpath`
+to look for the parent, which is called so, because in all our cases, it was a parent record. Implement your data 
+extraction in _write(). write() will take care of the rest and use your _write. 
+
+Example: `travelogues_extraction.dataextractors.dataextractors.übergeordnet.Schlagworte`
+`
+
+ 
+
+ 
+ 
+
+

travelogues_extraction/dataextractors/abstract.py

@@ -4,7 +4,6 @@ from dataclasses import dataclass
 import re as regex
 import typing
 
-import httpcore
 import httpx
 from pandas import DataFrame
 
@@ -66,7 +65,6 @@ class AbstractXpathJoinDirectlyToColumn(AbstractSingleDataExtractor):
             return result
 
 
-
 class AbstractMultifield(AbstractDataExtractor):
 
     column: str

travelogues_extraction/visualizeMassDownlaod.ipynb

-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import time"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: httpx in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (0.13.3)\r\n",
-      "Requirement already satisfied: certifi in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (2020.6.20)\r\n",
-      "Requirement already satisfied: rfc3986<2,>=1.3 in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (1.4.0)\r\n",
-      "Requirement already satisfied: httpcore==0.9.* in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (0.9.1)\r\n",
-      "Requirement already satisfied: hstspreload in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (2020.7.14)\r\n",
-      "Requirement already satisfied: chardet==3.* in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (3.0.4)\r\n",
-      "Requirement already satisfied: sniffio in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (1.1.0)\r\n",
-      "Requirement already satisfied: idna==2.* in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpx) (2.10)\r\n",
-      "Requirement already satisfied: h11<0.10,>=0.8 in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpcore==0.9.*->httpx) (0.9.0)\r\n",
-      "Requirement already satisfied: h2==3.* in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from httpcore==0.9.*->httpx) (3.2.0)\r\n",
-      "Requirement already satisfied: hpack<4,>=3.0 in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from h2==3.*->httpcore==0.9.*->httpx) (3.0.0)\r\n",
-      "Requirement already satisfied: hyperframe<6,>=5.2.0 in /home/phylogram/Documents/onb-homeoffice-local/TraveloguesExtraktion/venv/lib/python3.7/site-packages (from h2==3.*->httpcore==0.9.*->httpx) (5.2.0)\r\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install httpx"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'httpx'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-4-c0104c72085e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mcontroller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmain\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFromAlmaOutputToExcel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;32m~/Documents/onb-homeoffice-local/TraveloguesExtraktion/travelogues_extraction/controller/main.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mhttpx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTYPE_CHECKING\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'httpx'"
-     ]
-    }
-   ],
-   "source": [
-    "from controller.main import FromAlmaOutputToExcel"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}