import sys import time import requests from requests import HTTPError _datetime_format_string = '%Y%m%d%H%M%S' class SessionTimeoutError(Exception): pass class WebarchivSession: @property def version(self): """ Current protocol version """ return '0.1.0' @property def api_path(self): """ Protocol, domain and path prefix for the Webarchive API, with a single positional format string placeholder for the REST operation and parameters. """ return 'https://webarchiv.onb.ac.at/api/' @property def base_url(self): """ Protocol, domain and path prefix for the Webarchive API, with a single positional format string placeholder for the REST operation and parameters. """ return self.api_path + '/{}' @property def _error_template(self): """ A format string for displaying HTTP Errors. Must contain one placeholder 'status_code' for the HTTP status code. Must contain one placeholder 'response_text' for the body of the response. """ return 'HTTP ERROR - status code {status_code}\n----\n{response_text}\n----\n\n' def __init__(self, api_key): self.api_key = api_key self.token = None def connect(self): """ Connect to the Webarchive API, request and save a token. """ try: self.token = self._authenticate() except HTTPError as e: self._display_http_error(e) def _authenticate(self): r = requests.post(self.base_url.format('authentication'), data='''{{ "apikey": "{api_key}", "fingerprint": "string", "version": "{version}" }}'''.format(api_key=self.api_key, version=self.version), headers={ 'content-type': 'application/json', 'accept': 'application/ld+json' } ) if r.status_code == 201: return r.json()['t'] else: raise HTTPError(response=r) def _add_api_key_and_token(self, params_dict: dict): """ Add the saved api key and token to a given dictionary. :param params_dict: A dictionary that's probably used as a 'params' keyword parameter for calling requests.get(). :return: The same dictionary extended by 'apikey' and 't' keys. """ params_dict['apikey'] = self.api_key params_dict['t'] = self.token return params_dict def _display_http_error(self, e: HTTPError): print(self._error_template.format(status_code=e.response.status_code, response_text=e.response.text), file=sys.stderr) def _get(self, op, auto_connect=True, **kwargs, ): kwargs['params'] = self._add_api_key_and_token(kwargs.pop('params', {})) # kwargs = self._add_api_key_and_token(kwargs) r = requests.get(self.base_url.format(op), **kwargs) if r.ok: return r else: if r.status_code == 403: if auto_connect: self.connect() return self._get(op=op, auto_connect=False, **kwargs) else: print('Forbidden. Invalid Token or ApiKey transmitted', file=sys.stderr) return r elif r.status_code == 400: print('Bad request', file=sys.stderr) return r elif r.status_code == 410: print('The requested API Version (via X-API-VERSION Header) is not available', file=sys.stderr) return r raise HTTPError(response=r) def fulltext_search(self, query_string, from_=None, to_=None): """ Start a fulltext search query in the Webarchive. :param query_string: String to search for :param from_: Optional earliest date bound for the search in the format YYYYMM. :param to_: Optional latest date bound for the search in the format YYYYMM. :return: HTTP Response object """ params = {'q': query_string} if from_: params['from'] = from_ if to_: params['to'] = to_ try: response = self._get(op='/search/fulltext', params=params) return self.waitForResponse(response) except HTTPError as e: self._display_http_error(e) print('Query for "{}" not added'.format(query_string)) def wayback_search(self, query_string, from_=None, to_=None): """ Start a wayback search query in the Webarchive. :param query_string: String to search for :param from_: Optional earliest date bound for the search in the format YYYYMM. :param to_: Optional latest date bound for the search in the format YYYYMM. :return: HTTP Response object """ params = {'q': query_string} if from_: params['from'] = from_ if to_: params['to'] = to_ try: response = self._get(op='/search/wayback', params=params) return self.waitForResponse(response) except HTTPError as e: self._display_http_error(e) print('Error:'.format(query_string)) def waitForResponse(self, response): """ Polls until the server responds with a result :param response: String to search for :return: response """ if response.status_code == 400: return response while response.status_code != 200: time.sleep(0.5) response = self.status_query(response) return response def status_query(self, resp): """ this is the pollingrequest for the given typen of request :param response: String to search for :return: response """ requestid = resp.json()['requestid'] type = resp.json()['type'] if type == 1: r = self._get(op='/search/status/fulltext', params={'requestid': requestid}) elif type == 2: r = self._get(op='/search/status/wayback', params={'requestid': requestid}) return r def domain_name_search(self, query_string, page_=1, pagesize_=100): """ Start a domain name search in the Webarchive. :param query_string: String to search for :param page_: The page number parameter works with the page size parameter to control the offset of the records returned in the results. Default value is 1 :param pagesize_: The page size parameter works with the page number parameter to control the offset of the records returned in the results. It also controls how many results are returned with each request. Default value is 10 :return: result as json """ params = {'q': query_string} if page_: params['page'] = page_ if pagesize_: params['pagesize'] = pagesize_ try: response = self._get(op='/search/domainname', params=params) return self.waitForResponse(response) except HTTPError as e: self._display_http_error(e) print('Error:'.format(query_string)) def histogram_search(self, query_string, interval_=3, from_=None, to_=None): """ Start a domain name search in the Webarchive. :param query_string: String to search for :param page_: The page number parameter works with the page size parameter to control the offset of the records returned in the results. Default value is 1 :param pagesize_: The page size parameter works with the page number parameter to control the offset of the records returned in the results. It also controls how many results are returned with each request. Default value is 10 :return: result as json """ params = {'q': query_string} if interval_: params['interval'] = interval_ if from_: params['from'] = from_ if to_: params['to'] = to_ try: response = self._get(op='/search/fulltext/histogram', params=params) return self.waitForResponse(response) except HTTPError as e: self._display_http_error(e) print('Error:'.format(query_string)) def getSnapshotUrl(self, seed, capture, onlysvg): return self.api_path + 'snapshot?capture=' + capture + '&t=' + self.token + '&apikey=' + self.api_key + '&onlysvg=' + onlysvg + '&seed=' + seed; def savePage(self, url): self.connect() r = requests.post(self.base_url.format('savepage'), data='''{{ "apikey": "{api_key}", "t": "{token}", "url": "{url}" }}'''.format(api_key=self.api_key, token=self.token, url=url), headers={ 'content-type': 'application/json', 'accept': 'application/ld+json' } ) return r