"""This module provides functions for scraping links from the
`Esperanto Newspaper page
<https://www.onb.ac.at/sammlungen/sammlung-fuer-plansprachen/digitale-medien/zeitungsausschnitte-1898-1915/>`_
and downloading images contained in those collections.

"""

import requests
import json
import argparse
import os
import logging
from bs4 import BeautifulSoup
from urllib.parse import urlparse, parse_qs
from pathlib import Path
from dotenv import load_dotenv

load_dotenv('../.env')
ASTOR_API_KEY = os.getenv('ASTOR_API_KEY')

URL = 'https://www.onb.ac.at/bibliothek/sammlungen/plansprachen/digitale-medien/zeitungsausschnitte-1898-1915'
OPEN_ACCESS_URL = 'https://digital.onb.ac.at/rep/access/open/{}'
API_URL = 'https://astor.onb.ac.at/access/dip/{}?apikey=' + ASTOR_API_KEY


def get_doc_json(link) -> object:
    """Parse a link from Esperanto page and return JSON from API.

    :param link: Link from Esperanto page (either 'dtl' or 'rep')
    :returns: JSON from API at astor.onb.ac.at
    :rtype: object

    Transform example link https://data.onb.ac.at/rep/131ABBA7 into
    this API link (i.e. map 131ABBA7 to DOD_50996):
    f'https://astor.onb.ac.at/access/dip/DOD_50996?apikey={ASTOR_API_KEY}'
    """
    if 'dtl' in link:
        identifier = link.rsplit('/', 1)[-1]
        url = API_URL.format('DTL_{}'.format(identifier))
        return json.loads(requests.get(url).content)
    elif 'rep' in link:
        identifier = link.rsplit('/', 1)[-1]
        open_access_link = OPEN_ACCESS_URL.format(identifier)
        # Open link to get redirected URL
        open_access_link = requests.get(open_access_link).url
        doc_id = parse_qs(urlparse(open_access_link).query)['doc'][0]
        url = API_URL.format(doc_id)
        return json.loads(requests.get(url).content)
    elif 'DTL' in link:
        identifier = link.rsplit('/', 1)[-1]
        open_access_link = OPEN_ACCESS_URL.format(identifier)
        # Open link to get redirected URL
        open_access_link = requests.get(open_access_link).url
        doc_id = parse_qs(urlparse(open_access_link).query)['doc'][0]
        url = API_URL.format(doc_id)
        return json.loads(requests.get(url).content)


def get_area_links(areas):
    """Get all links from a specific area/country.

    :param area: A list of areas to get links for.
    :type area: List[str]
    :returns: A list of links to the set of docs.
    :rtype: List[str]
    """
    page = requests.get(URL)
    soup = BeautifulSoup(page.content, 'html.parser')
    all_links = []
    for area in areas:
        # Search for <p> after <h5> with text Frankreich
        target = soup.find('h4', string=area).find_next_sibling()
        # Find all hrefs in <p>
        all_links += [
            link['href']
            for link in target.find_all('a', href=True, limit=args.count)
        ]

    return all_links


def download_images(doc_json, output_path, limit, quality):
    """Iterate over files in DOC_JSON and download at most LIMIT
    images to OUTPUT_PATH.

    :param doc_json: JSON result from API.
    :param output_path: Dir to save images to.
    :param limit: Number of images to download.
    :returns: None
    """
    session = requests.Session()
    count = 0
    # Loop over each collection
    for doc in doc_json:
        # Loop over each file in collection
        for _, value in doc['files'].items():
            if count >= limit:
                break
            # Set path of jp2 image
            newpath = os.path.join(output_path, value['aipId'])
            # Make dirs necessary for saving
            if not os.path.exists(newpath):
                os.makedirs(newpath)

            count += 1
            # Change fileext from .jp2 to .jpg if lossy desired
            if quality == 'lossy':
                filename = os.path.splitext(
                    os.path.join(newpath, value['filename']))[0] + '.jpg'
            else:
                filename = os.path.join(newpath, value['filename'])

            if os.path.exists(filename):
                logging.info('{} {}/{} already exists. Skipping...'.format(
                    filename, count, limit))
                continue

            # imageUrl contains lossy jpegs, fileUrl lossless jp2s
            if quality == 'lossy':
                resp = session.get(value['imageUrl'])
            else:
                resp = session.get(value['fileUrl'])

            if resp.status_code != 200:
                logging.warn('Image download failed with code {}'.format(
                    resp.status_code))
            with open(filename, 'wb') as f:
                f.write(resp.content)
                logging.info('Downloaded image {}/{}: {}'.format(
                    count, limit, filename))


def dir_path(string):
    """Check if STRING is a valid dir path.

    :param string: Dir path as string.
    :returns: Either valid string or exception.
    :raises: NotADirectoryError
    """
    if os.path.isdir(string) or not os.path.exists(string):
        return string
    else:
        raise NotADirectoryError(string)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        prog='scrape',
        description='Scrape Esperanto newspaper images from ÖNB website.')

    parser.add_argument('area',
                        nargs='+',
                        choices=[
                            'Afrika und Asien', 'Amerika', 'Belgien',
                            'Brasilien', 'Dänemark', 'Deutsches Reich',
                            'Divers', 'Europa', 'Frankreich', 'Großbritannien',
                            'Island', 'Italien', 'Kanada', 'Niederlande',
                            'Österreich-Ungarn', 'Portugal', 'Rumänien',
                            'Russland', 'Schweden', 'Schweiz', 'Spanien'
                        ],
                        default='Frankreich',
                        help='From which area/country to download')
    parser.add_argument('-c',
                        '--count',
                        default='50',
                        type=int,
                        help='How many links to open')
    parser.add_argument('-o',
                        '--output',
                        type=dir_path,
                        default=Path('.', 'images'),
                        help='Path of directory to save images to')
    parser.add_argument('-l',
                        '--limit',
                        default=10000,
                        type=int,
                        help='How many images to download')
    parser.add_argument(
        '-q',
        '--quality',
        choices=['lossy', 'lossless'],
        default='lossy',
        help='Whether to download lossy jpegs or lossless jp2s')

    parser.add_argument(
        '--log',
        choices=['debug', 'info', 'warning', 'error', 'critical'],
        help='log level',
        default='warning')

    args = parser.parse_args()

    # Set logging level and message format
    numeric_level = getattr(logging, args.log.upper(), None)
    logging.basicConfig(format='%(levelname)s::%(asctime)s::%(message)s',
                        datefmt='%Y-%m-%dT%H:%M:%S',
                        level=numeric_level)

    # Download all links for areas
    links = get_area_links(args.area)

    # Query API to get JSON of collections
    json_infos = [get_doc_json(link) for link in links]

    download_images(json_infos, args.output, args.limit, args.quality)
