Skip to content

GitLab

W
webarchive-api
Commit 69a2b582 authored by Stefan Karner's avatar Stefan Karner
Browse files
Options

Merge branch 'savepage_sample' into 'master' 

Save Page sample

See merge request !9
parents f011cdff 70722f7e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 757 additions and 723 deletions
sample1.ipynb
View file @ 69a2b582
......@@ -61,7 +61,25 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Selective Crawls\n\nFrau/Gender\nPolitikkollektion\nMedienkollektion\n\nEvent Crawls\n\n100 Jahre Republik 2018\nEU-Ratsvorsitz 2018\nBundespräsidenten-Wahl 2016\nFlüchtlingskrise 2015\nSong Contest 2015\nErster Weltkrieg 2014\nEU-Wahl 2014\nOlympia 2014\nOlympia 2010\nBundespräsidenten-Wahl 2010\nEU-Wahl 2009\n"
"Selective Crawls\n",
"\n",
"Frau/Gender\n",
"Politikkollektion\n",
"Medienkollektion\n",
"\n",
"Event Crawls\n",
"\n",
"100 Jahre Republik 2018\n",
"EU-Ratsvorsitz 2018\n",
"Bundespräsidenten-Wahl 2016\n",
"Flüchtlingskrise 2015\n",
"Song Contest 2015\n",
"Erster Weltkrieg 2014\n",
"EU-Wahl 2014\n",
"Olympia 2014\n",
"Olympia 2010\n",
"Bundespräsidenten-Wahl 2010\n",
"EU-Wahl 2009\n"
]
}
],
......@@ -90,7 +108,88 @@
"name": "stdout",
"output_type": "stream",
"text": [
"http://conchitawurst.com\nhttp://conchitawurst.com/news/\nhttp://de.wikipedia.org/wiki/Eurovision_Song_Contest_2015\nhttp://de.wikipedia.org/wiki/The_Makemakes\nhttp://debatte.orf.at/stories/1754300/\nhttp://derstandard.at/r2000004087668/Song-Contest-2015\nhttp://derstandard.at/r2000015639396/Eurovisions---Song-Contest-Blog\nhttp://diepresse.com/home/kultur/songcontest/index.do\nhttp://goodnight.at/magazin/kultur/516-60-eurovision-songcontest-in-wien\nhttp://kundendienst.orf.at/programm/fernsehen/orf1/songcontest2015.html\nhttp://kurier.at/kultur/song-contest\nhttp://medienportal.univie.ac.at/uniview/forschung/detailansicht/artikel/song-contest-zwischen-pop-und-politik/\nhttp://oe3.orf.at/stories/2679710\nhttp://orf.at/\nhttp://orf.at/stories/2254095/\nhttp://orf.at/stories/2254095/2254088/\nhttp://orf.at/stories/2254095/2254098/\nhttp://songcontest.orf.at/\nhttp://steiermark.orf.at/news/stories/2684488/\nhttp://tvthek.orf.at/topic/Eurovision-Song-Contest-2015/9137507\nhttp://tvthek.orf.at/topic/Oesterreich-beim-Song-Contest/9174892\nhttp://wien.orf.at/news/stories/2648040/\nhttp://wien.orf.at/news/stories/2648857/\nhttp://wien.orf.at/news/stories/2649515/\nhttp://wien.orf.at/news/stories/2661128/\nhttp://wien.orf.at/news/stories/2661370/\nhttp://wien.orf.at/news/stories/2661798/\nhttp://wien.orf.at/news/stories/2661804/\nhttp://wien.orf.at/news/stories/2661845/\nhttp://wien.orf.at/news/stories/2661855/\nhttp://wien.orf.at/news/stories/2661910/\nhttp://wien.orf.at/news/stories/2662093/\nhttp://wien.orf.at/news/stories/2662947/\nhttp://wien.orf.at/news/stories/2663106/\nhttp://wien.orf.at/news/stories/2664083/\nhttp://wien.orf.at/news/stories/2666541/\nhttp://wien.orf.at/news/stories/2666581/\nhttp://wien.orf.at/news/stories/2668462/\nhttp://wien.orf.at/news/stories/2670490/\nhttp://wien.orf.at/news/stories/2676170/\nhttp://wien.orf.at/news/stories/2678309/\nhttp://wien.orf.at/news/stories/2679672/\nhttp://wien.orf.at/news/stories/2680454/\nhttp://wien.orf.at/news/stories/2682245/\nhttp://wien.orf.at/news/stories/2683703/\nhttp://wien.orf.at/news/stories/2684423/\nhttp://wien.orf.at/news/stories/2687164/\nhttp://wien.orf.at/news/stories/2706977/\nhttp://wien.orf.at/news/tags/esc\nhttp://www.apa.at/Site/ESC2015/index.html\nhttp://www.austria-trend.at/blog/\nhttp://www.bmlfuw.gv.at/umwelt/nachhaltigkeit/green-events/projekte/kultur_und_musik/green_song_contest.html\nhttp://www.eurovision.tv/\nhttp://www.heute.at/song+contest./\nhttp://www.krone.at/song-contest\nhttp://www.news.at/a/song-contest-telegramm-2015\nhttp://www.oe24.at/kultur/song-contest\nhttp://www.oeticket.com/de/kuenstler/eurovision-song-contest-1020/profile.html\nhttp://www.picturedesk.com/ESC2015\nhttp://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015\nhttp://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-family-show-finale.html\nhttp://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-jury-show-finale.html\nhttp://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-public-viewing-in-wien.html\nhttp://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-semifinale-1-2.html\nhttp://www.songcontest.at/\nhttp://www.stadthalle.com/de/schauen/events/75/Eurovision-Song-Contest\nhttp://www.vienna.at/specials/eurovision-song-contest\nhttp://www.wien-event.at/events/song-contest/\nhttp://www.wien-ticket.at/de/service/Eurovision-Song-Contest-Wiener-Stadthalle-Wien\nhttp://www.wien.info/de/musik-buehne/eurovision-song-contest\nhttp://www.wienerlinien.at/eportal2/ep/tab.do/pageTypeId/74690\nhttp://www.wienerzeitung.at/dossiers/eurovision_song_contest_2015/\nhttps://instagram.com/esc_buildingbridges/\nhttps://instagram.com/themakemakes/\nhttps://twitter.com/ESC2015ORF\nhttps://twitter.com/Eurovision\nhttps://twitter.com/hashtag/eurovision?src=hash\nhttps://www.facebook.com/ESCBuildingBridges\nhttps://www.facebook.com/EurovisionSongContest\nhttps://www.facebook.com/themakemakes\nhttps://www.stadthalle.com/de/schauen/events/75/Eurovision-Song-Contest\nhttps://www.wien.gv.at/kultur/esc/\n"
"http://conchitawurst.com\n",
"http://conchitawurst.com/news/\n",
"http://de.wikipedia.org/wiki/Eurovision_Song_Contest_2015\n",
"http://de.wikipedia.org/wiki/The_Makemakes\n",
"http://debatte.orf.at/stories/1754300/\n",
"http://derstandard.at/r2000004087668/Song-Contest-2015\n",
"http://derstandard.at/r2000015639396/Eurovisions---Song-Contest-Blog\n",
"http://diepresse.com/home/kultur/songcontest/index.do\n",
"http://goodnight.at/magazin/kultur/516-60-eurovision-songcontest-in-wien\n",
"http://kundendienst.orf.at/programm/fernsehen/orf1/songcontest2015.html\n",
"http://kurier.at/kultur/song-contest\n",
"http://medienportal.univie.ac.at/uniview/forschung/detailansicht/artikel/song-contest-zwischen-pop-und-politik/\n",
"http://oe3.orf.at/stories/2679710\n",
"http://orf.at/\n",
"http://orf.at/stories/2254095/\n",
"http://orf.at/stories/2254095/2254088/\n",
"http://orf.at/stories/2254095/2254098/\n",
"http://songcontest.orf.at/\n",
"http://steiermark.orf.at/news/stories/2684488/\n",
"http://tvthek.orf.at/topic/Eurovision-Song-Contest-2015/9137507\n",
"http://tvthek.orf.at/topic/Oesterreich-beim-Song-Contest/9174892\n",
"http://wien.orf.at/news/stories/2648040/\n",
"http://wien.orf.at/news/stories/2648857/\n",
"http://wien.orf.at/news/stories/2649515/\n",
"http://wien.orf.at/news/stories/2661128/\n",
"http://wien.orf.at/news/stories/2661370/\n",
"http://wien.orf.at/news/stories/2661798/\n",
"http://wien.orf.at/news/stories/2661804/\n",
"http://wien.orf.at/news/stories/2661845/\n",
"http://wien.orf.at/news/stories/2661855/\n",
"http://wien.orf.at/news/stories/2661910/\n",
"http://wien.orf.at/news/stories/2662093/\n",
"http://wien.orf.at/news/stories/2662947/\n",
"http://wien.orf.at/news/stories/2663106/\n",
"http://wien.orf.at/news/stories/2664083/\n",
"http://wien.orf.at/news/stories/2666541/\n",
"http://wien.orf.at/news/stories/2666581/\n",
"http://wien.orf.at/news/stories/2668462/\n",
"http://wien.orf.at/news/stories/2670490/\n",
"http://wien.orf.at/news/stories/2676170/\n",
"http://wien.orf.at/news/stories/2678309/\n",
"http://wien.orf.at/news/stories/2679672/\n",
"http://wien.orf.at/news/stories/2680454/\n",
"http://wien.orf.at/news/stories/2682245/\n",
"http://wien.orf.at/news/stories/2683703/\n",
"http://wien.orf.at/news/stories/2684423/\n",
"http://wien.orf.at/news/stories/2687164/\n",
"http://wien.orf.at/news/stories/2706977/\n",
"http://wien.orf.at/news/tags/esc\n",
"http://www.apa.at/Site/ESC2015/index.html\n",
"http://www.austria-trend.at/blog/\n",
"http://www.bmlfuw.gv.at/umwelt/nachhaltigkeit/green-events/projekte/kultur_und_musik/green_song_contest.html\n",
"http://www.eurovision.tv/\n",
"http://www.heute.at/song+contest./\n",
"http://www.krone.at/song-contest\n",
"http://www.news.at/a/song-contest-telegramm-2015\n",
"http://www.oe24.at/kultur/song-contest\n",
"http://www.oeticket.com/de/kuenstler/eurovision-song-contest-1020/profile.html\n",
"http://www.picturedesk.com/ESC2015\n",
"http://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015\n",
"http://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-family-show-finale.html\n",
"http://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-jury-show-finale.html\n",
"http://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-public-viewing-in-wien.html\n",
"http://www.railtours.at/sonderangebote/oesterreich/wien/eurovision-song-contest-2015-semifinale-1-2.html\n",
"http://www.songcontest.at/\n",
"http://www.stadthalle.com/de/schauen/events/75/Eurovision-Song-Contest\n",
"http://www.vienna.at/specials/eurovision-song-contest\n",
"http://www.wien-event.at/events/song-contest/\n",
"http://www.wien-ticket.at/de/service/Eurovision-Song-Contest-Wiener-Stadthalle-Wien\n",
"http://www.wien.info/de/musik-buehne/eurovision-song-contest\n",
"http://www.wienerlinien.at/eportal2/ep/tab.do/pageTypeId/74690\n",
"http://www.wienerzeitung.at/dossiers/eurovision_song_contest_2015/\n",
"https://instagram.com/esc_buildingbridges/\n",
"https://instagram.com/themakemakes/\n",
"https://twitter.com/ESC2015ORF\n",
"https://twitter.com/Eurovision\n",
"https://twitter.com/hashtag/eurovision?src=hash\n",
"https://www.facebook.com/ESCBuildingBridges\n",
"https://www.facebook.com/EurovisionSongContest\n",
"https://www.facebook.com/themakemakes\n",
"https://www.stadthalle.com/de/schauen/events/75/Eurovision-Song-Contest\n",
"https://www.wien.gv.at/kultur/esc/\n"
]
}
],
......@@ -105,23 +204,23 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 0
"nbformat_minor": 1
}
sample2.ipynb
View file @ 69a2b582
......@@ -9,16 +9,13 @@
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import datetime\n",
"from webarchiv import WebarchivSession\n",
"\n",
"apikey = 'Zz2tQls7fuaocX2pjrfc2npojqbGwXL2'\n",
"apikey = '2pm8i0hnmpcTK4Oj4CUeBoZd7vywrm4c'\n",
"w = WebarchivSession(apikey)"
]
},
......@@ -31,7 +28,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
......@@ -48,7 +45,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
......@@ -73,14 +70,14 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2473\n"
"2483\n"
]
}
],
......@@ -97,10 +94,8 @@
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
......@@ -111,7 +106,7 @@
}
],
"source": [
"capturedate = datetime.datetime.fromtimestamp(response.json()['hits'][0]['c']).strftime('%Y%m%d%H%M%S')\n",
"capturedate = response.json()['hits'][0]['c']\n",
"\n",
"captureurl = 'https://webarchiv.onb.ac.at/web/' + capturedate + '/' + url\n",
"\n",
......@@ -128,23 +123,23 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 0
"nbformat_minor": 1
}
sample3.ipynb
View file @ 69a2b582
......@@ -14,7 +14,7 @@
"outputs": [],
"source": [
"from webarchiv import WebarchivSession\n",
"apikey = 'Zz2tQls7fuaocX2pjrfc2npojqbGwXL2'\n",
"apikey = '2pm8i0hnmpcTK4Oj4CUeBoZd7vywrm4c'\n",
"w = WebarchivSession(apikey)"
]
},
......@@ -86,23 +86,23 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 0
"nbformat_minor": 1
}
sample4.ipynb
View file @ 69a2b582
This diff is collapsed.
sample5.ipynb
View file @ 69a2b582
This diff is collapsed.
sample6.ipynb
View file @ 69a2b582
......@@ -75,7 +75,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"['2009-01-01', '2010-01-01', '2011-01-01', '2012-01-01', '2013-01-01', '2014-01-01', '2015-01-01', '2016-01-01', '2017-01-01']\n[4, 4, 23, 28, 65, 859, 799, 77, 21]\n"
"['2009-01-01', '2010-01-01', '2011-01-01', '2012-01-01', '2013-01-01', '2014-01-01', '2015-01-01', '2016-01-01', '2017-01-01']\n",
"[4, 4, 23, 28, 65, 859, 799, 77, 21]\n"
]
}
],
......@@ -100,23 +101,23 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 0
"nbformat_minor": 1
}
sample7.ipynb
View file @ 69a2b582
This source diff could not be displayed because it is too large. You can view the blob instead.
sample8.ipynb 0 → 100644
View file @ 69a2b582
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import json"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Create a WebarchivSession Object with convenience methods for easy access with your API-Key "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from webarchiv import WebarchivSession\n",
"\n",
"apikey = '2pm8i0hnmpcTK4Oj4CUeBoZd7vywrm4c'\n",
"w = WebarchivSession(apikey)\n"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"request archiving a Webpage"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'nomination_id': 247, 'seed': 'http://www.onb.ac.at', 'nominationtype': 5}\n"
]
}
],
"source": [
"response = w.savePage(\"http://www.onb.ac.at\")\n",
"\n",
"if response.status_code == 201:\n",
" print(response.json())\n",
"else:\n",
" print(\"Error \", response.status_code)\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
webarchiv.py
View file @ 69a2b582
......@@ -248,18 +248,33 @@ class WebarchivSession:
self._display_http_error(e)
print('Error:'.format(query_string))
def getSnapshotUrl(self, id):
return self.api_path + 'snapshot/' + id + "?t=" + self.token + "&apikey=" + self.api_key;
def getSnapshotUrl(self, seed, capture, onlysvg):
return self.api_path + 'snapshot?capture=' + capture + '&t=' + self.token + '&apikey=' + self.api_key + '&onlysvg=' + onlysvg + '&seed=' + seed;
def savePage(self, url):
self.connect()
r = requests.post(self.base_url.format('savepage'),
data='''{{
"apikey": "{api_key}",
"t": "{token}",
"url": "{url}"
}}'''.format(api_key=self.api_key, token=self.token, url=url),
headers={
'content-type': 'application/json',
'accept': 'application/ld+json'
}
)
return r;
if __name__ == '__main__':
# noinspection SpellCheckingInspection
w = WebarchivSession('2pm8i0hnmpcTK4Oj4CUeBoZd7vywrm4c')
# response = w.wayback_search("http://www.onb.ac.at")
# response = w.wayback_search("http://frauenhetz.jetzt")
response = w.histogram_search("Nationalbibliothek Prunksaal Schwarzenegger")
# response = w.wayback_search("x")
#response = w.histogram_search("Nationalbibliothek Prunksaal Schwarzenegger")
response = w.savePage("http://www.onb.ac.at")
if response.status_code == 200:
if response.status_code == 201:
print(response.json())
else:
print("Error ", response.status_code)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment