make service argument optional
This commit is contained in:
Nick Sweeting
parent
612956e8c9
commit
35ee1f90cd
12
README.md
12
README.md
|
|
@ -4,11 +4,11 @@
|
|||
|
||||
(Your own personal Way-Back Machine) [DEMO: sweeting.me/pocket](https://home.sweeting.me/pocket)
|
||||
|
||||
Save an archived copy of all websites you star.
|
||||
Outputs browsable html archives of each site, a PDF, a screenshot, and a link to a copy on archive.org, all indexed in a nice html file.
|
||||
Save an archived copy of all websites you star.
|
||||
Outputs browsable html archives of each site, a PDF, a screenshot, and a link to a copy on archive.org, all indexed in a nice html file.
|
||||
|
||||
|
||||
(Powered by the [headless](https://developers.google.com/web/updates/2017/04/headless-chrome) Google Chrome and good 'ol `wget`.)
|
||||
(Powered by the [headless](https://developers.google.com/web/updates/2017/04/headless-chrome) Google Chrome and good 'ol `wget`.)
|
||||
NEW: Also submits each link to save on archive.org!
|
||||
|
||||
|
||||
|
|
@ -16,7 +16,7 @@ NEW: Also submits each link to save on archive.org!
|
|||
## Quickstart
|
||||
|
||||
```bash
|
||||
./archive.py link_export.html [pocket|pinboard|bookmarks]
|
||||
./archive.py bookmark_export.html
|
||||
```
|
||||
|
||||
`archive.py` is a script that takes a [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), or [Browser Bookmark](https://support.google.com/chrome/answer/96816?hl=en) html export file, and turns it into a browsable archive that you can store locally or host online.
|
||||
|
|
@ -43,7 +43,9 @@ google-chrome --version && which wget && which python3 && echo "[√] All depend
|
|||
1. Get your HTML export file from [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Chrome Bookmarks](https://support.google.com/chrome/answer/96816?hl=en), [Firefox Bookmarks](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), or [Safari Bookmarks](http://i.imgur.com/AtcvUZA.png)
|
||||
2. Clone this repo `git clone https://github.com/pirate/bookmark-archiver`
|
||||
3. `cd bookmark-archiver/`
|
||||
4. `./archive.py ~/Downloads/exported_file.html [bookmarks|pocket|pinboard]`
|
||||
4. `./archive.py ~/Downloads/bookmarks_export.html`
|
||||
|
||||
You may optionally specify a third argument to `archive.py export.html [pocket|pinboard|bookmarks]` to enforce the use of a specific link parser.
|
||||
|
||||
It produces a folder like `pocket/` containing an `index.html`, and archived copies of all the sites,
|
||||
organized by starred timestamp. For each sites it saves:
|
||||
|
|
|
|||
60
archive.py
60
archive.py
|
|
@ -35,7 +35,7 @@ def check_dependencies():
|
|||
if FETCH_PDF or FETCH_SCREENSHOT:
|
||||
if run(['which', CHROME_BINARY]).returncode:
|
||||
print('[X] Missing dependency: {}'.format(CHROME_BINARY))
|
||||
print(' See https://github.com/pirate/pocket-archive-stream for help.')
|
||||
print(' See https://github.com/pirate/bookmark-archiver for help.')
|
||||
raise SystemExit(1)
|
||||
|
||||
# parse chrome --version e.g. Google Chrome 61.0.3114.0 canary / Chromium 59.0.3029.110 built on Ubuntu, running on Ubuntu 16.04
|
||||
|
|
@ -43,19 +43,19 @@ def check_dependencies():
|
|||
version = result.stdout.decode('utf-8').replace('Google Chrome ', '').replace('Chromium ', '').split(' ', 1)[0].split('.', 1)[0] # TODO: regex might be better
|
||||
if int(version) < 59:
|
||||
print('[X] Chrome version must be 59 or greater for headless PDF and screenshot saving')
|
||||
print(' See https://github.com/pirate/pocket-archive-stream for help.')
|
||||
print(' See https://github.com/pirate/bookmark-archiver for help.')
|
||||
raise SystemExit(1)
|
||||
|
||||
if FETCH_WGET:
|
||||
if run(['which', 'wget']).returncode:
|
||||
print('[X] Missing dependency: wget')
|
||||
print(' See https://github.com/pirate/pocket-archive-stream for help.')
|
||||
print(' See https://github.com/pirate/bookmark-archiver for help.')
|
||||
raise SystemExit(1)
|
||||
|
||||
if FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG:
|
||||
if run(['which', 'curl']).returncode:
|
||||
print('[X] Missing dependency: curl')
|
||||
print(' See https://github.com/pirate/pocket-archive-stream for help.')
|
||||
print(' See https://github.com/pirate/bookmark-archiver for help.')
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
|
|
@ -72,9 +72,10 @@ def get_link_type(link):
|
|||
return 'youtube'
|
||||
return None
|
||||
|
||||
def parse_pocket_export(html):
|
||||
def parse_pocket_export(html_file):
|
||||
html_file.seek(0)
|
||||
pattern = re.compile("^\\s*<li><a href=\"(.+)\" time_added=\"(\\d+)\" tags=\"(.*)\">(.+)</a></li>", re.UNICODE) # see sample input in ./example_ril_export.html
|
||||
for line in html:
|
||||
for line in html_file:
|
||||
match = pattern.search(line)
|
||||
if match:
|
||||
fixed_url = match.group(1).replace('http://www.readability.com/read?url=', '') # remove old readability prefixes to get original url
|
||||
|
|
@ -91,8 +92,11 @@ def parse_pocket_export(html):
|
|||
info['type'] = get_link_type(info)
|
||||
yield info
|
||||
|
||||
def parse_pinboard_export(html):
|
||||
json_content = json.load(html)
|
||||
def parse_json_export(json_file):
|
||||
"""pinboard exports are json"""
|
||||
|
||||
json_file.seek(0)
|
||||
json_content = json.load(json_file)
|
||||
for line in json_content:
|
||||
if line:
|
||||
erg = line
|
||||
|
|
@ -108,9 +112,10 @@ def parse_pinboard_export(html):
|
|||
info['type'] = get_link_type(info)
|
||||
yield info
|
||||
|
||||
def parse_bookmarks_export(html):
|
||||
def parse_bookmarks_export(html_file):
|
||||
html_file.seek(0)
|
||||
pattern = re.compile("<a href=\"(.+?)\" add_date=\"(\\d+)\"[^>]*>(.+)</a>", re.UNICODE | re.IGNORECASE)
|
||||
for line in html:
|
||||
for line in html_file:
|
||||
match = pattern.search(line)
|
||||
if match:
|
||||
url = match.group(1)
|
||||
|
|
@ -336,25 +341,31 @@ def dump_website(link, service, overwrite=False):
|
|||
if FETCH_FAVICON:
|
||||
fetch_favicon(out_dir, link, overwrite=overwrite)
|
||||
|
||||
def create_archive(export_file, service, resume=None):
|
||||
print('[+] [{}] Starting {} archive from {}'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), service, export_file))
|
||||
|
||||
if not os.path.exists(service):
|
||||
os.makedirs(service)
|
||||
|
||||
if not os.path.exists(''.join((service, '/archive'))):
|
||||
os.makedirs(''.join((service, '/archive')))
|
||||
|
||||
def create_archive(export_file, service=None, resume=None):
|
||||
with open(export_file, 'r', encoding='utf-8') as f:
|
||||
print('[+] [{}] Starting archive from {} export file.'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), export_file))
|
||||
|
||||
if service == "pocket":
|
||||
links = parse_pocket_export(f)
|
||||
elif service == "pinboard":
|
||||
links = parse_pinboard_export(f)
|
||||
links = parse_json_export(f)
|
||||
elif service == "bookmarks":
|
||||
links = parse_bookmarks_export(f)
|
||||
else:
|
||||
# try all parsers until one works
|
||||
try:
|
||||
links = list(parse_json_export(f))
|
||||
service = 'pinboard'
|
||||
except Exception:
|
||||
links = list(parse_pocket_export(f))
|
||||
if links:
|
||||
service = 'pocket'
|
||||
else:
|
||||
links = list(parse_bookmarks_export(f))
|
||||
service = 'bookmarks'
|
||||
|
||||
links = valid_links(links) # remove chrome://, about:, mailto: etc.
|
||||
links = uniquefied_links(links) # fix duplicate timestamps, returns sorted list
|
||||
links = uniquefied_links(links) # fix duplicate timestamps, returns sorted list
|
||||
if resume:
|
||||
try:
|
||||
links = [link for link in links if float(link['timestamp']) >= float(resume)]
|
||||
|
|
@ -365,6 +376,11 @@ def create_archive(export_file, service, resume=None):
|
|||
print('[X] No links found in {}, is it a {} export file?'.format(export_file, service))
|
||||
raise SystemExit(1)
|
||||
|
||||
if not os.path.exists(service):
|
||||
os.makedirs(service)
|
||||
|
||||
if not os.path.exists(''.join((service, '/archive'))):
|
||||
os.makedirs(''.join((service, '/archive')))
|
||||
|
||||
dump_index(links, service)
|
||||
|
||||
|
|
@ -384,7 +400,7 @@ def create_archive(export_file, service, resume=None):
|
|||
if __name__ == '__main__':
|
||||
argc = len(sys.argv)
|
||||
export_file = sys.argv[1] if argc > 1 else "ril_export.html" # path to export file
|
||||
export_type = sys.argv[2] if argc > 2 else "pocket" # select export_type for file format select
|
||||
export_type = sys.argv[2] if argc > 2 else None # select export_type for file format select
|
||||
resume_from = sys.argv[3] if argc > 3 else None # timestamp to resume dowloading from
|
||||
|
||||
create_archive(export_file, export_type, resume=resume_from)
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@
|
|||
<img src="https://nicksweeting.com/images/archive.png" height="36px">
|
||||
Archived Sites <img src="https://getpocket.com/favicon.ico" height="36px"> <br/>
|
||||
<small>
|
||||
Via: <a href="https://getpocket.com/export">getpocket.com/export</a> + <a href="https://github.com/pirate/pocket-archive-stream">archive_pocket.py</a>
|
||||
Via: <a href="https://getpocket.com/export">getpocket.com/export</a> + <a href="https://github.com/pirate/bookmark-archiver">archive_pocket.py</a>
|
||||
| <a href="https://getpocket.com/users/USERNAME/feed/all">RSS Feed</a>
|
||||
</small>
|
||||
</h1>
|
||||
|
|
@ -76,7 +76,7 @@
|
|||
<table style="width:100%;height: 90%; overflow-y: scroll;table-layout: fixed">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width: 120px;"><img src="https://getpocket.com/favicon.ico" height="12px"> Pocketed Date</th>
|
||||
<th style="width: 120px;"><img src="https://getpocket.com/favicon.ico" height="12px"> Date Starred</th>
|
||||
<th style="width: 45vw;">Saved Article</th>
|
||||
<th style="width: 50px">Files</th>
|
||||
<th style="width: 50px">PDF</th>
|
||||
|
|
|
|||
Loading…
Reference in a new issue