2017-07-04 22:38:07 +12:00
|
|
|
import os
|
2017-10-19 11:38:17 +13:00
|
|
|
import json
|
|
|
|
|
2017-07-04 22:38:07 +12:00
|
|
|
from datetime import datetime
|
2017-07-05 21:59:09 +12:00
|
|
|
from string import Template
|
2019-03-27 12:21:34 +13:00
|
|
|
from typing import List, Tuple, Iterator, Optional
|
2019-03-28 03:31:22 +13:00
|
|
|
from dataclasses import fields
|
2019-03-26 20:20:41 +13:00
|
|
|
|
2019-03-27 12:21:34 +13:00
|
|
|
from schema import Link, ArchiveIndex, ArchiveResult
|
2017-07-06 09:33:51 +12:00
|
|
|
from config import (
|
2019-02-22 11:45:28 +13:00
|
|
|
OUTPUT_DIR,
|
2018-06-11 13:12:55 +12:00
|
|
|
TEMPLATES_DIR,
|
2017-10-19 11:38:17 +13:00
|
|
|
GIT_SHA,
|
2018-04-18 01:13:38 +12:00
|
|
|
FOOTER_INFO,
|
2019-03-28 03:31:22 +13:00
|
|
|
TIMEOUT,
|
2017-07-06 09:33:51 +12:00
|
|
|
)
|
2017-10-23 22:58:41 +13:00
|
|
|
from util import (
|
2019-03-27 15:26:21 +13:00
|
|
|
merge_links,
|
2017-10-23 22:58:41 +13:00
|
|
|
chmod_file,
|
2019-03-21 22:35:41 +13:00
|
|
|
urlencode,
|
2017-10-23 22:58:41 +13:00
|
|
|
derived_link_info,
|
2019-03-20 11:09:46 +13:00
|
|
|
wget_output_path,
|
2019-03-26 22:33:34 +13:00
|
|
|
ExtendedEncoder,
|
2019-03-27 15:26:21 +13:00
|
|
|
enforce_types,
|
2019-03-28 03:31:22 +13:00
|
|
|
TimedProgress,
|
2019-03-28 04:39:51 +13:00
|
|
|
copy_and_overwrite,
|
2017-10-23 22:58:41 +13:00
|
|
|
)
|
2019-03-21 18:28:12 +13:00
|
|
|
from parse import parse_links
|
|
|
|
from links import validate_links
|
|
|
|
from logs import (
|
2019-03-23 08:09:39 +13:00
|
|
|
log_indexing_process_started,
|
2019-03-21 18:28:12 +13:00
|
|
|
log_indexing_started,
|
|
|
|
log_indexing_finished,
|
|
|
|
log_parsing_started,
|
|
|
|
log_parsing_finished,
|
|
|
|
)
|
2017-10-19 11:38:17 +13:00
|
|
|
|
2019-03-20 11:09:46 +13:00
|
|
|
TITLE_LOADING_MSG = 'Not yet archived...'
|
|
|
|
|
2017-10-19 11:38:17 +13:00
|
|
|
|
|
|
|
### Homepage index for all the links
|
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None:
|
2017-10-19 11:38:17 +13:00
|
|
|
"""create index.html file for a given list of links"""
|
|
|
|
|
2019-03-23 08:09:39 +13:00
|
|
|
log_indexing_process_started()
|
2019-02-22 11:45:28 +13:00
|
|
|
|
2019-03-23 08:09:39 +13:00
|
|
|
log_indexing_started(out_dir, 'index.json')
|
2019-03-28 03:31:22 +13:00
|
|
|
timer = TimedProgress(TIMEOUT * 2, prefix=' ')
|
2019-02-07 19:06:21 +13:00
|
|
|
write_json_links_index(out_dir, links)
|
2019-03-28 03:31:22 +13:00
|
|
|
timer.end()
|
2019-03-21 18:28:12 +13:00
|
|
|
log_indexing_finished(out_dir, 'index.json')
|
2019-02-07 19:06:21 +13:00
|
|
|
|
2019-03-23 08:09:39 +13:00
|
|
|
log_indexing_started(out_dir, 'index.html')
|
2019-03-28 03:31:22 +13:00
|
|
|
timer = TimedProgress(TIMEOUT * 2, prefix=' ')
|
2019-03-20 11:08:42 +13:00
|
|
|
write_html_links_index(out_dir, links, finished=finished)
|
2019-03-28 03:31:22 +13:00
|
|
|
timer.end()
|
2019-03-21 18:28:12 +13:00
|
|
|
log_indexing_finished(out_dir, 'index.html')
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
|
|
|
|
@enforce_types
|
2019-03-27 16:25:07 +13:00
|
|
|
def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) -> Tuple[List[Link], List[Link]]:
|
2019-03-21 18:28:12 +13:00
|
|
|
"""parse and load existing index with any new links from import_path merged in"""
|
|
|
|
|
2019-03-26 20:20:41 +13:00
|
|
|
existing_links: List[Link] = []
|
2019-03-21 18:28:12 +13:00
|
|
|
if out_dir:
|
2019-03-27 12:21:34 +13:00
|
|
|
existing_links = list(parse_json_links_index(out_dir))
|
2019-03-21 18:28:12 +13:00
|
|
|
|
2019-03-26 20:20:41 +13:00
|
|
|
new_links: List[Link] = []
|
2019-03-21 18:28:12 +13:00
|
|
|
if import_path:
|
|
|
|
# parse and validate the import file
|
|
|
|
log_parsing_started(import_path)
|
|
|
|
raw_links, parser_name = parse_links(import_path)
|
2019-03-27 12:21:34 +13:00
|
|
|
new_links = list(validate_links(raw_links))
|
2019-03-21 18:28:12 +13:00
|
|
|
|
|
|
|
# merge existing links in out_dir and new links
|
2019-03-27 12:21:34 +13:00
|
|
|
all_links = list(validate_links(existing_links + new_links))
|
2019-03-21 18:28:12 +13:00
|
|
|
|
|
|
|
if import_path and parser_name:
|
2019-03-28 08:16:53 +13:00
|
|
|
num_parsed = len(raw_links)
|
|
|
|
num_new_links = len(all_links) - len(existing_links)
|
|
|
|
log_parsing_finished(num_parsed, num_new_links, parser_name)
|
2019-03-21 18:28:12 +13:00
|
|
|
|
|
|
|
return all_links, new_links
|
2017-07-04 22:38:07 +12:00
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def write_json_links_index(out_dir: str, links: List[Link]) -> None:
|
2017-10-19 11:38:17 +13:00
|
|
|
"""write the json link index to a given path"""
|
|
|
|
|
2019-03-27 12:21:34 +13:00
|
|
|
assert isinstance(links, List), 'Links must be a list, not a generator.'
|
|
|
|
assert isinstance(links[0].history, dict)
|
|
|
|
assert isinstance(links[0].sources, list)
|
|
|
|
|
|
|
|
if links[0].history.get('title'):
|
|
|
|
assert isinstance(links[0].history['title'][0], ArchiveResult)
|
|
|
|
|
|
|
|
if links[0].sources:
|
|
|
|
assert isinstance(links[0].sources[0], str)
|
2019-02-22 11:45:28 +13:00
|
|
|
|
2017-10-19 11:38:17 +13:00
|
|
|
path = os.path.join(out_dir, 'index.json')
|
|
|
|
|
2019-03-28 08:32:39 +13:00
|
|
|
index_json = {
|
|
|
|
'info': 'ArchiveBox Index',
|
|
|
|
'source': 'https://github.com/pirate/ArchiveBox',
|
|
|
|
'docs': 'https://github.com/pirate/ArchiveBox/wiki',
|
|
|
|
'version': VERSION,
|
|
|
|
'num_links': len(links),
|
|
|
|
'updated': datetime.now(),
|
|
|
|
'links': links,
|
|
|
|
}
|
2017-10-19 11:38:17 +13:00
|
|
|
|
|
|
|
with open(path, 'w', encoding='utf-8') as f:
|
2019-03-28 08:32:39 +13:00
|
|
|
json.dump(index_json, f, indent=4, cls=ExtendedEncoder)
|
2017-10-19 11:38:17 +13:00
|
|
|
|
|
|
|
chmod_file(path)
|
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-27 12:21:34 +13:00
|
|
|
def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]:
|
2019-03-21 18:28:12 +13:00
|
|
|
"""parse a archive index json file and return the list of links"""
|
2019-03-27 12:21:34 +13:00
|
|
|
|
2019-03-28 03:31:07 +13:00
|
|
|
allowed_fields = {f.name for f in fields(Link)}
|
|
|
|
|
2017-10-23 22:58:41 +13:00
|
|
|
index_path = os.path.join(out_dir, 'index.json')
|
|
|
|
if os.path.exists(index_path):
|
|
|
|
with open(index_path, 'r', encoding='utf-8') as f:
|
2019-02-22 11:45:28 +13:00
|
|
|
links = json.load(f)['links']
|
2019-03-28 03:31:07 +13:00
|
|
|
for link_json in links:
|
|
|
|
yield Link(**{
|
|
|
|
key: val
|
|
|
|
for key, val in link_json.items()
|
|
|
|
if key in allowed_fields
|
|
|
|
})
|
2017-10-23 22:58:41 +13:00
|
|
|
|
2019-03-27 12:21:34 +13:00
|
|
|
return ()
|
2017-10-23 22:58:41 +13:00
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None:
|
2017-10-19 11:38:17 +13:00
|
|
|
"""write the html link index to a given path"""
|
|
|
|
|
|
|
|
path = os.path.join(out_dir, 'index.html')
|
2017-07-04 22:38:07 +12:00
|
|
|
|
2019-03-28 04:39:51 +13:00
|
|
|
copy_and_overwrite(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static'))
|
2018-04-17 23:00:40 +12:00
|
|
|
|
2018-09-13 11:25:48 +12:00
|
|
|
with open(os.path.join(out_dir, 'robots.txt'), 'w+') as f:
|
|
|
|
f.write('User-agent: *\nDisallow: /')
|
|
|
|
|
2018-06-11 13:12:55 +12:00
|
|
|
with open(os.path.join(TEMPLATES_DIR, 'index.html'), 'r', encoding='utf-8') as f:
|
2017-07-04 22:38:07 +12:00
|
|
|
index_html = f.read()
|
|
|
|
|
2018-06-11 13:12:55 +12:00
|
|
|
with open(os.path.join(TEMPLATES_DIR, 'index_row.html'), 'r', encoding='utf-8') as f:
|
2017-10-19 11:38:17 +13:00
|
|
|
link_row_html = f.read()
|
2017-07-04 22:38:07 +12:00
|
|
|
|
2017-10-19 11:38:17 +13:00
|
|
|
link_rows = '\n'.join(
|
2019-03-20 11:09:46 +13:00
|
|
|
Template(link_row_html).substitute(**{
|
2019-03-27 12:21:34 +13:00
|
|
|
**derived_link_info(link),
|
2019-03-20 11:09:46 +13:00
|
|
|
'title': (
|
2019-03-27 12:21:34 +13:00
|
|
|
link.title
|
|
|
|
or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
|
2019-03-20 11:09:46 +13:00
|
|
|
),
|
2019-03-27 20:49:39 +13:00
|
|
|
'tags': (link.tags or '') + (' {}'.format(link.extension) if link.is_static else ''),
|
2019-03-20 21:30:00 +13:00
|
|
|
'favicon_url': (
|
2019-03-27 12:21:34 +13:00
|
|
|
os.path.join('archive', link.timestamp, 'favicon.ico')
|
2019-03-20 21:30:00 +13:00
|
|
|
# if link['is_archived'] else 'data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs='
|
|
|
|
),
|
2019-03-21 22:35:41 +13:00
|
|
|
'archive_url': urlencode(
|
2019-03-20 11:09:46 +13:00
|
|
|
wget_output_path(link) or 'index.html'
|
|
|
|
),
|
|
|
|
})
|
2019-03-27 12:21:34 +13:00
|
|
|
for link in links
|
2017-07-04 22:38:07 +12:00
|
|
|
)
|
|
|
|
|
2017-07-04 23:24:03 +12:00
|
|
|
template_vars = {
|
|
|
|
'num_links': len(links),
|
|
|
|
'date_updated': datetime.now().strftime('%Y-%m-%d'),
|
|
|
|
'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
|
2018-04-18 01:14:01 +12:00
|
|
|
'footer_info': FOOTER_INFO,
|
|
|
|
'git_sha': GIT_SHA,
|
|
|
|
'short_git_sha': GIT_SHA[:8],
|
2017-10-19 11:38:17 +13:00
|
|
|
'rows': link_rows,
|
2019-03-20 11:08:42 +13:00
|
|
|
'status': 'finished' if finished else 'running',
|
2017-07-04 23:24:03 +12:00
|
|
|
}
|
2017-07-04 22:38:07 +12:00
|
|
|
|
2017-10-19 11:38:17 +13:00
|
|
|
with open(path, 'w', encoding='utf-8') as f:
|
2017-07-06 09:33:51 +12:00
|
|
|
f.write(Template(index_html).substitute(**template_vars))
|
|
|
|
|
2017-10-23 22:58:41 +13:00
|
|
|
chmod_file(path)
|
2017-07-06 09:33:51 +12:00
|
|
|
|
2017-10-19 11:38:17 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
|
2019-03-20 11:09:46 +13:00
|
|
|
"""hack to in-place update one row's info in the generated index html"""
|
|
|
|
|
2019-03-27 12:21:34 +13:00
|
|
|
title = link.title or link.latest_outputs()['title']
|
|
|
|
successful = link.num_outputs
|
2019-02-22 11:45:28 +13:00
|
|
|
|
2019-03-20 11:09:46 +13:00
|
|
|
# Patch JSON index
|
2019-02-22 11:45:28 +13:00
|
|
|
changed = False
|
2019-03-21 18:28:12 +13:00
|
|
|
json_file_links = parse_json_links_index(out_dir)
|
2019-03-27 12:21:34 +13:00
|
|
|
patched_links = []
|
2019-03-21 18:28:12 +13:00
|
|
|
for saved_link in json_file_links:
|
2019-03-27 12:21:34 +13:00
|
|
|
if saved_link.url == link.url:
|
2019-03-27 20:49:39 +13:00
|
|
|
patched_links.append(saved_link.overwrite(
|
|
|
|
title=title,
|
|
|
|
history=link.history,
|
|
|
|
updated=link.updated,
|
|
|
|
))
|
2019-03-27 12:21:34 +13:00
|
|
|
else:
|
|
|
|
patched_links.append(saved_link)
|
|
|
|
|
|
|
|
write_json_links_index(out_dir, patched_links)
|
2019-02-22 11:45:28 +13:00
|
|
|
|
2019-03-20 11:09:46 +13:00
|
|
|
# Patch HTML index
|
2019-03-21 18:28:12 +13:00
|
|
|
html_path = os.path.join(out_dir, 'index.html')
|
2019-03-20 11:09:46 +13:00
|
|
|
html = open(html_path, 'r').read().split('\n')
|
|
|
|
for idx, line in enumerate(html):
|
2019-03-27 12:21:34 +13:00
|
|
|
if title and ('<span data-title-for="{}"'.format(link.url) in line):
|
2019-03-20 11:09:46 +13:00
|
|
|
html[idx] = '<span>{}</span>'.format(title)
|
2019-03-27 12:21:34 +13:00
|
|
|
elif successful and ('<span data-number-for="{}"'.format(link.url) in line):
|
2019-03-20 11:09:46 +13:00
|
|
|
html[idx] = '<span>{}</span>'.format(successful)
|
|
|
|
break
|
|
|
|
|
|
|
|
with open(html_path, 'w') as f:
|
|
|
|
f.write('\n'.join(html))
|
2019-02-22 11:45:28 +13:00
|
|
|
|
2019-03-21 18:28:12 +13:00
|
|
|
|
2017-10-23 22:58:41 +13:00
|
|
|
### Individual link index
|
2017-10-19 11:38:17 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def write_link_index(out_dir: str, link: Link) -> None:
|
2017-10-19 11:38:17 +13:00
|
|
|
write_json_link_index(out_dir, link)
|
|
|
|
write_html_link_index(out_dir, link)
|
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def write_json_link_index(out_dir: str, link: Link) -> None:
|
2017-10-19 11:38:17 +13:00
|
|
|
"""write a json file with some info about the link"""
|
|
|
|
|
|
|
|
path = os.path.join(out_dir, 'index.json')
|
|
|
|
|
|
|
|
with open(path, 'w', encoding='utf-8') as f:
|
2019-03-27 12:21:34 +13:00
|
|
|
json.dump(link._asdict(), f, indent=4, cls=ExtendedEncoder)
|
2017-10-19 11:38:17 +13:00
|
|
|
|
|
|
|
chmod_file(path)
|
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-27 12:21:34 +13:00
|
|
|
def parse_json_link_index(out_dir: str) -> Optional[Link]:
|
2017-10-23 22:58:41 +13:00
|
|
|
"""load the json link index from a given directory"""
|
|
|
|
existing_index = os.path.join(out_dir, 'index.json')
|
|
|
|
if os.path.exists(existing_index):
|
|
|
|
with open(existing_index, 'r', encoding='utf-8') as f:
|
2019-02-22 11:45:28 +13:00
|
|
|
link_json = json.load(f)
|
2019-03-27 12:21:34 +13:00
|
|
|
return Link(**link_json)
|
|
|
|
return None
|
2017-10-23 22:58:41 +13:00
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def load_json_link_index(out_dir: str, link: Link) -> Link:
|
2019-03-21 18:28:12 +13:00
|
|
|
"""check for an existing link archive in the given directory,
|
|
|
|
and load+merge it into the given link dict
|
|
|
|
"""
|
2019-03-27 12:21:34 +13:00
|
|
|
existing_link = parse_json_link_index(out_dir)
|
2019-03-27 15:26:21 +13:00
|
|
|
if existing_link:
|
|
|
|
return merge_links(existing_link, link)
|
|
|
|
return link
|
2019-03-21 18:28:12 +13:00
|
|
|
|
2019-03-26 22:33:34 +13:00
|
|
|
|
2019-03-27 15:26:21 +13:00
|
|
|
@enforce_types
|
2019-03-26 20:20:41 +13:00
|
|
|
def write_html_link_index(out_dir: str, link: Link) -> None:
|
2019-03-09 11:46:14 +13:00
|
|
|
with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f:
|
2017-10-19 11:38:17 +13:00
|
|
|
link_html = f.read()
|
|
|
|
|
|
|
|
path = os.path.join(out_dir, 'index.html')
|
|
|
|
|
|
|
|
with open(path, 'w', encoding='utf-8') as f:
|
|
|
|
f.write(Template(link_html).substitute({
|
2019-03-27 12:21:34 +13:00
|
|
|
**derived_link_info(link),
|
2019-03-20 11:09:46 +13:00
|
|
|
'title': (
|
2019-03-27 12:21:34 +13:00
|
|
|
link.title
|
|
|
|
or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
|
2019-03-20 11:09:46 +13:00
|
|
|
),
|
2019-03-21 22:35:41 +13:00
|
|
|
'archive_url': urlencode(
|
2019-03-20 11:09:46 +13:00
|
|
|
wget_output_path(link)
|
2019-03-27 12:21:34 +13:00
|
|
|
or (link.domain if link.is_archived else 'about:blank')
|
2019-03-20 11:09:46 +13:00
|
|
|
),
|
2019-03-27 12:21:34 +13:00
|
|
|
'extension': link.extension or 'html',
|
|
|
|
'tags': link.tags or 'untagged',
|
2019-03-27 20:49:39 +13:00
|
|
|
'status': 'archived' if link.is_archived else 'not yet archived',
|
2019-03-27 12:21:34 +13:00
|
|
|
'status_color': 'success' if link.is_archived else 'danger',
|
2017-10-19 11:38:17 +13:00
|
|
|
}))
|
|
|
|
|
|
|
|
chmod_file(path)
|