make everything take link_dir as an optional arg since its derivable from link url
This commit is contained in:
Nick Sweeting
parent
9fc1e3c3e1
commit
a214bd7c02
|
|
@ -180,7 +180,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]
|
|||
all_links, new_links = load_links_index(out_dir=OUTPUT_DIR, import_path=import_path)
|
||||
|
||||
# Step 2: Write updated index with deduped old and new links back to disk
|
||||
write_links_index(out_dir=OUTPUT_DIR, links=list(all_links))
|
||||
write_links_index(links=list(all_links), out_dir=OUTPUT_DIR)
|
||||
|
||||
# Step 3: Run the archive methods for each link
|
||||
links = new_links if ONLY_NEW else all_links
|
||||
|
|
@ -189,7 +189,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]
|
|||
link: Optional[Link] = None
|
||||
try:
|
||||
for idx, link in enumerate(links_after_timestamp(links, resume)):
|
||||
archive_link(link)
|
||||
archive_link(link, link_dir=link.link_dir)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
log_archiving_paused(len(links), idx, link.timestamp if link else '0')
|
||||
|
|
@ -203,7 +203,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]
|
|||
|
||||
# Step 4: Re-write links index with updated titles, icons, and resources
|
||||
all_links, _ = load_links_index(out_dir=OUTPUT_DIR)
|
||||
write_links_index(out_dir=OUTPUT_DIR, links=list(all_links), finished=True)
|
||||
write_links_index(links=list(all_links), out_dir=OUTPUT_DIR, finished=True)
|
||||
return all_links
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import os
|
||||
|
||||
from typing import Dict, List, Tuple
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
|
|
@ -69,7 +69,7 @@ class ArchiveError(Exception):
|
|||
|
||||
|
||||
@enforce_types
|
||||
def archive_link(link: Link, page=None) -> Link:
|
||||
def archive_link(link: Link, link_dir: Optional[str]=None) -> Link:
|
||||
"""download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
|
||||
|
||||
ARCHIVE_METHODS = (
|
||||
|
|
@ -84,13 +84,14 @@ def archive_link(link: Link, page=None) -> Link:
|
|||
('archive_org', should_fetch_archive_dot_org, archive_dot_org),
|
||||
)
|
||||
|
||||
link_dir = link_dir or link.link_dir
|
||||
try:
|
||||
is_new = not os.path.exists(link.link_dir)
|
||||
is_new = not os.path.exists(link_dir)
|
||||
if is_new:
|
||||
os.makedirs(link.link_dir)
|
||||
os.makedirs(link_dir)
|
||||
|
||||
link = load_json_link_index(link.link_dir, link)
|
||||
log_link_archiving_started(link.link_dir, link, is_new)
|
||||
link = load_json_link_index(link, link_dir)
|
||||
log_link_archiving_started(link, link_dir, is_new)
|
||||
link = link.overwrite(updated=datetime.now())
|
||||
stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
|
||||
|
||||
|
|
@ -99,10 +100,10 @@ def archive_link(link: Link, page=None) -> Link:
|
|||
if method_name not in link.history:
|
||||
link.history[method_name] = []
|
||||
|
||||
if should_run(link.link_dir, link):
|
||||
if should_run(link, link_dir):
|
||||
log_archive_method_started(method_name)
|
||||
|
||||
result = method_function(link.link_dir, link)
|
||||
result = method_function(link, link_dir)
|
||||
|
||||
link.history[method_name].append(result)
|
||||
|
||||
|
|
@ -126,7 +127,7 @@ def archive_link(link: Link, page=None) -> Link:
|
|||
patch_links_index(link)
|
||||
|
||||
|
||||
log_link_archiving_finished(link.link_dir, link, is_new, stats)
|
||||
log_link_archiving_finished(link, link.link_dir, is_new, stats)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
|
|
@ -141,7 +142,7 @@ def archive_link(link: Link, page=None) -> Link:
|
|||
### Archive Method Functions
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_title(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_title(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
# if link already has valid title, skip it
|
||||
if link.title and not link.title.lower().startswith('http'):
|
||||
return False
|
||||
|
|
@ -152,7 +153,7 @@ def should_fetch_title(link_dir: str, link: Link) -> bool:
|
|||
return FETCH_TITLE
|
||||
|
||||
@enforce_types
|
||||
def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_title(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""try to guess the page's title from its content"""
|
||||
|
||||
output = None
|
||||
|
|
@ -186,14 +187,14 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_favicon(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_favicon(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if os.path.exists(os.path.join(link_dir, 'favicon.ico')):
|
||||
return False
|
||||
|
||||
return FETCH_FAVICON
|
||||
|
||||
@enforce_types
|
||||
def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_favicon(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""download site favicon from google's favicon api"""
|
||||
|
||||
output = 'favicon.ico'
|
||||
|
|
@ -226,7 +227,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes
|
|||
)
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_wget(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_wget(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
output_path = wget_output_path(link)
|
||||
if output_path and os.path.exists(os.path.join(link_dir, output_path)):
|
||||
return False
|
||||
|
|
@ -235,7 +236,7 @@ def should_fetch_wget(link_dir: str, link: Link) -> bool:
|
|||
|
||||
|
||||
@enforce_types
|
||||
def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""download full site using wget"""
|
||||
|
||||
if FETCH_WARC:
|
||||
|
|
@ -315,7 +316,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult
|
|||
)
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_pdf(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_pdf(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
|
|
@ -326,7 +327,7 @@ def should_fetch_pdf(link_dir: str, link: Link) -> bool:
|
|||
|
||||
|
||||
@enforce_types
|
||||
def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_pdf(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""print PDF of site to file using chrome --headless"""
|
||||
|
||||
output = 'output.pdf'
|
||||
|
|
@ -361,7 +362,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
|||
)
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_screenshot(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_screenshot(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
|
|
@ -371,7 +372,7 @@ def should_fetch_screenshot(link_dir: str, link: Link) -> bool:
|
|||
return FETCH_SCREENSHOT
|
||||
|
||||
@enforce_types
|
||||
def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_screenshot(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""take screenshot of site using chrome --headless"""
|
||||
|
||||
output = 'screenshot.png'
|
||||
|
|
@ -406,7 +407,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive
|
|||
)
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_dom(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_dom(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
|
|
@ -416,7 +417,7 @@ def should_fetch_dom(link_dir: str, link: Link) -> bool:
|
|||
return FETCH_DOM
|
||||
|
||||
@enforce_types
|
||||
def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_dom(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""print HTML of site to file using chrome --dump-html"""
|
||||
|
||||
output = 'output.html'
|
||||
|
|
@ -453,7 +454,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
|||
)
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_git(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_git(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
|
|
@ -471,7 +472,7 @@ def should_fetch_git(link_dir: str, link: Link) -> bool:
|
|||
|
||||
|
||||
@enforce_types
|
||||
def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def fetch_git(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""download full site using git"""
|
||||
|
||||
output = 'git'
|
||||
|
|
@ -514,7 +515,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_media(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_media(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
|
|
@ -524,7 +525,7 @@ def should_fetch_media(link_dir: str, link: Link) -> bool:
|
|||
return FETCH_MEDIA
|
||||
|
||||
@enforce_types
|
||||
def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
|
||||
def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
|
||||
"""Download playlists or individual video, audio, and subtitles using youtube-dl"""
|
||||
|
||||
output = 'media'
|
||||
|
|
@ -588,7 +589,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool:
|
||||
def should_fetch_archive_dot_org(link: Link, link_dir: Optional[str]=None) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
|
|
@ -599,7 +600,7 @@ def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool:
|
|||
return SUBMIT_ARCHIVE_DOT_ORG
|
||||
|
||||
@enforce_types
|
||||
def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""submit site to archive.org for archiving via their service, save returned archive url"""
|
||||
|
||||
output = 'archive.org.txt'
|
||||
|
|
|
|||
|
|
@ -39,23 +39,25 @@ from .logs import (
|
|||
TITLE_LOADING_MSG = 'Not yet archived...'
|
||||
|
||||
|
||||
|
||||
|
||||
### Homepage index for all the links
|
||||
|
||||
@enforce_types
|
||||
def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None:
|
||||
def write_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None:
|
||||
"""create index.html file for a given list of links"""
|
||||
|
||||
log_indexing_process_started()
|
||||
|
||||
log_indexing_started(out_dir, 'index.json')
|
||||
timer = TimedProgress(TIMEOUT * 2, prefix=' ')
|
||||
write_json_links_index(out_dir, links)
|
||||
write_json_links_index(links, out_dir=out_dir)
|
||||
timer.end()
|
||||
log_indexing_finished(out_dir, 'index.json')
|
||||
|
||||
log_indexing_started(out_dir, 'index.html')
|
||||
timer = TimedProgress(TIMEOUT * 2, prefix=' ')
|
||||
write_html_links_index(out_dir, links, finished=finished)
|
||||
write_html_links_index(links, out_dir=out_dir, finished=finished)
|
||||
timer.end()
|
||||
log_indexing_finished(out_dir, 'index.html')
|
||||
|
||||
|
|
@ -87,7 +89,7 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) -
|
|||
|
||||
|
||||
@enforce_types
|
||||
def write_json_links_index(out_dir: str, links: List[Link]) -> None:
|
||||
def write_json_links_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None:
|
||||
"""write the json link index to a given path"""
|
||||
|
||||
assert isinstance(links, List), 'Links must be a list, not a generator.'
|
||||
|
|
@ -199,7 +201,6 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
|
|||
successful = link.num_outputs
|
||||
|
||||
# Patch JSON index
|
||||
changed = False
|
||||
json_file_links = parse_json_links_index(out_dir)
|
||||
patched_links = []
|
||||
for saved_link in json_file_links:
|
||||
|
|
@ -212,7 +213,7 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
|
|||
else:
|
||||
patched_links.append(saved_link)
|
||||
|
||||
write_json_links_index(out_dir, patched_links)
|
||||
write_json_links_index(patched_links, out_dir=out_dir)
|
||||
|
||||
# Patch HTML index
|
||||
html_path = os.path.join(out_dir, 'index.html')
|
||||
|
|
@ -231,27 +232,27 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None:
|
|||
### Individual link index
|
||||
|
||||
@enforce_types
|
||||
def write_link_index(out_dir: str, link: Link) -> None:
|
||||
write_json_link_index(out_dir, link)
|
||||
write_html_link_index(out_dir, link)
|
||||
def write_link_index(link: Link, link_dir: Optional[str]=None) -> None:
|
||||
link_dir = link_dir or link.link_dir
|
||||
|
||||
write_json_link_index(link, link_dir)
|
||||
write_html_link_index(link, link_dir)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def write_json_link_index(out_dir: str, link: Link) -> None:
|
||||
def write_json_link_index(link: Link, link_dir: Optional[str]=None) -> None:
|
||||
"""write a json file with some info about the link"""
|
||||
|
||||
path = os.path.join(out_dir, 'index.json')
|
||||
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(link._asdict(), f, indent=4, cls=ExtendedEncoder)
|
||||
link_dir = link_dir or link.link_dir
|
||||
path = os.path.join(link_dir, 'index.json')
|
||||
|
||||
chmod_file(path)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def parse_json_link_index(out_dir: str) -> Optional[Link]:
|
||||
def parse_json_link_index(link_dir: str) -> Optional[Link]:
|
||||
"""load the json link index from a given directory"""
|
||||
existing_index = os.path.join(out_dir, 'index.json')
|
||||
existing_index = os.path.join(link_dir, 'index.json')
|
||||
if os.path.exists(existing_index):
|
||||
with open(existing_index, 'r', encoding='utf-8') as f:
|
||||
link_json = json.load(f)
|
||||
|
|
@ -260,18 +261,21 @@ def parse_json_link_index(out_dir: str) -> Optional[Link]:
|
|||
|
||||
|
||||
@enforce_types
|
||||
def load_json_link_index(out_dir: str, link: Link) -> Link:
|
||||
def load_json_link_index(link: Link, link_dir: Optional[str]=None) -> Link:
|
||||
"""check for an existing link archive in the given directory,
|
||||
and load+merge it into the given link dict
|
||||
"""
|
||||
existing_link = parse_json_link_index(out_dir)
|
||||
link_dir = link_dir or link.link_dir
|
||||
existing_link = parse_json_link_index(link_dir)
|
||||
if existing_link:
|
||||
return merge_links(existing_link, link)
|
||||
return link
|
||||
|
||||
|
||||
@enforce_types
|
||||
def write_html_link_index(out_dir: str, link: Link) -> None:
|
||||
def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None:
|
||||
link_dir = link_dir or link.link_dir
|
||||
|
||||
with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f:
|
||||
link_html = f.read()
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from dataclasses import dataclass
|
|||
from typing import Optional
|
||||
|
||||
from .schema import Link, ArchiveResult
|
||||
from .config import ANSI, REPO_DIR, OUTPUT_DIR
|
||||
from .config import ANSI, OUTPUT_DIR
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -17,14 +17,14 @@ class RuntimeStats:
|
|||
succeeded: int = 0
|
||||
failed: int = 0
|
||||
|
||||
parse_start_ts: datetime = None
|
||||
parse_end_ts: datetime = None
|
||||
parse_start_ts: Optional[datetime] = None
|
||||
parse_end_ts: Optional[datetime] = None
|
||||
|
||||
index_start_ts: datetime = None
|
||||
index_end_ts: datetime = None
|
||||
index_start_ts: Optional[datetime] = None
|
||||
index_end_ts: Optional[datetime] = None
|
||||
|
||||
archiving_start_ts: datetime = None
|
||||
archiving_end_ts: datetime = None
|
||||
archiving_start_ts: Optional[datetime] = None
|
||||
archiving_end_ts: Optional[datetime] = None
|
||||
|
||||
# globals are bad, mmkay
|
||||
_LAST_RUN_STATS = RuntimeStats()
|
||||
|
|
@ -131,7 +131,7 @@ def log_archiving_finished(num_links: int):
|
|||
print(' {}/index.html'.format(OUTPUT_DIR))
|
||||
|
||||
|
||||
def log_link_archiving_started(link_dir: str, link: Link, is_new: bool):
|
||||
def log_link_archiving_started(link: Link, link_dir: str, is_new: bool):
|
||||
# [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
|
||||
# http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
|
||||
# > output/archive/1478739709
|
||||
|
|
@ -149,7 +149,7 @@ def log_link_archiving_started(link_dir: str, link: Link, is_new: bool):
|
|||
pretty_path(link_dir),
|
||||
))
|
||||
|
||||
def log_link_archiving_finished(link_dir: str, link: Link, is_new: bool, stats: dict):
|
||||
def log_link_archiving_finished(link: Link, link_dir: str, is_new: bool, stats: dict):
|
||||
total = sum(stats.values())
|
||||
|
||||
if stats['failed'] > 0 :
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import shutil
|
||||
|
||||
from json import JSONEncoder
|
||||
from typing import List, Optional, Any
|
||||
from typing import List, Optional, Any, Union
|
||||
from inspect import signature, _empty
|
||||
from functools import wraps
|
||||
from hashlib import sha256
|
||||
|
|
|
|||
Loading…
Reference in a new issue