From 63abc8980cc7416dc3f1b168c7aca8c0b4bff276 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 03:20:41 -0400 Subject: [PATCH 001/365] add mypy type hints --- archivebox/archive_methods.py | 174 ++++++++++++++++++---------------- archivebox/index.py | 29 +++--- archivebox/util.py | 111 ++++++++++++---------- 3 files changed, 171 insertions(+), 143 deletions(-) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index b3915e2f..f0223bbb 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -1,5 +1,7 @@ import os +import json +from typing import Union, Dict, List, Tuple, NamedTuple from collections import defaultdict from datetime import datetime @@ -40,13 +42,15 @@ from util import ( without_query, without_fragment, fetch_page_title, + read_js_script, is_static_file, TimedProgress, chmod_file, wget_output_path, chrome_args, check_link_structure, - run, PIPE, DEVNULL + run, PIPE, DEVNULL, + Link, ) from logs import ( log_link_archiving_started, @@ -55,15 +59,22 @@ from logs import ( log_archive_method_finished, ) - - class ArchiveError(Exception): def __init__(self, message, hints=None): super().__init__(message) self.hints = hints +class ArchiveResult(NamedTuple): + cmd: List[str] + pwd: str + output: Union[str, Exception, None] + status: str + start_ts: datetime + end_ts: datetime + duration: int -def archive_link(link_dir, link): + +def archive_link(link_dir: str, link: Link, page=None) -> Link: """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" ARCHIVE_METHODS = ( @@ -95,10 +106,11 @@ def archive_link(link_dir, link): log_archive_method_started(method_name) result = method_function(link_dir, link) - link['history'][method_name].append(result) - stats[result['status']] += 1 - log_archive_method_finished(result) + link['history'][method_name].append(result._asdict()) + + stats[result.status] += 1 + log_archive_method_finished(result._asdict()) else: stats['skipped'] += 1 @@ -117,7 +129,7 @@ def archive_link(link_dir, link): ### Archive Method Functions -def should_fetch_title(link_dir, link): +def should_fetch_title(link_dir: str, link: Link) -> bool: # if link already has valid title, skip it if link['title'] and not link['title'].lower().startswith('http'): return False @@ -127,7 +139,7 @@ def should_fetch_title(link_dir, link): return FETCH_TITLE -def fetch_title(link_dir, link, timeout=TIMEOUT): +def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """try to guess the page's title from its content""" output = None @@ -150,22 +162,22 @@ def fetch_title(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_favicon(link_dir, link): +def should_fetch_favicon(link_dir: str, link: Link) -> bool: if os.path.exists(os.path.join(link_dir, 'favicon.ico')): return False return FETCH_FAVICON -def fetch_favicon(link_dir, link, timeout=TIMEOUT): +def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """download site favicon from google's favicon api""" output = 'favicon.ico' @@ -188,15 +200,15 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_wget(link_dir, link): +def should_fetch_wget(link_dir: str, link: Link) -> bool: output_path = wget_output_path(link) if output_path and os.path.exists(os.path.join(link_dir, output_path)): return False @@ -204,7 +216,7 @@ def should_fetch_wget(link_dir, link): return FETCH_WGET -def fetch_wget(link_dir, link, timeout=TIMEOUT): +def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using wget""" if FETCH_WARC: @@ -274,15 +286,15 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_pdf(link_dir, link): +def should_fetch_pdf(link_dir: str, link: Link) -> bool: if is_static_file(link['url']): return False @@ -292,7 +304,7 @@ def should_fetch_pdf(link_dir, link): return FETCH_PDF -def fetch_pdf(link_dir, link, timeout=TIMEOUT): +def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """print PDF of site to file using chrome --headless""" output = 'output.pdf' @@ -317,15 +329,15 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_screenshot(link_dir, link): +def should_fetch_screenshot(link_dir: str, link: Link) -> bool: if is_static_file(link['url']): return False @@ -334,7 +346,7 @@ def should_fetch_screenshot(link_dir, link): return FETCH_SCREENSHOT -def fetch_screenshot(link_dir, link, timeout=TIMEOUT): +def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """take screenshot of site using chrome --headless""" output = 'screenshot.png' @@ -359,15 +371,15 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_dom(link_dir, link): +def should_fetch_dom(link_dir: str, link: Link) -> bool: if is_static_file(link['url']): return False @@ -376,7 +388,7 @@ def should_fetch_dom(link_dir, link): return FETCH_DOM -def fetch_dom(link_dir, link, timeout=TIMEOUT): +def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """print HTML of site to file using chrome --dump-html""" output = 'output.html' @@ -403,15 +415,15 @@ def fetch_dom(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_git(link_dir, link): +def should_fetch_git(link_dir: str, link: Link) -> bool: if is_static_file(link['url']): return False @@ -428,7 +440,7 @@ def should_fetch_git(link_dir, link): return FETCH_GIT -def fetch_git(link_dir, link, timeout=TIMEOUT): +def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using git""" output = 'git' @@ -460,16 +472,16 @@ def fetch_git(link_dir, link, timeout=TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_media(link_dir, link): +def should_fetch_media(link_dir: str, link: Link) -> bool: if is_static_file(link['url']): return False @@ -478,7 +490,7 @@ def should_fetch_media(link_dir, link): return FETCH_MEDIA -def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT): +def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: """Download playlists or individual video, audio, and subtitles using youtube-dl""" output = 'media' @@ -531,16 +543,16 @@ def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT): finally: timer.end() - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def should_fetch_archive_dot_org(link_dir, link): +def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: if is_static_file(link['url']): return False @@ -550,7 +562,7 @@ def should_fetch_archive_dot_org(link_dir, link): return SUBMIT_ARCHIVE_DOT_ORG -def archive_dot_org(link_dir, link, timeout=TIMEOUT): +def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """submit site to archive.org for archiving via their service, save returned archive url""" output = 'archive.org.txt' @@ -596,17 +608,17 @@ def archive_dot_org(link_dir, link, timeout=TIMEOUT): chmod_file('archive.org.txt', cwd=link_dir) output = archive_org_url - return { - 'cmd': cmd, - 'pwd': link_dir, - 'output': output, - 'status': status, + return ArchiveResult( + cmd=cmd, + pwd=link_dir, + output=output, + status=status, **timer.stats, - } + ) -def parse_archive_dot_org_response(response): +def parse_archive_dot_org_response(response: bytes) -> Tuple[List[str], List[str]]: # Parse archive.org response headers - headers = defaultdict(list) + headers: Dict[str, List[str]] = defaultdict(list) # lowercase all the header names and store in dict for header in response.splitlines(): diff --git a/archivebox/index.py b/archivebox/index.py index 3f4ada3f..503b82ad 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -3,6 +3,8 @@ import json from datetime import datetime from string import Template +from typing import List, Tuple + try: from distutils.dir_util import copy_tree except ImportError: @@ -23,6 +25,7 @@ from util import ( check_links_structure, wget_output_path, latest_output, + Link, ) from parse import parse_links from links import validate_links @@ -39,7 +42,7 @@ TITLE_LOADING_MSG = 'Not yet archived...' ### Homepage index for all the links -def write_links_index(out_dir, links, finished=False): +def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: """create index.html file for a given list of links""" log_indexing_process_started() @@ -53,15 +56,15 @@ def write_links_index(out_dir, links, finished=False): write_html_links_index(out_dir, links, finished=finished) log_indexing_finished(out_dir, 'index.html') -def load_links_index(out_dir=OUTPUT_DIR, import_path=None): +def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[List[Link], List[Link]]: """parse and load existing index with any new links from import_path merged in""" - existing_links = [] + existing_links: List[Link] = [] if out_dir: existing_links = parse_json_links_index(out_dir) check_links_structure(existing_links) - new_links = [] + new_links: List[Link] = [] if import_path: # parse and validate the import file log_parsing_started(import_path) @@ -79,7 +82,7 @@ def load_links_index(out_dir=OUTPUT_DIR, import_path=None): return all_links, new_links -def write_json_links_index(out_dir, links): +def write_json_links_index(out_dir: str, links: List[Link]) -> None: """write the json link index to a given path""" check_links_structure(links) @@ -100,7 +103,7 @@ def write_json_links_index(out_dir, links): chmod_file(path) -def parse_json_links_index(out_dir=OUTPUT_DIR): +def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> List[Link]: """parse a archive index json file and return the list of links""" index_path = os.path.join(out_dir, 'index.json') if os.path.exists(index_path): @@ -111,7 +114,7 @@ def parse_json_links_index(out_dir=OUTPUT_DIR): return [] -def write_html_links_index(out_dir, links, finished=False): +def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: """write the html link index to a given path""" check_links_structure(links) @@ -166,7 +169,7 @@ def write_html_links_index(out_dir, links, finished=False): chmod_file(path) -def patch_links_index(link, out_dir=OUTPUT_DIR): +def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: """hack to in-place update one row's info in the generated index html""" title = link['title'] or latest_output(link)['title'] @@ -200,12 +203,12 @@ def patch_links_index(link, out_dir=OUTPUT_DIR): ### Individual link index -def write_link_index(out_dir, link): +def write_link_index(out_dir: str, link: Link) -> None: link['updated'] = str(datetime.now().timestamp()) write_json_link_index(out_dir, link) write_html_link_index(out_dir, link) -def write_json_link_index(out_dir, link): +def write_json_link_index(out_dir: str, link: Link) -> None: """write a json file with some info about the link""" check_link_structure(link) @@ -216,7 +219,7 @@ def write_json_link_index(out_dir, link): chmod_file(path) -def parse_json_link_index(out_dir): +def parse_json_link_index(out_dir: str) -> dict: """load the json link index from a given directory""" existing_index = os.path.join(out_dir, 'index.json') if os.path.exists(existing_index): @@ -226,7 +229,7 @@ def parse_json_link_index(out_dir): return link_json return {} -def load_json_link_index(out_dir, link): +def load_json_link_index(out_dir: str, link: Link) -> Link: """check for an existing link archive in the given directory, and load+merge it into the given link dict """ @@ -241,7 +244,7 @@ def load_json_link_index(out_dir, link): check_link_structure(link) return link -def write_html_link_index(out_dir, link): +def write_html_link_index(out_dir: str, link: Link) -> None: check_link_structure(link) with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: link_html = f.read() diff --git a/archivebox/util.py b/archivebox/util.py index cec23035..1a8a445e 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -3,6 +3,8 @@ import re import sys import time +from typing import List, Dict, Any, Optional, Union + from urllib.request import Request, urlopen from urllib.parse import urlparse, quote from decimal import Decimal @@ -30,6 +32,7 @@ from config import ( CHECK_SSL_VALIDITY, WGET_USER_AGENT, CHROME_OPTIONS, + PYTHON_PATH, ) from logs import pretty_path @@ -86,9 +89,11 @@ STATICFILE_EXTENSIONS = { # html, htm, shtml, xhtml, xml, aspx, php, cgi } +Link = Dict[str, Any] + ### Checks & Tests -def check_link_structure(link): +def check_link_structure(link: Link) -> None: """basic sanity check invariants to make sure the data is valid""" assert isinstance(link, dict) assert isinstance(link.get('url'), str) @@ -100,13 +105,13 @@ def check_link_structure(link): assert isinstance(key, str) assert isinstance(val, list), 'history must be a Dict[str, List], got: {}'.format(link['history']) -def check_links_structure(links): +def check_links_structure(links: List[Link]) -> None: """basic sanity check invariants to make sure the data is valid""" assert isinstance(links, list) if links: check_link_structure(links[0]) -def check_url_parsing_invariants(): +def check_url_parsing_invariants() -> None: """Check that plain text regex URL parsing works as expected""" # this is last-line-of-defense to make sure the URL_REGEX isn't @@ -137,7 +142,7 @@ def check_url_parsing_invariants(): ### Random Helpers -def save_stdin_source(raw_text): +def save_stdin_source(raw_text: str) -> str: if not os.path.exists(SOURCES_DIR): os.makedirs(SOURCES_DIR) @@ -150,7 +155,7 @@ def save_stdin_source(raw_text): return source_path -def save_remote_source(url, timeout=TIMEOUT): +def save_remote_source(url: str, timeout: int=TIMEOUT) -> str: """download a given url's content into output/sources/domain-.txt""" if not os.path.exists(SOURCES_DIR): @@ -187,7 +192,7 @@ def save_remote_source(url, timeout=TIMEOUT): return source_path -def fetch_page_title(url, timeout=10, progress=SHOW_PROGRESS): +def fetch_page_title(url: str, timeout: int=10, progress: bool=SHOW_PROGRESS) -> Optional[str]: """Attempt to guess a page's title by downloading the html""" if not FETCH_TITLE: @@ -209,7 +214,7 @@ def fetch_page_title(url, timeout=10, progress=SHOW_PROGRESS): # )) return None -def wget_output_path(link): +def wget_output_path(link: Link) -> Optional[str]: """calculate the path to the wgetted .html file, since wget may adjust some paths to be different than the base_url path. @@ -278,9 +283,15 @@ def wget_output_path(link): return None +def read_js_script(script_name: str) -> str: + script_path = os.path.join(PYTHON_PATH, 'scripts', script_name) + + with open(script_path, 'r') as f: + return f.read().split('// INFO BELOW HERE')[0].strip() + ### String Manipulation & Logging Helpers -def str_between(string, start, end=None): +def str_between(string: str, start: str, end: str=None) -> str: """(12345, , ) -> 12345""" content = string.split(start, 1)[-1] @@ -292,7 +303,7 @@ def str_between(string, start, end=None): ### Link Helpers -def merge_links(a, b): +def merge_links(a: Link, b: Link) -> Link: """deterministially merge two links, favoring longer field values over shorter, and "cleaner" values over worse ones. """ @@ -310,7 +321,7 @@ def merge_links(a, b): 'sources': list(set(a.get('sources', []) + b.get('sources', []))), } -def is_static_file(url): +def is_static_file(url: str) -> bool: """Certain URLs just point to a single static file, and don't need to be re-archived in many formats """ @@ -318,7 +329,7 @@ def is_static_file(url): # TODO: the proper way is with MIME type detection, not using extension return extension(url) in STATICFILE_EXTENSIONS -def derived_link_info(link): +def derived_link_info(link: Link) -> dict: """extend link info with the archive urls and other derived data""" url = link['url'] @@ -373,7 +384,7 @@ def derived_link_info(link): return extended_info -def latest_output(link, status=None): +def latest_output(link: Link, status: str=None) -> Dict[str, Optional[str]]: """get the latest output that each archive method produced for link""" latest = { @@ -440,7 +451,42 @@ def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, return CompletedProcess(process.args, retcode, stdout, stderr) -def progress_bar(seconds, prefix): +class TimedProgress: + """Show a progress bar and measure elapsed time until .end() is called""" + + def __init__(self, seconds, prefix=''): + if SHOW_PROGRESS: + self.p = Process(target=progress_bar, args=(seconds, prefix)) + self.p.start() + + self.stats = { + 'start_ts': datetime.now(), + 'end_ts': None, + 'duration': None, + } + + def end(self): + """immediately end progress, clear the progressbar line, and save end_ts""" + + end_ts = datetime.now() + self.stats.update({ + 'end_ts': end_ts, + 'duration': (end_ts - self.stats['start_ts']).seconds, + }) + + if SHOW_PROGRESS: + # protect from double termination + #if p is None or not hasattr(p, 'kill'): + # return + if self.p is not None: + self.p.terminate() + self.p = None + + sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH), ANSI['reset'])) # clear whole terminal line + sys.stdout.flush() + + +def progress_bar(seconds: int, prefix: str='') -> None: """show timer in the form of progress bar, with percentage and seconds remaining""" chunk = '█' if sys.stdout.encoding == 'UTF-8' else '#' chunks = TERM_WIDTH - len(prefix) - 20 # number of progress chunks to show (aka max bar width) @@ -477,41 +523,8 @@ def progress_bar(seconds, prefix): print() pass -class TimedProgress: - """Show a progress bar and measure elapsed time until .end() is called""" - def __init__(self, seconds, prefix=''): - if SHOW_PROGRESS: - self.p = Process(target=progress_bar, args=(seconds, prefix)) - self.p.start() - - self.stats = { - 'start_ts': datetime.now(), - 'end_ts': None, - 'duration': None, - } - - def end(self): - """immediately end progress, clear the progressbar line, and save end_ts""" - - end_ts = datetime.now() - self.stats.update({ - 'end_ts': end_ts, - 'duration': (end_ts - self.stats['start_ts']).seconds, - }) - - if SHOW_PROGRESS: - # protect from double termination - #if p is None or not hasattr(p, 'kill'): - # return - if self.p is not None: - self.p.terminate() - self.p = None - - sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH), ANSI['reset'])) # clear whole terminal line - sys.stdout.flush() - -def download_url(url, timeout=TIMEOUT): +def download_url(url: str, timeout: int=TIMEOUT) -> str: """Download the contents of a remote url and return the text""" req = Request(url, headers={'User-Agent': WGET_USER_AGENT}) @@ -526,7 +539,7 @@ def download_url(url, timeout=TIMEOUT): encoding = resp.headers.get_content_charset() or 'utf-8' return resp.read().decode(encoding) -def chmod_file(path, cwd='.', permissions=OUTPUT_PERMISSIONS, timeout=30): +def chmod_file(path: str, cwd: str='.', permissions: str=OUTPUT_PERMISSIONS, timeout: int=30) -> None: """chmod -R /""" if not os.path.exists(os.path.join(cwd, path)): @@ -538,7 +551,7 @@ def chmod_file(path, cwd='.', permissions=OUTPUT_PERMISSIONS, timeout=30): raise Exception('Failed to chmod {}/{}'.format(cwd, path)) -def chrome_args(**options): +def chrome_args(**options) -> List[str]: """helper to build up a chrome shell command with arguments""" options = {**CHROME_OPTIONS, **options} From 4f8c99011ac82ce1da0fc8f85c13fa6bff8fbdf7 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 05:30:23 -0400 Subject: [PATCH 002/365] fix terminal resizing making progress bar go crazy --- archivebox/config.py | 2 +- archivebox/util.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index d8e01b24..7865e976 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -77,7 +77,7 @@ USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC try: ### Terminal Configuration - TERM_WIDTH = shutil.get_terminal_size((100, 10)).columns + TERM_WIDTH = lambda: shutil.get_terminal_size((100, 10)).columns ANSI = { 'reset': '\033[00;00m', 'lightblue': '\033[01;30m', diff --git a/archivebox/util.py b/archivebox/util.py index 1a8a445e..1835bd16 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -482,16 +482,17 @@ class TimedProgress: self.p.terminate() self.p = None - sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH), ANSI['reset'])) # clear whole terminal line + sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH()), ANSI['reset'])) # clear whole terminal line sys.stdout.flush() def progress_bar(seconds: int, prefix: str='') -> None: """show timer in the form of progress bar, with percentage and seconds remaining""" chunk = '█' if sys.stdout.encoding == 'UTF-8' else '#' - chunks = TERM_WIDTH - len(prefix) - 20 # number of progress chunks to show (aka max bar width) + chunks = TERM_WIDTH() - len(prefix) - 20 # number of progress chunks to show (aka max bar width) try: for s in range(seconds * chunks): + chunks = TERM_WIDTH() - len(prefix) - 20 progress = s / chunks / seconds * 100 bar_width = round(progress/(100/chunks)) From 52871f42e71e6fe399010611c104283b8c921492 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 05:31:27 -0400 Subject: [PATCH 003/365] cleaner config with type hints --- archivebox/config.py | 282 ++++++++++++++++++++----------------------- 1 file changed, 130 insertions(+), 152 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 7865e976..13d64c3a 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -3,8 +3,12 @@ import re import sys import shutil +from typing import Optional from subprocess import run, PIPE, DEVNULL + +OUTPUT_DIR: str + # ****************************************************************************** # Documentation: https://github.com/pirate/ArchiveBox/wiki/Configuration # Use the 'env' command to pass config options to ArchiveBox. e.g.: @@ -14,9 +18,11 @@ from subprocess import run, PIPE, DEVNULL IS_TTY = sys.stdout.isatty() USE_COLOR = os.getenv('USE_COLOR', str(IS_TTY) ).lower() == 'true' SHOW_PROGRESS = os.getenv('SHOW_PROGRESS', str(IS_TTY) ).lower() == 'true' + +OUTPUT_DIR = os.getenv('OUTPUT_DIR', '') ONLY_NEW = os.getenv('ONLY_NEW', 'False' ).lower() == 'true' -MEDIA_TIMEOUT = int(os.getenv('MEDIA_TIMEOUT', '3600')) TIMEOUT = int(os.getenv('TIMEOUT', '60')) +MEDIA_TIMEOUT = int(os.getenv('MEDIA_TIMEOUT', '3600')) OUTPUT_PERMISSIONS = os.getenv('OUTPUT_PERMISSIONS', '755' ) FOOTER_INFO = os.getenv('FOOTER_INFO', 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.',) @@ -47,20 +53,18 @@ WGET_BINARY = os.getenv('WGET_BINARY', 'wget') YOUTUBEDL_BINARY = os.getenv('YOUTUBEDL_BINARY', 'youtube-dl') CHROME_BINARY = os.getenv('CHROME_BINARY', None) -try: - OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR')) -except Exception: - OUTPUT_DIR = None - +CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' # ****************************************************************************** -# **************************** Derived Settings ******************************** +# *************************** Directory Settings ******************************* # ****************************************************************************** REPO_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) -if not OUTPUT_DIR: +if OUTPUT_DIR: + OUTPUT_DIR = os.path.abspath(OUTPUT_DIR) +else: OUTPUT_DIR = os.path.join(REPO_DIR, 'output') - + ARCHIVE_DIR_NAME = 'archive' SOURCES_DIR_NAME = 'sources' ARCHIVE_DIR = os.path.join(OUTPUT_DIR, ARCHIVE_DIR_NAME) @@ -69,13 +73,87 @@ SOURCES_DIR = os.path.join(OUTPUT_DIR, SOURCES_DIR_NAME) PYTHON_PATH = os.path.join(REPO_DIR, 'archivebox') TEMPLATES_DIR = os.path.join(PYTHON_PATH, 'templates') -CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' -USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM -USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC +if COOKIES_FILE: + COOKIES_FILE = os.path.abspath(COOKIES_FILE) + +# ****************************************************************************** +# ************************ Environment & Dependencies ************************** +# ****************************************************************************** + +def check_version(binary: str) -> str: + """check the presence and return valid version line of a specified binary""" + if run(['which', binary], stdout=DEVNULL, stderr=DEVNULL).returncode: + print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI)) + print(' Install it, then confirm it works with: {} --version'.format(binary)) + print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') + raise SystemExit(1) + + try: + version_str = run([binary, "--version"], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode() + return version_str.split('\n')[0].strip() + except Exception: + print('{red}[X] Unable to find a working version of {cmd}, is it installed and in your $PATH?'.format(cmd=binary, **ANSI)) + raise SystemExit(1) + +def find_chrome_binary() -> Optional[str]: + """find any installed chrome binaries in the default locations""" + # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev + default_executable_paths = ( + 'chromium-browser', + 'chromium', + '/Applications/Chromium.app/Contents/MacOS/Chromium', + 'google-chrome', + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + 'google-chrome-stable', + 'google-chrome-beta', + 'google-chrome-canary', + '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', + 'google-chrome-unstable', + 'google-chrome-dev', + ) + for name in default_executable_paths: + full_path_exists = shutil.which(name) + if full_path_exists: + return name + + print('{red}[X] Unable to find a working version of Chrome/Chromium, is it installed and in your $PATH?'.format(**ANSI)) + raise SystemExit(1) + +def find_chrome_data_dir() -> Optional[str]: + """find any installed chrome user data directories in the default locations""" + # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev + default_profile_paths = ( + '~/.config/chromium', + '~/Library/Application Support/Chromium', + '~/AppData/Local/Chromium/User Data', + '~/.config/google-chrome', + '~/Library/Application Support/Google/Chrome', + '~/AppData/Local/Google/Chrome/User Data', + '~/.config/google-chrome-stable', + '~/.config/google-chrome-beta', + '~/Library/Application Support/Google/Chrome Canary', + '~/AppData/Local/Google/Chrome SxS/User Data', + '~/.config/google-chrome-unstable', + '~/.config/google-chrome-dev', + ) + for path in default_profile_paths: + full_path = os.path.expanduser(path) + if os.path.exists(full_path): + return full_path + return None + +def get_git_version() -> str: + """get the git commit hash of the python code folder (aka code version)""" + try: + return run([GIT_BINARY, 'rev-list', '-1', 'HEAD', './'], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode() + except Exception: + print('[!] Warning: unable to determine git version, is git installed and in your $PATH?') + return 'unknown' -########################### Environment & Dependencies ######################### try: + GIT_SHA = get_git_version() + ### Terminal Configuration TERM_WIDTH = lambda: shutil.get_terminal_size((100, 10)).columns ANSI = { @@ -93,66 +171,6 @@ try: # dont show colors if USE_COLOR is False ANSI = {k: '' for k in ANSI.keys()} - - if not CHROME_BINARY: - # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev - default_executable_paths = ( - 'chromium-browser', - 'chromium', - '/Applications/Chromium.app/Contents/MacOS/Chromium', - 'google-chrome', - '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', - 'google-chrome-stable', - 'google-chrome-beta', - 'google-chrome-canary', - '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', - 'google-chrome-unstable', - 'google-chrome-dev', - ) - for name in default_executable_paths: - full_path_exists = shutil.which(name) - if full_path_exists: - CHROME_BINARY = name - break - else: - CHROME_BINARY = 'chromium-browser' - # print('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY)) - - if CHROME_USER_DATA_DIR is None: - # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev - default_profile_paths = ( - '~/.config/chromium', - '~/Library/Application Support/Chromium', - '~/AppData/Local/Chromium/User Data', - '~/.config/google-chrome', - '~/Library/Application Support/Google/Chrome', - '~/AppData/Local/Google/Chrome/User Data', - '~/.config/google-chrome-stable', - '~/.config/google-chrome-beta', - '~/Library/Application Support/Google/Chrome Canary', - '~/AppData/Local/Google/Chrome SxS/User Data', - '~/.config/google-chrome-unstable', - '~/.config/google-chrome-dev', - ) - for path in default_profile_paths: - full_path = os.path.expanduser(path) - if os.path.exists(full_path): - CHROME_USER_DATA_DIR = full_path - break - # print('[i] Using Chrome data dir: {}'.format(os.path.abspath(CHROME_USER_DATA_DIR))) - - CHROME_OPTIONS = { - 'TIMEOUT': TIMEOUT, - 'RESOLUTION': RESOLUTION, - 'CHECK_SSL_VALIDITY': CHECK_SSL_VALIDITY, - 'CHROME_BINARY': CHROME_BINARY, - 'CHROME_HEADLESS': CHROME_HEADLESS, - 'CHROME_SANDBOX': CHROME_SANDBOX, - 'CHROME_USER_AGENT': CHROME_USER_AGENT, - 'CHROME_USER_DATA_DIR': CHROME_USER_DATA_DIR, - } - - ### Check Python environment python_vers = float('{}.{}'.format(sys.version_info.major, sys.version_info.minor)) if python_vers < 3.5: @@ -170,98 +188,58 @@ try: print(' Alternatively, run this script with:') print(' env PYTHONIOENCODING=UTF-8 ./archive.py export.html') - ### Get code version by parsing git log - GIT_SHA = 'unknown' - try: - GIT_SHA = run([GIT_BINARY, 'rev-list', '-1', 'HEAD', './'], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode() - except Exception: - print('[!] Warning: unable to determine git version, is git installed and in your $PATH?') - - ### Get absolute path for cookies file - try: - COOKIES_FILE = os.path.abspath(COOKIES_FILE) if COOKIES_FILE else None - except Exception: - print('[!] Warning: unable to get full path to COOKIES_FILE, are you sure you specified it correctly?') - raise ### Make sure curl is installed - if FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG: - if run(['which', CURL_BINARY], stdout=DEVNULL, stderr=DEVNULL).returncode or run([CURL_BINARY, '--version'], stdout=DEVNULL, stderr=DEVNULL).returncode: - print('{red}[X] Missing dependency: curl{reset}'.format(**ANSI)) - print(' Install it, then confirm it works with: {} --version'.format(CURL_BINARY)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) + USE_CURL = FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG + CURL_VERSION = USE_CURL and check_version(CURL_BINARY) ### Make sure wget is installed and calculate version - if FETCH_WGET or FETCH_WARC: - if run(['which', WGET_BINARY], stdout=DEVNULL, stderr=DEVNULL).returncode or run([WGET_BINARY, '--version'], stdout=DEVNULL, stderr=DEVNULL).returncode: - print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI)) - print(' Install it, then confirm it works with: {} --version'.format(WGET_BINARY)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) - - WGET_VERSION = 'unknown' - try: - wget_vers_str = run([WGET_BINARY, "--version"], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode() - WGET_VERSION = wget_vers_str.split('\n')[0].split(' ')[2] - except Exception: - if USE_WGET: - print('[!] Warning: unable to determine wget version, is wget installed and in your $PATH?') - - WGET_USER_AGENT = WGET_USER_AGENT.format(GIT_SHA=GIT_SHA[:9], WGET_VERSION=WGET_VERSION) - + USE_WGET = FETCH_WGET or FETCH_WARC + WGET_VERSION = USE_WGET and check_version(WGET_BINARY) + WGET_USER_AGENT = WGET_USER_AGENT.format( + GIT_SHA=GIT_SHA[:9], + WGET_VERSION=WGET_VERSION or '', + ) + ### Make sure chrome is installed and calculate version - if FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM: - if run(['which', CHROME_BINARY], stdout=DEVNULL, stderr=DEVNULL).returncode: - print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset'])) - print(' Install it, then confirm it works with: {} --version'.format(CHROME_BINARY)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) + USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM + CHROME_VERSION = None + if USE_CHROME: + if CHROME_BINARY is None: + CHROME_BINARY = find_chrome_binary() + CHROME_VERSION = check_version(CHROME_BINARY) + # print('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY)) - # parse chrome --version e.g. Google Chrome 61.0.3114.0 canary / Chromium 59.0.3029.110 built on Ubuntu, running on Ubuntu 16.04 - try: - result = run([CHROME_BINARY, '--version'], stdout=PIPE) - version_str = result.stdout.decode('utf-8') - version_lines = re.sub("(Google Chrome|Chromium) (\\d+?)\\.(\\d+?)\\.(\\d+?).*?$", "\\2", version_str).split('\n') - version = [l for l in version_lines if l.isdigit()][-1] - if int(version) < 59: - print(version_lines) - print('{red}[X] Chrome version must be 59 or greater for headless PDF, screenshot, and DOM saving{reset}'.format(**ANSI)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) - except (IndexError, TypeError, OSError): - print('{red}[X] Failed to parse Chrome version, is it installed properly?{reset}'.format(**ANSI)) - print(' Install it, then confirm it works with: {} --version'.format(CHROME_BINARY)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) - - CHROME_VERSION = 'unknown' - try: - chrome_vers_str = run([CHROME_BINARY, "--version"], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode() - CHROME_VERSION = [v for v in chrome_vers_str.strip().split(' ') if v.replace('.', '').isdigit()][0] - except Exception: - if USE_CHROME: - print('[!] Warning: unable to determine chrome version, is chrome installed and in your $PATH?') + if CHROME_USER_DATA_DIR is None: + CHROME_USER_DATA_DIR = find_chrome_data_dir() + # print('[i] Using Chrome data dir: {}'.format(os.path.abspath(CHROME_USER_DATA_DIR))) ### Make sure git is installed - if FETCH_GIT: - if run(['which', GIT_BINARY], stdout=DEVNULL, stderr=DEVNULL).returncode or run([GIT_BINARY, '--version'], stdout=DEVNULL, stderr=DEVNULL).returncode: - print('{red}[X] Missing dependency: git{reset}'.format(**ANSI)) - print(' Install it, then confirm it works with: {} --version'.format(GIT_BINARY)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) + GIT_VERSION = FETCH_GIT and check_version(GIT_BINARY) ### Make sure youtube-dl is installed - if FETCH_MEDIA: - if run(['which', YOUTUBEDL_BINARY], stdout=DEVNULL, stderr=DEVNULL).returncode or run([YOUTUBEDL_BINARY, '--version'], stdout=DEVNULL, stderr=DEVNULL).returncode: - print('{red}[X] Missing dependency: youtube-dl{reset}'.format(**ANSI)) - print(' Install it, then confirm it was installed with: {} --version'.format(YOUTUBEDL_BINARY)) - print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') - raise SystemExit(1) + YOUTUBEDL_VERSION = FETCH_MEDIA and check_version(YOUTUBEDL_BINARY) + + ### Chrome housekeeping options + CHROME_OPTIONS = { + 'TIMEOUT': TIMEOUT, + 'RESOLUTION': RESOLUTION, + 'CHECK_SSL_VALIDITY': CHECK_SSL_VALIDITY, + 'CHROME_BINARY': CHROME_BINARY, + 'CHROME_HEADLESS': CHROME_HEADLESS, + 'CHROME_SANDBOX': CHROME_SANDBOX, + 'CHROME_USER_AGENT': CHROME_USER_AGENT, + 'CHROME_USER_DATA_DIR': CHROME_USER_DATA_DIR, + } + # PYPPETEER_ARGS = { + # 'headless': CHROME_HEADLESS, + # 'ignoreHTTPSErrors': not CHECK_SSL_VALIDITY, + # # 'executablePath': CHROME_BINARY, + # } except KeyboardInterrupt: raise SystemExit(1) except: - print('[X] There was an error during the startup procedure, your archive data is unaffected.') + print('[X] There was an error while reading configuration. Your archive data is unaffected.') raise From 0a44779b2157ed80f24c7f833dfe051b556b4ae9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 05:32:16 -0400 Subject: [PATCH 004/365] save command versions in archive results --- archivebox/archive_methods.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index f0223bbb..5f6f0e78 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -35,6 +35,11 @@ from config import ( WGET_USER_AGENT, CHECK_SSL_VALIDITY, COOKIES_FILE, + CURL_VERSION, + WGET_VERSION, + CHROME_VERSION, + GIT_VERSION, + YOUTUBEDL_VERSION, ) from util import ( domain, @@ -42,7 +47,6 @@ from util import ( without_query, without_fragment, fetch_page_title, - read_js_script, is_static_file, TimedProgress, chmod_file, @@ -59,19 +63,7 @@ from logs import ( log_archive_method_finished, ) -class ArchiveError(Exception): - def __init__(self, message, hints=None): - super().__init__(message) - self.hints = hints -class ArchiveResult(NamedTuple): - cmd: List[str] - pwd: str - output: Union[str, Exception, None] - status: str - start_ts: datetime - end_ts: datetime - duration: int def archive_link(link_dir: str, link: Link, page=None) -> Link: @@ -165,6 +157,7 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=CURL_VERSION, output=output, status=status, **timer.stats, @@ -203,6 +196,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=CURL_VERSION, output=output, status=status, **timer.stats, @@ -289,6 +283,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=WGET_VERSION, output=output, status=status, **timer.stats, @@ -332,6 +327,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=CHROME_VERSION, output=output, status=status, **timer.stats, @@ -374,6 +370,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=CHROME_VERSION, output=output, status=status, **timer.stats, @@ -418,6 +415,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=CHROME_VERSION, output=output, status=status, **timer.stats, @@ -475,6 +473,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=GIT_VERSION, output=output, status=status, **timer.stats, @@ -546,6 +545,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=YOUTUBEDL_VERSION, output=output, status=status, **timer.stats, @@ -611,6 +611,7 @@ def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveR return ArchiveResult( cmd=cmd, pwd=link_dir, + cmd_version=CURL_VERSION, output=output, status=status, **timer.stats, From 76abc58135f43e49f645e5f5dfa860f47d69134a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 05:33:34 -0400 Subject: [PATCH 005/365] switch to strict type hints with NamedTuples instead of dicts --- archivebox/archive.py | 15 +++--- archivebox/archive_methods.py | 6 +-- archivebox/index.py | 34 ++++++++---- archivebox/links.py | 22 ++++---- archivebox/logs.py | 99 +++++++++++++++++------------------ archivebox/parse.py | 20 +++---- archivebox/schema.py | 55 +++++++++++++++++++ archivebox/util.py | 48 ++++++++++++++--- 8 files changed, 201 insertions(+), 98 deletions(-) create mode 100644 archivebox/schema.py diff --git a/archivebox/archive.py b/archivebox/archive.py index 5c0d195d..46ada292 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -12,6 +12,9 @@ Usage & Documentation: import os import sys +from typing import List + +from schema import Link from links import links_after_timestamp from index import write_links_index, load_links_index from archive_methods import archive_link @@ -50,7 +53,7 @@ def print_help(): print(" ./archive 15109948213.123\n") -def main(*args): +def main(*args) -> List[Link]: if set(args).intersection(('-h', '--help', 'help')) or len(args) > 2: print_help() raise SystemExit(0) @@ -95,10 +98,10 @@ def main(*args): import_path = save_remote_source(import_path) ### Run the main archive update process - update_archive_data(import_path=import_path, resume=resume) + return update_archive_data(import_path=import_path, resume=resume) -def update_archive_data(import_path=None, resume=None): +def update_archive_data(import_path: str=None, resume: float=None) -> List[Link]: """The main ArchiveBox entrancepoint. Everything starts here.""" # Step 1: Load list of links from the existing index @@ -111,14 +114,14 @@ def update_archive_data(import_path=None, resume=None): # Step 3: Run the archive methods for each link links = new_links if ONLY_NEW else all_links log_archiving_started(len(links), resume) - idx, link = 0, 0 + idx, link = 0, {'timestamp': 0} try: for idx, link in enumerate(links_after_timestamp(links, resume)): link_dir = os.path.join(ARCHIVE_DIR, link['timestamp']) archive_link(link_dir, link) except KeyboardInterrupt: - log_archiving_paused(len(links), idx, link and link['timestamp']) + log_archiving_paused(len(links), idx, link['timestamp']) raise SystemExit(0) except: @@ -130,7 +133,7 @@ def update_archive_data(import_path=None, resume=None): # Step 4: Re-write links index with updated titles, icons, and resources all_links, _ = load_links_index(out_dir=OUTPUT_DIR) write_links_index(out_dir=OUTPUT_DIR, links=all_links, finished=True) - + return all_links if __name__ == '__main__': main(*sys.argv) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 5f6f0e78..e214a909 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -1,10 +1,10 @@ import os -import json -from typing import Union, Dict, List, Tuple, NamedTuple +from typing import Dict, List, Tuple from collections import defaultdict from datetime import datetime +from schema import Link, ArchiveResult, ArchiveError from index import ( write_link_index, patch_links_index, @@ -102,7 +102,7 @@ def archive_link(link_dir: str, link: Link, page=None) -> Link: link['history'][method_name].append(result._asdict()) stats[result.status] += 1 - log_archive_method_finished(result._asdict()) + log_archive_method_finished(result) else: stats['skipped'] += 1 diff --git a/archivebox/index.py b/archivebox/index.py index 503b82ad..3c31ac84 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -11,6 +11,7 @@ except ImportError: print('[X] Missing "distutils" python package. To install it, run:') print(' pip install distutils') +from schema import Link, ArchiveIndex from config import ( OUTPUT_DIR, TEMPLATES_DIR, @@ -25,7 +26,7 @@ from util import ( check_links_structure, wget_output_path, latest_output, - Link, + ExtendedEncoder, ) from parse import parse_links from links import validate_links @@ -56,6 +57,7 @@ def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> write_html_links_index(out_dir, links, finished=finished) log_indexing_finished(out_dir, 'index.html') + def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[List[Link], List[Link]]: """parse and load existing index with any new links from import_path merged in""" @@ -82,6 +84,7 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[Li return all_links, new_links + def write_json_links_index(out_dir: str, links: List[Link]) -> None: """write the json link index to a given path""" @@ -89,20 +92,24 @@ def write_json_links_index(out_dir: str, links: List[Link]) -> None: path = os.path.join(out_dir, 'index.json') - index_json = { - 'info': 'ArchiveBox Index', - 'help': 'https://github.com/pirate/ArchiveBox', - 'version': GIT_SHA, - 'num_links': len(links), - 'updated': str(datetime.now().timestamp()), - 'links': links, - } + index_json = ArchiveIndex( + info='ArchiveBox Index', + source='https://github.com/pirate/ArchiveBox', + docs='https://github.com/pirate/ArchiveBox/wiki', + version=GIT_SHA, + num_links=len(links), + updated=str(datetime.now().timestamp()), + links=links, + ) + + assert isinstance(index_json._asdict(), dict) with open(path, 'w', encoding='utf-8') as f: - json.dump(index_json, f, indent=4, default=str) + json.dump(index_json._asdict(), f, indent=4, cls=ExtendedEncoder) chmod_file(path) + def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> List[Link]: """parse a archive index json file and return the list of links""" index_path = os.path.join(out_dir, 'index.json') @@ -114,6 +121,7 @@ def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> List[Link]: return [] + def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: """write the html link index to a given path""" @@ -208,6 +216,7 @@ def write_link_index(out_dir: str, link: Link) -> None: write_json_link_index(out_dir, link) write_html_link_index(out_dir, link) + def write_json_link_index(out_dir: str, link: Link) -> None: """write a json file with some info about the link""" @@ -215,10 +224,11 @@ def write_json_link_index(out_dir: str, link: Link) -> None: path = os.path.join(out_dir, 'index.json') with open(path, 'w', encoding='utf-8') as f: - json.dump(link, f, indent=4, default=str) + json.dump(link, f, indent=4, cls=ExtendedEncoder) chmod_file(path) + def parse_json_link_index(out_dir: str) -> dict: """load the json link index from a given directory""" existing_index = os.path.join(out_dir, 'index.json') @@ -229,6 +239,7 @@ def parse_json_link_index(out_dir: str) -> dict: return link_json return {} + def load_json_link_index(out_dir: str, link: Link) -> Link: """check for an existing link archive in the given directory, and load+merge it into the given link dict @@ -244,6 +255,7 @@ def load_json_link_index(out_dir: str, link: Link) -> Link: check_link_structure(link) return link + def write_html_link_index(out_dir: str, link: Link) -> None: check_link_structure(link) with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: diff --git a/archivebox/links.py b/archivebox/links.py index ba8057a5..41aceebc 100644 --- a/archivebox/links.py +++ b/archivebox/links.py @@ -19,17 +19,19 @@ Link { } """ -from html import unescape +from typing import List, Iterable from collections import OrderedDict +from schema import Link from util import ( merge_links, check_link_structure, check_links_structure, + htmldecode, ) -def validate_links(links): +def validate_links(links: Iterable[Link]) -> List[Link]: check_links_structure(links) links = archivable_links(links) # remove chrome://, about:, mailto: etc. links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls @@ -40,13 +42,13 @@ def validate_links(links): raise SystemExit(1) for link in links: - link['title'] = unescape(link['title'].strip()) if link['title'] else None + link['title'] = htmldecode(link['title'].strip()) if link['title'] else None check_link_structure(link) return list(links) -def archivable_links(links): +def archivable_links(links: Iterable[Link]) -> Iterable[Link]: """remove chrome://, about:// or other schemed links that cant be archived""" return ( link @@ -55,12 +57,12 @@ def archivable_links(links): ) -def uniquefied_links(sorted_links): +def uniquefied_links(sorted_links: Iterable[Link]) -> Iterable[Link]: """ ensures that all non-duplicate links have monotonically increasing timestamps """ - unique_urls = OrderedDict() + unique_urls: OrderedDict[str, Link] = OrderedDict() lower = lambda url: url.lower().strip() without_www = lambda url: url.replace('://www.', '://', 1) @@ -73,7 +75,7 @@ def uniquefied_links(sorted_links): link = merge_links(unique_urls[fuzzy_url], link) unique_urls[fuzzy_url] = link - unique_timestamps = OrderedDict() + unique_timestamps: OrderedDict[str, Link] = OrderedDict() for link in unique_urls.values(): link['timestamp'] = lowest_uniq_timestamp(unique_timestamps, link['timestamp']) unique_timestamps[link['timestamp']] = link @@ -81,12 +83,12 @@ def uniquefied_links(sorted_links): return unique_timestamps.values() -def sorted_links(links): +def sorted_links(links: Iterable[Link]) -> Iterable[Link]: sort_func = lambda link: (link['timestamp'].split('.', 1)[0], link['url']) return sorted(links, key=sort_func, reverse=True) -def links_after_timestamp(links, timestamp=None): +def links_after_timestamp(links: Iterable[Link], timestamp: str=None) -> Iterable[Link]: if not timestamp: yield from links return @@ -99,7 +101,7 @@ def links_after_timestamp(links, timestamp=None): print('Resume value and all timestamp values must be valid numbers.') -def lowest_uniq_timestamp(used_timestamps, timestamp): +def lowest_uniq_timestamp(used_timestamps: OrderedDict, timestamp: str) -> str: """resolve duplicate timestamps by appending a decimal 1234, 1234 -> 1234.1, 1234.2""" timestamp = timestamp.split('.')[0] diff --git a/archivebox/logs.py b/archivebox/logs.py index 4dc2c051..769257a6 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -1,43 +1,44 @@ import sys from datetime import datetime + +from schema import Link, ArchiveResult, RuntimeStats from config import ANSI, REPO_DIR, OUTPUT_DIR - # globals are bad, mmkay -_LAST_RUN_STATS = { - 'skipped': 0, - 'succeeded': 0, - 'failed': 0, +_LAST_RUN_STATS = RuntimeStats( + skipped=0, + succeeded=0, + failed=0, - 'parsing_start_ts': 0, - 'parsing_end_ts': 0, + parse_start_ts=0, + parse_end_ts=0, - 'indexing_start_ts': 0, - 'indexing_end_ts': 0, + index_start_ts=0, + index_end_ts=0, - 'archiving_start_ts': 0, - 'archiving_end_ts': 0, + archiving_start_ts=0, + archiving_end_ts=0, +) - 'links': {}, -} - -def pretty_path(path): +def pretty_path(path: str) -> str: """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc""" return path.replace(REPO_DIR + '/', '') ### Parsing Stage -def log_parsing_started(source_file): +def log_parsing_started(source_file: str): start_ts = datetime.now() - _LAST_RUN_STATS['parse_start_ts'] = start_ts + _LAST_RUN_STATS.parse_start_ts = start_ts print('{green}[*] [{}] Parsing new links from output/sources/{}...{reset}'.format( start_ts.strftime('%Y-%m-%d %H:%M:%S'), source_file.rsplit('/', 1)[-1], **ANSI, )) -def log_parsing_finished(num_new_links, parser_name): +def log_parsing_finished(num_new_links: int, parser_name: str): + end_ts = datetime.now() + _LAST_RUN_STATS.parse_end_ts = end_ts print(' > Adding {} new links to index (parsed import as {})'.format( num_new_links, parser_name, @@ -48,26 +49,26 @@ def log_parsing_finished(num_new_links, parser_name): def log_indexing_process_started(): start_ts = datetime.now() - _LAST_RUN_STATS['index_start_ts'] = start_ts + _LAST_RUN_STATS.index_start_ts = start_ts print('{green}[*] [{}] Saving main index files...{reset}'.format( start_ts.strftime('%Y-%m-%d %H:%M:%S'), **ANSI, )) -def log_indexing_started(out_dir, out_file): +def log_indexing_started(out_dir: str, out_file: str): sys.stdout.write(' > {}/{}'.format(pretty_path(out_dir), out_file)) -def log_indexing_finished(out_dir, out_file): +def log_indexing_finished(out_dir: str, out_file: str): end_ts = datetime.now() - _LAST_RUN_STATS['index_end_ts'] = end_ts + _LAST_RUN_STATS.index_end_ts = end_ts print('\r √ {}/{}'.format(pretty_path(out_dir), out_file)) ### Archiving Stage -def log_archiving_started(num_links, resume): +def log_archiving_started(num_links: int, resume: float): start_ts = datetime.now() - _LAST_RUN_STATS['start_ts'] = start_ts + _LAST_RUN_STATS.archiving_start_ts = start_ts if resume: print('{green}[▶] [{}] Resuming archive updating for {} pages starting from {}...{reset}'.format( start_ts.strftime('%Y-%m-%d %H:%M:%S'), @@ -82,9 +83,9 @@ def log_archiving_started(num_links, resume): **ANSI, )) -def log_archiving_paused(num_links, idx, timestamp): +def log_archiving_paused(num_links: int, idx: int, timestamp: str): end_ts = datetime.now() - _LAST_RUN_STATS['end_ts'] = end_ts + _LAST_RUN_STATS.archiving_end_ts = end_ts print() print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format( **ANSI, @@ -100,10 +101,10 @@ def log_archiving_paused(num_links, idx, timestamp): timestamp, )) -def log_archiving_finished(num_links): +def log_archiving_finished(num_links: int): end_ts = datetime.now() - _LAST_RUN_STATS['end_ts'] = end_ts - seconds = end_ts.timestamp() - _LAST_RUN_STATS['start_ts'].timestamp() + _LAST_RUN_STATS.archiving_end_ts = end_ts + seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp() if seconds > 60: duration = '{0:.2f} min'.format(seconds / 60, 2) else: @@ -116,13 +117,13 @@ def log_archiving_finished(num_links): duration, ANSI['reset'], )) - print(' - {} links skipped'.format(_LAST_RUN_STATS['skipped'])) - print(' - {} links updated'.format(_LAST_RUN_STATS['succeeded'])) - print(' - {} links had errors'.format(_LAST_RUN_STATS['failed'])) + print(' - {} links skipped'.format(_LAST_RUN_STATS.skipped)) + print(' - {} links updated'.format(_LAST_RUN_STATS.succeeded)) + print(' - {} links had errors'.format(_LAST_RUN_STATS.failed)) print(' To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', ''))) -def log_link_archiving_started(link_dir, link, is_new): +def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford" # http://www.benstopford.com/2015/02/14/log-structured-merge-trees/ # > output/archive/1478739709 @@ -140,40 +141,34 @@ def log_link_archiving_started(link_dir, link, is_new): pretty_path(link_dir), )) -def log_link_archiving_finished(link_dir, link, is_new, stats): +def log_link_archiving_finished(link_dir: str, link: Link, is_new: bool, stats: dict): total = sum(stats.values()) if stats['failed'] > 0 : - _LAST_RUN_STATS['failed'] += 1 + _LAST_RUN_STATS.failed += 1 elif stats['skipped'] == total: - _LAST_RUN_STATS['skipped'] += 1 + _LAST_RUN_STATS.skipped += 1 else: - _LAST_RUN_STATS['succeeded'] += 1 + _LAST_RUN_STATS.succeeded += 1 -def log_archive_method_started(method): +def log_archive_method_started(method: str): print(' > {}'.format(method)) -def log_archive_method_finished(result): + +def log_archive_method_finished(result: ArchiveResult): """quote the argument with whitespace in a command so the user can copy-paste the outputted string directly to run the cmd """ - required_keys = ('cmd', 'pwd', 'output', 'status', 'start_ts', 'end_ts') - assert ( - isinstance(result, dict) - and all(key in result for key in required_keys) - and ('output' in result) - ), 'Archive method did not return a valid result.' - # Prettify CMD string and make it safe to copy-paste by quoting arguments quoted_cmd = ' '.join( '"{}"'.format(arg) if ' ' in arg else arg - for arg in result['cmd'] + for arg in result.cmd ) - if result['status'] == 'failed': + if result.status == 'failed': # Prettify error output hints string and limit to five lines - hints = getattr(result['output'], 'hints', None) or () + hints = getattr(result.output, 'hints', None) or () if hints: hints = hints if isinstance(hints, (list, tuple)) else hints.split('\n') hints = ( @@ -185,13 +180,13 @@ def log_archive_method_finished(result): output_lines = [ '{}Failed:{} {}{}'.format( ANSI['red'], - result['output'].__class__.__name__.replace('ArchiveError', ''), - result['output'], + result.output.__class__.__name__.replace('ArchiveError', ''), + result.output, ANSI['reset'] ), *hints, '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']), - ' cd {};'.format(result['pwd']), + *((' cd {};'.format(result.pwd),) if result.pwd else ()), ' {}'.format(quoted_cmd), ] print('\n'.join( diff --git a/archivebox/parse.py b/archivebox/parse.py index baaa447e..3da3cb35 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -20,6 +20,7 @@ Link: { import re import json +from typing import Tuple, List, IO, Iterable from datetime import datetime import xml.etree.ElementTree as etree @@ -29,10 +30,11 @@ from util import ( URL_REGEX, check_url_parsing_invariants, TimedProgress, + Link, ) -def parse_links(source_file): +def parse_links(source_file: str) -> Tuple[List[Link], str]: """parse a list of URLs with their metadata from an RSS feed, bookmarks export, or text file """ @@ -74,7 +76,7 @@ def parse_links(source_file): ### Import Parser Functions -def parse_pocket_html_export(html_file): +def parse_pocket_html_export(html_file: IO[str]) -> Iterable[Link]: """Parse Pocket-format bookmarks export files (produced by getpocket.com/export/)""" html_file.seek(0) @@ -98,7 +100,7 @@ def parse_pocket_html_export(html_file): } -def parse_json_export(json_file): +def parse_json_export(json_file: IO[str]) -> Iterable[Link]: """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)""" json_file.seek(0) @@ -150,7 +152,7 @@ def parse_json_export(json_file): } -def parse_rss_export(rss_file): +def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse RSS XML-format files into links""" rss_file.seek(0) @@ -187,7 +189,7 @@ def parse_rss_export(rss_file): } -def parse_shaarli_rss_export(rss_file): +def parse_shaarli_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Shaarli-specific RSS XML-format files into links""" rss_file.seek(0) @@ -224,7 +226,7 @@ def parse_shaarli_rss_export(rss_file): } -def parse_netscape_html_export(html_file): +def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: """Parse netscape-format bookmarks export files (produced by all browsers)""" html_file.seek(0) @@ -247,7 +249,7 @@ def parse_netscape_html_export(html_file): } -def parse_pinboard_rss_export(rss_file): +def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Pinboard RSS feed files into links""" rss_file.seek(0) @@ -278,7 +280,7 @@ def parse_pinboard_rss_export(rss_file): } -def parse_medium_rss_export(rss_file): +def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Medium RSS feed files into links""" rss_file.seek(0) @@ -299,7 +301,7 @@ def parse_medium_rss_export(rss_file): } -def parse_plain_text_export(text_file): +def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: """Parse raw links from each line in a text file""" text_file.seek(0) diff --git a/archivebox/schema.py b/archivebox/schema.py new file mode 100644 index 00000000..719298e8 --- /dev/null +++ b/archivebox/schema.py @@ -0,0 +1,55 @@ +from datetime import datetime + +from typing import List, Dict, Any, Optional, Union, NamedTuple +from recordclass import RecordClass + +Link = Dict[str, Any] + +class ArchiveIndex(NamedTuple): + info: str + version: str + source: str + docs: str + num_links: int + updated: str + links: List[Link] + +class ArchiveResult(NamedTuple): + cmd: List[str] + pwd: Optional[str] + cmd_version: Optional[str] + output: Union[str, Exception, None] + status: str + start_ts: datetime + end_ts: datetime + duration: int + + +class ArchiveError(Exception): + def __init__(self, message, hints=None): + super().__init__(message) + self.hints = hints + + +class LinkDict(NamedTuple): + timestamp: str + url: str + title: Optional[str] + tags: str + sources: List[str] + history: Dict[str, ArchiveResult] + + +class RuntimeStats(RecordClass): + skipped: int + succeeded: int + failed: int + + parse_start_ts: datetime + parse_end_ts: datetime + + index_start_ts: datetime + index_end_ts: datetime + + archiving_start_ts: datetime + archiving_end_ts: datetime diff --git a/archivebox/util.py b/archivebox/util.py index 1835bd16..2c2c6a05 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -3,11 +3,13 @@ import re import sys import time -from typing import List, Dict, Any, Optional, Union +from json import JSONEncoder + +from typing import List, Dict, Optional, Iterable from urllib.request import Request, urlopen -from urllib.parse import urlparse, quote -from decimal import Decimal +from urllib.parse import urlparse, quote, unquote +from html import escape, unescape from datetime import datetime from multiprocessing import Process from subprocess import ( @@ -19,6 +21,7 @@ from subprocess import ( CalledProcessError, ) +from schema import Link from config import ( ANSI, TERM_WIDTH, @@ -38,7 +41,8 @@ from logs import pretty_path ### Parsing Helpers -# Url Parsing: https://docs.python.org/3/library/urllib.parse.html#url-parsing +# All of these are (str) -> str +# shortcuts to: https://docs.python.org/3/library/urllib.parse.html#url-parsing scheme = lambda url: urlparse(url).scheme without_scheme = lambda url: urlparse(url)._replace(scheme='').geturl().strip('//') without_query = lambda url: urlparse(url)._replace(query='').geturl().strip('//') @@ -54,6 +58,9 @@ base_url = lambda url: without_scheme(url) # uniq base url used to dedupe links short_ts = lambda ts: ts.split('.')[0] urlencode = lambda s: quote(s, encoding='utf-8', errors='replace') +urldecode = lambda s: unquote(s) +htmlencode = lambda s: escape(s, quote=True) +htmldecode = lambda s: unescape(s) URL_REGEX = re.compile( r'http[s]?://' # start matching from allowed schemes @@ -89,7 +96,7 @@ STATICFILE_EXTENSIONS = { # html, htm, shtml, xhtml, xml, aspx, php, cgi } -Link = Dict[str, Any] + ### Checks & Tests @@ -105,7 +112,7 @@ def check_link_structure(link: Link) -> None: assert isinstance(key, str) assert isinstance(val, list), 'history must be a Dict[str, List], got: {}'.format(link['history']) -def check_links_structure(links: List[Link]) -> None: +def check_links_structure(links: Iterable[Link]) -> None: """basic sanity check invariants to make sure the data is valid""" assert isinstance(links, list) if links: @@ -334,7 +341,7 @@ def derived_link_info(link: Link) -> dict: url = link['url'] - to_date_str = lambda ts: datetime.fromtimestamp(Decimal(ts)).strftime('%Y-%m-%d %H:%M') + to_date_str = lambda ts: datetime.fromtimestamp(float(ts)).strftime('%Y-%m-%d %H:%M') extended_info = { **link, @@ -582,3 +589,30 @@ def chrome_args(**options) -> List[str]: cmd_args.append('--user-data-dir={}'.format(options['CHROME_USER_DATA_DIR'])) return cmd_args + + +class ExtendedEncoder(JSONEncoder): + """ + Extended json serializer that supports serializing several model + fields and objects + """ + + def default(self, obj): + cls_name = obj.__class__.__name__ + + if hasattr(obj, '_asdict'): + return obj._asdict() + + elif isinstance(obj, bytes): + return obj.decode() + + elif isinstance(obj, datetime): + return obj.isoformat() + + elif isinstance(obj, Exception): + return '{}: {}'.format(obj.__class__.__name__, obj) + + elif cls_name in ('dict_items', 'dict_keys', 'dict_values'): + return tuple(obj) + + return JSONEncoder.default(self, obj) From 346811fb780747bca9da8d5881fb6aa7dbbcb06c Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 05:35:20 -0400 Subject: [PATCH 006/365] ignore mypy cache --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5a6fcf3d..1dcc07e1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ # python __pycache__/ +.mypy_cache/ venv .venv archivebox/.venv From 25a107df4353aaef66c713840e80aaa6b0c64f30 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 26 Mar 2019 19:21:34 -0400 Subject: [PATCH 007/365] switch to dataclasses, working Link type hints everywhere --- archivebox/archive.py | 15 +- archivebox/archive_methods.py | 79 +++++---- archivebox/config.py | 38 +++-- archivebox/index.py | 124 +++++++------- archivebox/links.py | 50 +++--- archivebox/logs.py | 7 +- archivebox/parse.py | 122 +++++++------- archivebox/schema.py | 250 ++++++++++++++++++++++++---- archivebox/templates/index_row.html | 6 +- archivebox/util.py | 176 +++++++------------- 10 files changed, 504 insertions(+), 363 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index 46ada292..c6e10bd2 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -12,14 +12,13 @@ Usage & Documentation: import os import sys -from typing import List +from typing import List, Optional from schema import Link from links import links_after_timestamp from index import write_links_index, load_links_index from archive_methods import archive_link from config import ( - ARCHIVE_DIR, ONLY_NEW, OUTPUT_DIR, GIT_SHA, @@ -109,19 +108,19 @@ def update_archive_data(import_path: str=None, resume: float=None) -> List[Link] all_links, new_links = load_links_index(out_dir=OUTPUT_DIR, import_path=import_path) # Step 2: Write updated index with deduped old and new links back to disk - write_links_index(out_dir=OUTPUT_DIR, links=all_links) + write_links_index(out_dir=OUTPUT_DIR, links=list(all_links)) # Step 3: Run the archive methods for each link links = new_links if ONLY_NEW else all_links log_archiving_started(len(links), resume) - idx, link = 0, {'timestamp': 0} + idx: int = 0 + link: Optional[Link] = None try: for idx, link in enumerate(links_after_timestamp(links, resume)): - link_dir = os.path.join(ARCHIVE_DIR, link['timestamp']) - archive_link(link_dir, link) + archive_link(link) except KeyboardInterrupt: - log_archiving_paused(len(links), idx, link['timestamp']) + log_archiving_paused(len(links), idx, link.timestamp if link else '0') raise SystemExit(0) except: @@ -132,7 +131,7 @@ def update_archive_data(import_path: str=None, resume: float=None) -> List[Link] # Step 4: Re-write links index with updated titles, icons, and resources all_links, _ = load_links_index(out_dir=OUTPUT_DIR) - write_links_index(out_dir=OUTPUT_DIR, links=all_links, finished=True) + write_links_index(out_dir=OUTPUT_DIR, links=list(all_links), finished=True) return all_links if __name__ == '__main__': diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index e214a909..76153e70 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -52,7 +52,6 @@ from util import ( chmod_file, wget_output_path, chrome_args, - check_link_structure, run, PIPE, DEVNULL, Link, ) @@ -64,9 +63,7 @@ from logs import ( ) - - -def archive_link(link_dir: str, link: Link, page=None) -> Link: +def archive_link(link: Link, page=None) -> Link: """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" ARCHIVE_METHODS = ( @@ -82,24 +79,24 @@ def archive_link(link_dir: str, link: Link, page=None) -> Link: ) try: - is_new = not os.path.exists(link_dir) + is_new = not os.path.exists(link.link_dir) if is_new: - os.makedirs(link_dir) + os.makedirs(link.link_dir) - link = load_json_link_index(link_dir, link) - log_link_archiving_started(link_dir, link, is_new) + link = load_json_link_index(link.link_dir, link) + log_link_archiving_started(link.link_dir, link, is_new) stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} for method_name, should_run, method_function in ARCHIVE_METHODS: - if method_name not in link['history']: - link['history'][method_name] = [] + if method_name not in link.history: + link.history[method_name] = [] - if should_run(link_dir, link): + if should_run(link.link_dir, link): log_archive_method_started(method_name) - result = method_function(link_dir, link) + result = method_function(link.link_dir, link) - link['history'][method_name].append(result._asdict()) + link.history[method_name].append(result) stats[result.status] += 1 log_archive_method_finished(result) @@ -108,14 +105,22 @@ def archive_link(link_dir: str, link: Link, page=None) -> Link: # print(' ', stats) - write_link_index(link_dir, link) + link = Link(**{ + **link._asdict(), + 'updated': datetime.now(), + }) + + write_link_index(link.link_dir, link) patch_links_index(link) - log_link_archiving_finished(link_dir, link, is_new, stats) + log_link_archiving_finished(link.link_dir, link, is_new, stats) + + except KeyboardInterrupt: + raise except Exception as err: print(' ! Failed to archive link: {}: {}'.format(err.__class__.__name__, err)) raise - + return link @@ -123,10 +128,10 @@ def archive_link(link_dir: str, link: Link, page=None) -> Link: def should_fetch_title(link_dir: str, link: Link) -> bool: # if link already has valid title, skip it - if link['title'] and not link['title'].lower().startswith('http'): + if link.title and not link.title.lower().startswith('http'): return False - if is_static_file(link['url']): + if is_static_file(link.url): return False return FETCH_TITLE @@ -137,7 +142,7 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul output = None cmd = [ CURL_BINARY, - link['url'], + link.url, '|', 'grep', '', @@ -145,7 +150,7 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') try: - output = fetch_page_title(link['url'], timeout=timeout, progress=False) + output = fetch_page_title(link.url, timeout=timeout, progress=False) if not output: raise ArchiveError('Unable to detect page title') except Exception as err: @@ -180,7 +185,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes '--location', '--output', output, *(() if CHECK_SSL_VALIDITY else ('--insecure',)), - 'https://www.google.com/s2/favicons?domain={}'.format(domain(link['url'])), + 'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)), ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -240,7 +245,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult *(('--user-agent={}'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()), *(('--load-cookies', COOKIES_FILE) if COOKIES_FILE else ()), *((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', '--no-hsts'))), - link['url'], + link.url, ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -290,7 +295,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult ) def should_fetch_pdf(link_dir: str, link: Link) -> bool: - if is_static_file(link['url']): + if is_static_file(link.url): return False if os.path.exists(os.path.join(link_dir, 'output.pdf')): @@ -306,7 +311,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: cmd = [ *chrome_args(TIMEOUT=timeout), '--print-to-pdf', - link['url'], + link.url, ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -334,7 +339,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) def should_fetch_screenshot(link_dir: str, link: Link) -> bool: - if is_static_file(link['url']): + if is_static_file(link.url): return False if os.path.exists(os.path.join(link_dir, 'screenshot.png')): @@ -349,7 +354,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive cmd = [ *chrome_args(TIMEOUT=timeout), '--screenshot', - link['url'], + link.url, ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -377,7 +382,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive ) def should_fetch_dom(link_dir: str, link: Link) -> bool: - if is_static_file(link['url']): + if is_static_file(link.url): return False if os.path.exists(os.path.join(link_dir, 'output.html')): @@ -393,7 +398,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: cmd = [ *chrome_args(TIMEOUT=timeout), '--dump-dom', - link['url'] + link.url ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -422,15 +427,15 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) def should_fetch_git(link_dir: str, link: Link) -> bool: - if is_static_file(link['url']): + if is_static_file(link.url): return False if os.path.exists(os.path.join(link_dir, 'git')): return False is_clonable_url = ( - (domain(link['url']) in GIT_DOMAINS) - or (extension(link['url']) == 'git') + (domain(link.url) in GIT_DOMAINS) + or (extension(link.url) == 'git') ) if not is_clonable_url: return False @@ -450,7 +455,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: '--mirror', '--recursive', *(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')), - without_query(without_fragment(link['url'])), + without_query(without_fragment(link.url)), ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -481,7 +486,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: def should_fetch_media(link_dir: str, link: Link) -> bool: - if is_static_file(link['url']): + if is_static_file(link.url): return False if os.path.exists(os.path.join(link_dir, 'media')): @@ -515,7 +520,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv '--embed-thumbnail', '--add-metadata', *(() if CHECK_SSL_VALIDITY else ('--no-check-certificate',)), - link['url'], + link.url, ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -553,7 +558,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: - if is_static_file(link['url']): + if is_static_file(link.url): return False if os.path.exists(os.path.join(link_dir, 'archive.org.txt')): @@ -567,7 +572,7 @@ def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveR output = 'archive.org.txt' archive_org_url = None - submit_url = 'https://web.archive.org/save/{}'.format(link['url']) + submit_url = 'https://web.archive.org/save/{}'.format(link.url) cmd = [ CURL_BINARY, '--location', @@ -586,7 +591,7 @@ def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveR archive_org_url = 'https://web.archive.org{}'.format(content_location[0]) elif len(errors) == 1 and 'RobotAccessControlException' in errors[0]: archive_org_url = None - # raise ArchiveError('Archive.org denied by {}/robots.txt'.format(domain(link['url']))) + # raise ArchiveError('Archive.org denied by {}/robots.txt'.format(domain(link.url))) elif errors: raise ArchiveError(', '.join(errors)) else: diff --git a/archivebox/config.py b/archivebox/config.py index 13d64c3a..ec38b367 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1,5 +1,4 @@ import os -import re import sys import shutil @@ -77,7 +76,7 @@ if COOKIES_FILE: COOKIES_FILE = os.path.abspath(COOKIES_FILE) # ****************************************************************************** -# ************************ Environment & Dependencies ************************** +# ***************************** Helper Functions ******************************* # ****************************************************************************** def check_version(binary: str) -> str: @@ -95,6 +94,7 @@ def check_version(binary: str) -> str: print('{red}[X] Unable to find a working version of {cmd}, is it installed and in your $PATH?'.format(cmd=binary, **ANSI)) raise SystemExit(1) + def find_chrome_binary() -> Optional[str]: """find any installed chrome binaries in the default locations""" # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev @@ -119,6 +119,7 @@ def find_chrome_binary() -> Optional[str]: print('{red}[X] Unable to find a working version of Chrome/Chromium, is it installed and in your $PATH?'.format(**ANSI)) raise SystemExit(1) + def find_chrome_data_dir() -> Optional[str]: """find any installed chrome user data directories in the default locations""" # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev @@ -142,6 +143,7 @@ def find_chrome_data_dir() -> Optional[str]: return full_path return None + def get_git_version() -> str: """get the git commit hash of the python code folder (aka code version)""" try: @@ -151,6 +153,10 @@ def get_git_version() -> str: return 'unknown' +# ****************************************************************************** +# ************************ Environment & Dependencies ************************** +# ****************************************************************************** + try: GIT_SHA = get_git_version() @@ -188,19 +194,33 @@ try: print(' Alternatively, run this script with:') print(' env PYTHONIOENCODING=UTF-8 ./archive.py export.html') - ### Make sure curl is installed USE_CURL = FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG - CURL_VERSION = USE_CURL and check_version(CURL_BINARY) + CURL_VERSION = None + if USE_CURL: + CURL_VERSION = check_version(CURL_BINARY) ### Make sure wget is installed and calculate version USE_WGET = FETCH_WGET or FETCH_WARC - WGET_VERSION = USE_WGET and check_version(WGET_BINARY) + WGET_VERSION = None + if USE_WGET: + WGET_VERSION = check_version(WGET_BINARY) + WGET_USER_AGENT = WGET_USER_AGENT.format( GIT_SHA=GIT_SHA[:9], WGET_VERSION=WGET_VERSION or '', ) + ### Make sure git is installed + GIT_VERSION = None + if FETCH_GIT: + GIT_VERSION = check_version(GIT_BINARY) + + ### Make sure youtube-dl is installed + YOUTUBEDL_VERSION = None + if FETCH_MEDIA: + check_version(YOUTUBEDL_BINARY) + ### Make sure chrome is installed and calculate version USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM CHROME_VERSION = None @@ -214,13 +234,6 @@ try: CHROME_USER_DATA_DIR = find_chrome_data_dir() # print('[i] Using Chrome data dir: {}'.format(os.path.abspath(CHROME_USER_DATA_DIR))) - ### Make sure git is installed - GIT_VERSION = FETCH_GIT and check_version(GIT_BINARY) - - ### Make sure youtube-dl is installed - YOUTUBEDL_VERSION = FETCH_MEDIA and check_version(YOUTUBEDL_BINARY) - - ### Chrome housekeeping options CHROME_OPTIONS = { 'TIMEOUT': TIMEOUT, 'RESOLUTION': RESOLUTION, @@ -236,7 +249,6 @@ try: # 'ignoreHTTPSErrors': not CHECK_SSL_VALIDITY, # # 'executablePath': CHROME_BINARY, # } - except KeyboardInterrupt: raise SystemExit(1) diff --git a/archivebox/index.py b/archivebox/index.py index 3c31ac84..0a60dd23 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -1,9 +1,10 @@ import os import json +from itertools import chain from datetime import datetime from string import Template -from typing import List, Tuple +from typing import List, Tuple, Iterator, Optional try: from distutils.dir_util import copy_tree @@ -11,7 +12,7 @@ except ImportError: print('[X] Missing "distutils" python package. To install it, run:') print(' pip install distutils') -from schema import Link, ArchiveIndex +from schema import Link, ArchiveIndex, ArchiveResult from config import ( OUTPUT_DIR, TEMPLATES_DIR, @@ -22,11 +23,10 @@ from util import ( chmod_file, urlencode, derived_link_info, + wget_output_path, + ExtendedEncoder, check_link_structure, check_links_structure, - wget_output_path, - latest_output, - ExtendedEncoder, ) from parse import parse_links from links import validate_links @@ -47,7 +47,6 @@ def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> """create index.html file for a given list of links""" log_indexing_process_started() - check_links_structure(links) log_indexing_started(out_dir, 'index.json') write_json_links_index(out_dir, links) @@ -63,20 +62,17 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[Li existing_links: List[Link] = [] if out_dir: - existing_links = parse_json_links_index(out_dir) - check_links_structure(existing_links) + existing_links = list(parse_json_links_index(out_dir)) new_links: List[Link] = [] if import_path: # parse and validate the import file log_parsing_started(import_path) raw_links, parser_name = parse_links(import_path) - new_links = validate_links(raw_links) - check_links_structure(new_links) + new_links = list(validate_links(raw_links)) # merge existing links in out_dir and new links - all_links = validate_links(existing_links + new_links) - check_links_structure(all_links) + all_links = list(validate_links(existing_links + new_links)) num_new_links = len(all_links) - len(existing_links) if import_path and parser_name: @@ -88,7 +84,15 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[Li def write_json_links_index(out_dir: str, links: List[Link]) -> None: """write the json link index to a given path""" - check_links_structure(links) + assert isinstance(links, List), 'Links must be a list, not a generator.' + assert isinstance(links[0].history, dict) + assert isinstance(links[0].sources, list) + + if links[0].history.get('title'): + assert isinstance(links[0].history['title'][0], ArchiveResult) + + if links[0].sources: + assert isinstance(links[0].sources[0], str) path = os.path.join(out_dir, 'index.json') @@ -98,7 +102,7 @@ def write_json_links_index(out_dir: str, links: List[Link]) -> None: docs='https://github.com/pirate/ArchiveBox/wiki', version=GIT_SHA, num_links=len(links), - updated=str(datetime.now().timestamp()), + updated=datetime.now(), links=links, ) @@ -110,23 +114,23 @@ def write_json_links_index(out_dir: str, links: List[Link]) -> None: chmod_file(path) -def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> List[Link]: +def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: """parse a archive index json file and return the list of links""" + index_path = os.path.join(out_dir, 'index.json') if os.path.exists(index_path): with open(index_path, 'r', encoding='utf-8') as f: links = json.load(f)['links'] check_links_structure(links) - return links + for link in links: + yield Link(**link) - return [] + return () def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: """write the html link index to a given path""" - check_links_structure(links) - path = os.path.join(out_dir, 'index.html') copy_tree(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static')) @@ -140,24 +144,22 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False with open(os.path.join(TEMPLATES_DIR, 'index_row.html'), 'r', encoding='utf-8') as f: link_row_html = f.read() - full_links_info = (derived_link_info(link) for link in links) - link_rows = '\n'.join( Template(link_row_html).substitute(**{ - **link, + **derived_link_info(link), 'title': ( - link['title'] - or (link['base_url'] if link['is_archived'] else TITLE_LOADING_MSG) + link.title + or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), 'favicon_url': ( - os.path.join('archive', link['timestamp'], 'favicon.ico') + os.path.join('archive', link.timestamp, 'favicon.ico') # if link['is_archived'] else 'data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=' ), 'archive_url': urlencode( wget_output_path(link) or 'index.html' ), }) - for link in full_links_info + for link in links ) template_vars = { @@ -180,28 +182,33 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: """hack to in-place update one row's info in the generated index html""" - title = link['title'] or latest_output(link)['title'] - successful = len(tuple(filter(None, latest_output(link).values()))) + title = link.title or link.latest_outputs()['title'] + successful = link.num_outputs # Patch JSON index changed = False json_file_links = parse_json_links_index(out_dir) + patched_links = [] for saved_link in json_file_links: - if saved_link['url'] == link['url']: - saved_link['title'] = title - saved_link['history'] = link['history'] - changed = True - break - if changed: - write_json_links_index(out_dir, json_file_links) + if saved_link.url == link.url: + patched_links.append(Link(**{ + **saved_link._asdict(), + 'title': title, + 'history': link.history, + 'updated': link.updated, + })) + else: + patched_links.append(saved_link) + + write_json_links_index(out_dir, patched_links) # Patch HTML index html_path = os.path.join(out_dir, 'index.html') html = open(html_path, 'r').read().split('\n') for idx, line in enumerate(html): - if title and ('<span data-title-for="{}"'.format(link['url']) in line): + if title and ('<span data-title-for="{}"'.format(link.url) in line): html[idx] = '<span>{}</span>'.format(title) - elif successful and ('<span data-number-for="{}"'.format(link['url']) in line): + elif successful and ('<span data-number-for="{}"'.format(link.url) in line): html[idx] = '<span>{}</span>'.format(successful) break @@ -212,7 +219,6 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: ### Individual link index def write_link_index(out_dir: str, link: Link) -> None: - link['updated'] = str(datetime.now().timestamp()) write_json_link_index(out_dir, link) write_html_link_index(out_dir, link) @@ -220,66 +226,58 @@ def write_link_index(out_dir: str, link: Link) -> None: def write_json_link_index(out_dir: str, link: Link) -> None: """write a json file with some info about the link""" - check_link_structure(link) path = os.path.join(out_dir, 'index.json') with open(path, 'w', encoding='utf-8') as f: - json.dump(link, f, indent=4, cls=ExtendedEncoder) + json.dump(link._asdict(), f, indent=4, cls=ExtendedEncoder) chmod_file(path) -def parse_json_link_index(out_dir: str) -> dict: +def parse_json_link_index(out_dir: str) -> Optional[Link]: """load the json link index from a given directory""" existing_index = os.path.join(out_dir, 'index.json') if os.path.exists(existing_index): with open(existing_index, 'r', encoding='utf-8') as f: link_json = json.load(f) check_link_structure(link_json) - return link_json - return {} + return Link(**link_json) + return None def load_json_link_index(out_dir: str, link: Link) -> Link: """check for an existing link archive in the given directory, and load+merge it into the given link dict """ - link = { - **parse_json_link_index(out_dir), - **link, - } - link.update({ - 'history': link.get('history') or {}, - }) - check_link_structure(link) - return link + existing_link = parse_json_link_index(out_dir) + existing_link = existing_link._asdict() if existing_link else {} + new_link = link._asdict() + + return Link(**{**existing_link, **new_link}) def write_html_link_index(out_dir: str, link: Link) -> None: - check_link_structure(link) with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: link_html = f.read() path = os.path.join(out_dir, 'index.html') - link = derived_link_info(link) - with open(path, 'w', encoding='utf-8') as f: f.write(Template(link_html).substitute({ - **link, + **derived_link_info(link), 'title': ( - link['title'] - or (link['base_url'] if link['is_archived'] else TITLE_LOADING_MSG) + link.title + or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), 'archive_url': urlencode( wget_output_path(link) - or (link['domain'] if link['is_archived'] else 'about:blank') + or (link.domain if link.is_archived else 'about:blank') ), - 'extension': link['extension'] or 'html', - 'tags': link['tags'].strip() or 'untagged', - 'status': 'Archived' if link['is_archived'] else 'Not yet archived', - 'status_color': 'success' if link['is_archived'] else 'danger', + 'extension': link.extension or 'html', + 'tags': link.tags or 'untagged', + 'status': 'Archived' if link.is_archived else 'Not yet archived', + 'status_color': 'success' if link.is_archived else 'danger', })) chmod_file(path) diff --git a/archivebox/links.py b/archivebox/links.py index 41aceebc..4692943c 100644 --- a/archivebox/links.py +++ b/archivebox/links.py @@ -11,7 +11,7 @@ Link { sources: [str], history: { pdf: [ - {start_ts, end_ts, duration, cmd, pwd, status, output}, + {start_ts, end_ts, cmd, pwd, cmd_version, status, output}, ... ], ... @@ -19,41 +19,36 @@ Link { } """ -from typing import List, Iterable +from typing import Iterable from collections import OrderedDict from schema import Link from util import ( + scheme, + fuzzy_url, merge_links, - check_link_structure, - check_links_structure, htmldecode, + hashurl, ) -def validate_links(links: Iterable[Link]) -> List[Link]: - check_links_structure(links) +def validate_links(links: Iterable[Link]) -> Iterable[Link]: links = archivable_links(links) # remove chrome://, about:, mailto: etc. - links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls links = sorted_links(links) # deterministically sort the links based on timstamp, url + links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls if not links: print('[X] No links found :(') raise SystemExit(1) - for link in links: - link['title'] = htmldecode(link['title'].strip()) if link['title'] else None - check_link_structure(link) - - return list(links) - + return links def archivable_links(links: Iterable[Link]) -> Iterable[Link]: """remove chrome://, about:// or other schemed links that cant be archived""" return ( link for link in links - if any(link['url'].lower().startswith(s) for s in ('http://', 'https://', 'ftp://')) + if scheme(link.url) in ('http', 'https', 'ftp') ) @@ -64,38 +59,37 @@ def uniquefied_links(sorted_links: Iterable[Link]) -> Iterable[Link]: unique_urls: OrderedDict[str, Link] = OrderedDict() - lower = lambda url: url.lower().strip() - without_www = lambda url: url.replace('://www.', '://', 1) - without_trailing_slash = lambda url: url[:-1] if url[-1] == '/' else url.replace('/?', '?') - for link in sorted_links: - fuzzy_url = without_www(without_trailing_slash(lower(link['url']))) - if fuzzy_url in unique_urls: + fuzzy = fuzzy_url(link.url) + if fuzzy in unique_urls: # merge with any other links that share the same url - link = merge_links(unique_urls[fuzzy_url], link) - unique_urls[fuzzy_url] = link + link = merge_links(unique_urls[fuzzy], link) + unique_urls[fuzzy] = link unique_timestamps: OrderedDict[str, Link] = OrderedDict() for link in unique_urls.values(): - link['timestamp'] = lowest_uniq_timestamp(unique_timestamps, link['timestamp']) - unique_timestamps[link['timestamp']] = link + new_link = Link(**{ + **link._asdict(), + 'timestamp': lowest_uniq_timestamp(unique_timestamps, link.timestamp), + }) + unique_timestamps[new_link.timestamp] = new_link return unique_timestamps.values() def sorted_links(links: Iterable[Link]) -> Iterable[Link]: - sort_func = lambda link: (link['timestamp'].split('.', 1)[0], link['url']) + sort_func = lambda link: (link.timestamp.split('.', 1)[0], link.url) return sorted(links, key=sort_func, reverse=True) -def links_after_timestamp(links: Iterable[Link], timestamp: str=None) -> Iterable[Link]: - if not timestamp: +def links_after_timestamp(links: Iterable[Link], resume: float=None) -> Iterable[Link]: + if not resume: yield from links return for link in links: try: - if float(link['timestamp']) <= float(timestamp): + if float(link.timestamp) <= resume: yield link except (ValueError, TypeError): print('Resume value and all timestamp values must be valid numbers.') diff --git a/archivebox/logs.py b/archivebox/logs.py index 769257a6..fd1f0bc5 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -1,6 +1,7 @@ import sys from datetime import datetime +from typing import Optional from schema import Link, ArchiveResult, RuntimeStats from config import ANSI, REPO_DIR, OUTPUT_DIR @@ -66,7 +67,7 @@ def log_indexing_finished(out_dir: str, out_file: str): ### Archiving Stage -def log_archiving_started(num_links: int, resume: float): +def log_archiving_started(num_links: int, resume: Optional[float]): start_ts = datetime.now() _LAST_RUN_STATS.archiving_start_ts = start_ts if resume: @@ -132,10 +133,10 @@ def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): symbol_color=ANSI['green' if is_new else 'black'], symbol='+' if is_new else '*', now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - title=link['title'] or link['url'], + title=link.title or link.base_url, **ANSI, )) - print(' {blue}{url}{reset}'.format(url=link['url'], **ANSI)) + print(' {blue}{url}{reset}'.format(url=link.url, **ANSI)) print(' {} {}'.format( '>' if is_new else '√', pretty_path(link_dir), diff --git a/archivebox/parse.py b/archivebox/parse.py index 3da3cb35..ba200ff3 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -26,6 +26,7 @@ import xml.etree.ElementTree as etree from config import TIMEOUT from util import ( + htmldecode, str_between, URL_REGEX, check_url_parsing_invariants, @@ -91,13 +92,13 @@ def parse_pocket_html_export(html_file: IO[str]) -> Iterable[Link]: tags = match.group(3) title = match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') - yield { - 'url': url, - 'timestamp': str(time.timestamp()), - 'title': title or None, - 'tags': tags or '', - 'sources': [html_file.name], - } + yield Link( + url=url, + timestamp=str(time.timestamp()), + title=title or None, + tags=tags or '', + sources=[html_file.name], + ) def parse_json_export(json_file: IO[str]) -> Iterable[Link]: @@ -137,19 +138,19 @@ def parse_json_export(json_file: IO[str]) -> Iterable[Link]: # Parse the title title = None if link.get('title'): - title = link['title'].strip() or None + title = link['title'].strip() elif link.get('description'): - title = link['description'].replace(' — Readability', '').strip() or None + title = link['description'].replace(' — Readability', '').strip() elif link.get('name'): - title = link['name'].strip() or None + title = link['name'].strip() - yield { - 'url': url, - 'timestamp': ts_str, - 'title': title, - 'tags': link.get('tags') or '', - 'sources': [json_file.name], - } + yield Link( + url=url, + timestamp=ts_str, + title=htmldecode(title) or None, + tags=link.get('tags') or '', + sources=[json_file.name], + ) def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: @@ -178,15 +179,15 @@ def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: url = str_between(get_row('link'), '<link>', '</link>') ts_str = str_between(get_row('pubDate'), '<pubDate>', '</pubDate>') time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z") - title = str_between(get_row('title'), '<![CDATA[', ']]').strip() or None + title = str_between(get_row('title'), '<![CDATA[', ']]').strip() - yield { - 'url': url, - 'timestamp': str(time.timestamp()), - 'title': title, - 'tags': '', - 'sources': [rss_file.name], - } + yield Link( + url=url, + timestamp=str(time.timestamp()), + title=htmldecode(title) or None, + tags='', + sources=[rss_file.name], + ) def parse_shaarli_rss_export(rss_file: IO[str]) -> Iterable[Link]: @@ -217,13 +218,13 @@ def parse_shaarli_rss_export(rss_file: IO[str]) -> Iterable[Link]: ts_str = str_between(get_row('published'), '<published>', '</published>') time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") - yield { - 'url': url, - 'timestamp': str(time.timestamp()), - 'title': title or None, - 'tags': '', - 'sources': [rss_file.name], - } + yield Link( + url=url, + timestamp=str(time.timestamp()), + title=htmldecode(title) or None, + tags='', + sources=[rss_file.name], + ) def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: @@ -239,14 +240,15 @@ def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: if match: url = match.group(1) time = datetime.fromtimestamp(float(match.group(2))) + title = match.group(3).strip() - yield { - 'url': url, - 'timestamp': str(time.timestamp()), - 'title': match.group(3).strip() or None, - 'tags': '', - 'sources': [html_file.name], - } + yield Link( + url=url, + timestamp=str(time.timestamp()), + title=htmldecode(title) or None, + tags='', + sources=[html_file.name], + ) def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: @@ -271,13 +273,13 @@ def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: else: time = datetime.now() - yield { - 'url': url, - 'timestamp': str(time.timestamp()), - 'title': title or None, - 'tags': tags or '', - 'sources': [rss_file.name], - } + yield Link( + url=url, + timestamp=str(time.timestamp()), + title=htmldecode(title) or None, + tags=tags or '', + sources=[rss_file.name], + ) def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: @@ -292,13 +294,13 @@ def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: ts_str = item.find("pubDate").text time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z") - yield { - 'url': url, - 'timestamp': str(time.timestamp()), - 'title': title or None, - 'tags': '', - 'sources': [rss_file.name], - } + yield Link( + url=url, + timestamp=str(time.timestamp()), + title=htmldecode(title) or None, + tags='', + sources=[rss_file.name], + ) def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: @@ -308,10 +310,10 @@ def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: for line in text_file.readlines(): urls = re.findall(URL_REGEX, line) if line.strip() else () for url in urls: - yield { - 'url': url, - 'timestamp': str(datetime.now().timestamp()), - 'title': None, - 'tags': '', - 'sources': [text_file.name], - } + yield Link( + url=url, + timestamp=str(datetime.now().timestamp()), + title=None, + tags='', + sources=[text_file.name], + ) diff --git a/archivebox/schema.py b/archivebox/schema.py index 719298e8..b92d1779 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -1,11 +1,223 @@ +import os + from datetime import datetime -from typing import List, Dict, Any, Optional, Union, NamedTuple -from recordclass import RecordClass +from typing import List, Dict, Any, Optional, Union -Link = Dict[str, Any] +from dataclasses import dataclass, asdict, field -class ArchiveIndex(NamedTuple): + +class ArchiveError(Exception): + def __init__(self, message, hints=None): + super().__init__(message) + self.hints = hints + +LinkDict = Dict[str, Any] + +@dataclass(frozen=True) +class ArchiveResult: + cmd: List[str] + pwd: Optional[str] + cmd_version: Optional[str] + output: Union[str, Exception, None] + status: str + start_ts: datetime + end_ts: datetime + + def _asdict(self): + return asdict(self) + + @property + def duration(self) -> int: + return (self.end_ts - self.start_ts).seconds + +@dataclass(frozen=True) +class Link: + timestamp: str + url: str + title: Optional[str] + tags: Optional[str] + sources: List[str] + history: Dict[str, List[ArchiveResult]] = field(default_factory=lambda: {}) + updated: Optional[str] = None + + def __hash__(self): + return self.urlhash + + def __eq__(self, other): + if not isinstance(other, Link): + return NotImplemented + return self.urlhash == other.urlhash + + def __gt__(self, other): + if not isinstance(other, Link): + return NotImplemented + if not self.timestamp or not other.timestamp: + return + return float(self.timestamp) > float(other.timestamp) + + def _asdict(self, extended=False): + info = { + 'url': self.url, + 'title': self.title or None, + 'timestamp': self.timestamp, + 'updated': self.updated or None, + 'tags': self.tags or None, + 'sources': self.sources or [], + 'history': self.history or {}, + } + if extended: + info.update({ + 'link_dir': self.link_dir, + 'archive_path': self.archive_path, + 'bookmarked_date': self.bookmarked_date, + 'updated_date': self.updated_date, + 'domain': self.domain, + 'path': self.path, + 'basename': self.basename, + 'extension': self.extension, + 'base_url': self.base_url, + 'is_static': self.is_static, + 'is_archived': self.is_archived, + 'num_outputs': self.num_outputs, + }) + return info + + @property + def link_dir(self) -> str: + from config import ARCHIVE_DIR + return os.path.join(ARCHIVE_DIR, self.timestamp) + + @property + def archive_path(self) -> str: + from config import ARCHIVE_DIR_NAME + return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp) + + ### URL Helpers + @property + def urlhash(self): + from util import hashurl + + return hashurl(self.url) + + @property + def extension(self) -> str: + from util import extension + return extension(self.url) + + @property + def domain(self) -> str: + from util import domain + return domain(self.url) + + @property + def path(self) -> str: + from util import path + return path(self.url) + + @property + def basename(self) -> str: + from util import basename + return basename(self.url) + + @property + def base_url(self) -> str: + from util import base_url + return base_url(self.url) + + ### Pretty Printing Helpers + @property + def bookmarked_date(self) -> Optional[str]: + from util import ts_to_date + return ts_to_date(self.timestamp) if self.timestamp else None + + @property + def updated_date(self) -> Optional[str]: + from util import ts_to_date + return ts_to_date(self.updated) if self.updated else None + + ### Archive Status Helpers + @property + def num_outputs(self) -> int: + return len(tuple(filter(None, self.latest_outputs().values()))) + + @property + def is_static(self) -> bool: + from util import is_static_file + return is_static_file(self.url) + + @property + def is_archived(self) -> bool: + from config import ARCHIVE_DIR + from util import domain + + return os.path.exists(os.path.join( + ARCHIVE_DIR, + self.timestamp, + domain(self.url), + )) + + def latest_outputs(self, status: str=None) -> Dict[str, Optional[str]]: + """get the latest output that each archive method produced for link""" + + latest = { + 'title': None, + 'favicon': None, + 'wget': None, + 'warc': None, + 'pdf': None, + 'screenshot': None, + 'dom': None, + 'git': None, + 'media': None, + 'archive_org': None, + } + for archive_method in latest.keys(): + # get most recent succesful result in history for each archive method + history = self.history.get(archive_method) or [] + history = filter(lambda result: result.output, reversed(history)) + if status is not None: + history = filter(lambda result: result.status == status, history) + + history = list(history) + if history: + latest[archive_method] = history[0].output + + return latest + + def canonical_outputs(self) -> Dict[str, Optional[str]]: + from util import wget_output_path + canonical = { + 'index_url': 'index.html', + 'favicon_url': 'favicon.ico', + 'google_favicon_url': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain), + 'archive_url': wget_output_path(self), + 'warc_url': 'warc', + 'pdf_url': 'output.pdf', + 'screenshot_url': 'screenshot.png', + 'dom_url': 'output.html', + 'archive_org_url': 'https://web.archive.org/web/{}'.format(self.base_url), + 'git_url': 'git', + 'media_url': 'media', + } + if self.is_static: + # static binary files like PDF and images are handled slightly differently. + # they're just downloaded once and aren't archived separately multiple times, + # so the wget, screenshot, & pdf urls should all point to the same file + + static_url = wget_output_path(self) + canonical.update({ + 'title': self.basename, + 'archive_url': static_url, + 'pdf_url': static_url, + 'screenshot_url': static_url, + 'dom_url': static_url, + }) + return canonical + + +@dataclass(frozen=True) +class ArchiveIndex: info: str version: str source: str @@ -14,33 +226,11 @@ class ArchiveIndex(NamedTuple): updated: str links: List[Link] -class ArchiveResult(NamedTuple): - cmd: List[str] - pwd: Optional[str] - cmd_version: Optional[str] - output: Union[str, Exception, None] - status: str - start_ts: datetime - end_ts: datetime - duration: int + def _asdict(self): + return asdict(self) - -class ArchiveError(Exception): - def __init__(self, message, hints=None): - super().__init__(message) - self.hints = hints - - -class LinkDict(NamedTuple): - timestamp: str - url: str - title: Optional[str] - tags: str - sources: List[str] - history: Dict[str, ArchiveResult] - - -class RuntimeStats(RecordClass): +@dataclass +class RuntimeStats: skipped: int succeeded: int failed: int diff --git a/archivebox/templates/index_row.html b/archivebox/templates/index_row.html index d3174ec0..766f8038 100644 --- a/archivebox/templates/index_row.html +++ b/archivebox/templates/index_row.html @@ -1,14 +1,14 @@ <tr> <td title="$timestamp">$bookmarked_date</td> <td style="text-align:left"> - <a href="$link_dir/$index_url"><img src="$favicon_url" class="link-favicon" decoding="async"></a> - <a href="$link_dir/$archive_url" title="$title"> + <a href="$archive_path/$index_url"><img src="$favicon_url" class="link-favicon" decoding="async"></a> + <a href="$archive_path/$archive_url" title="$title"> <span data-title-for="$url" data-archived="$is_archived">$title</span> <small>$tags</small> </a> </td> <td> - <a href="$link_dir/$index_url">📄 + <a href="$archive_path/$index_url">📄 <span data-number-for="$url" title="Fetching any missing files...">$num_outputs <img src="static/spinner.gif" class="files-spinner" decoding="async"/></span> </a> </td> diff --git a/archivebox/util.py b/archivebox/util.py index 2c2c6a05..ef0b8fe6 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -4,9 +4,8 @@ import sys import time from json import JSONEncoder - -from typing import List, Dict, Optional, Iterable - +from typing import List, Optional, Iterable +from hashlib import sha256 from urllib.request import Request, urlopen from urllib.parse import urlparse, quote, unquote from html import escape, unescape @@ -21,17 +20,17 @@ from subprocess import ( CalledProcessError, ) -from schema import Link +from base32_crockford import encode as base32_encode + +from schema import Link, LinkDict, ArchiveResult from config import ( ANSI, TERM_WIDTH, SOURCES_DIR, - ARCHIVE_DIR, OUTPUT_PERMISSIONS, TIMEOUT, SHOW_PROGRESS, FETCH_TITLE, - ARCHIVE_DIR_NAME, CHECK_SSL_VALIDITY, WGET_USER_AGENT, CHROME_OPTIONS, @@ -43,7 +42,7 @@ from logs import pretty_path # All of these are (str) -> str # shortcuts to: https://docs.python.org/3/library/urllib.parse.html#url-parsing -scheme = lambda url: urlparse(url).scheme +scheme = lambda url: urlparse(url).scheme.lower() without_scheme = lambda url: urlparse(url)._replace(scheme='').geturl().strip('//') without_query = lambda url: urlparse(url)._replace(query='').geturl().strip('//') without_fragment = lambda url: urlparse(url)._replace(fragment='').geturl().strip('//') @@ -56,11 +55,33 @@ fragment = lambda url: urlparse(url).fragment extension = lambda url: basename(url).rsplit('.', 1)[-1].lower() if '.' in basename(url) else '' base_url = lambda url: without_scheme(url) # uniq base url used to dedupe links -short_ts = lambda ts: ts.split('.')[0] -urlencode = lambda s: quote(s, encoding='utf-8', errors='replace') -urldecode = lambda s: unquote(s) -htmlencode = lambda s: escape(s, quote=True) -htmldecode = lambda s: unescape(s) + +without_www = lambda url: url.replace('://www.', '://', 1) +without_trailing_slash = lambda url: url[:-1] if url[-1] == '/' else url.replace('/?', '?') +fuzzy_url = lambda url: without_trailing_slash(without_www(without_scheme(url.lower()))) + +short_ts = lambda ts: ( + str(ts.timestamp()).split('.')[0] + if isinstance(ts, datetime) else + str(ts).split('.')[0] +) +ts_to_date = lambda ts: ( + ts.strftime('%Y-%m-%d %H:%M') + if isinstance(ts, datetime) else + datetime.fromtimestamp(float(ts)).strftime('%Y-%m-%d %H:%M') +) +ts_to_iso = lambda ts: ( + ts.isoformat() + if isinstance(ts, datetime) else + datetime.fromtimestamp(float(ts)).isoformat() +) + +urlencode = lambda s: s and quote(s, encoding='utf-8', errors='replace') +urldecode = lambda s: s and unquote(s) +htmlencode = lambda s: s and escape(s, quote=True) +htmldecode = lambda s: s and unescape(s) + +hashurl = lambda url: base32_encode(int(sha256(base_url(url).encode('utf-8')).hexdigest(), 16))[:20] URL_REGEX = re.compile( r'http[s]?://' # start matching from allowed schemes @@ -80,7 +101,8 @@ STATICFILE_EXTENSIONS = { # that can be downloaded as-is, not html pages that need to be rendered 'gif', 'jpeg', 'jpg', 'png', 'tif', 'tiff', 'wbmp', 'ico', 'jng', 'bmp', 'svg', 'svgz', 'webp', 'ps', 'eps', 'ai', - 'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v', 'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8' + 'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v', + 'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8' 'pdf', 'txt', 'rtf', 'rtfd', 'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'atom', 'rss', 'css', 'js', 'json', 'dmg', 'iso', 'img', @@ -100,7 +122,7 @@ STATICFILE_EXTENSIONS = { ### Checks & Tests -def check_link_structure(link: Link) -> None: +def check_link_structure(link: LinkDict) -> None: """basic sanity check invariants to make sure the data is valid""" assert isinstance(link, dict) assert isinstance(link.get('url'), str) @@ -112,7 +134,7 @@ def check_link_structure(link: Link) -> None: assert isinstance(key, str) assert isinstance(val, list), 'history must be a Dict[str, List], got: {}'.format(link['history']) -def check_links_structure(links: Iterable[Link]) -> None: +def check_links_structure(links: Iterable[LinkDict]) -> None: """basic sanity check invariants to make sure the data is valid""" assert isinstance(links, list) if links: @@ -213,7 +235,7 @@ def fetch_page_title(url: str, timeout: int=10, progress: bool=SHOW_PROGRESS) -> html = download_url(url, timeout=timeout) match = re.search(HTML_TITLE_REGEX, html) - return match.group(1).strip() if match else None + return htmldecode(match.group(1).strip()) if match else None except Exception as err: # noqa # print('[!] Failed to fetch title because of {}: {}'.format( # err.__class__.__name__, @@ -228,8 +250,8 @@ def wget_output_path(link: Link) -> Optional[str]: See docs on wget --adjust-extension (-E) """ - if is_static_file(link['url']): - return without_scheme(without_fragment(link['url'])) + if is_static_file(link.url): + return without_scheme(without_fragment(link.url)) # Wget downloads can save in a number of different ways depending on the url: # https://example.com @@ -262,11 +284,10 @@ def wget_output_path(link: Link) -> Optional[str]: # and there's no way to get the computed output path from wget # in order to avoid having to reverse-engineer how they calculate it, # we just look in the output folder read the filename wget used from the filesystem - link_dir = os.path.join(ARCHIVE_DIR, link['timestamp']) - full_path = without_fragment(without_query(path(link['url']))).strip('/') + full_path = without_fragment(without_query(path(link.url))).strip('/') search_dir = os.path.join( - link_dir, - domain(link['url']), + link.link_dir, + domain(link.url), full_path, ) @@ -278,13 +299,13 @@ def wget_output_path(link: Link) -> Optional[str]: if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", f, re.I | re.M) ] if html_files: - path_from_link_dir = search_dir.split(link_dir)[-1].strip('/') + path_from_link_dir = search_dir.split(link.link_dir)[-1].strip('/') return os.path.join(path_from_link_dir, html_files[0]) # Move up one directory level search_dir = search_dir.rsplit('/', 1)[0] - if search_dir == link_dir: + if search_dir == link.link_dir: break return None @@ -314,19 +335,20 @@ def merge_links(a: Link, b: Link) -> Link: """deterministially merge two links, favoring longer field values over shorter, and "cleaner" values over worse ones. """ + a, b = a._asdict(), b._asdict() longer = lambda key: (a[key] if len(a[key]) > len(b[key]) else b[key]) if (a[key] and b[key]) else (a[key] or b[key]) earlier = lambda key: a[key] if a[key] < b[key] else b[key] url = longer('url') longest_title = longer('title') cleanest_title = a['title'] if '://' not in (a['title'] or '') else b['title'] - return { - 'url': url, - 'timestamp': earlier('timestamp'), - 'title': longest_title if '://' not in (longest_title or '') else cleanest_title, - 'tags': longer('tags'), - 'sources': list(set(a.get('sources', []) + b.get('sources', []))), - } + return Link( + url=url, + timestamp=earlier('timestamp'), + title=longest_title if '://' not in (longest_title or '') else cleanest_title, + tags=longer('tags'), + sources=list(set(a.get('sources', []) + b.get('sources', []))), + ) def is_static_file(url: str) -> bool: """Certain URLs just point to a single static file, and @@ -339,85 +361,11 @@ def is_static_file(url: str) -> bool: def derived_link_info(link: Link) -> dict: """extend link info with the archive urls and other derived data""" - url = link['url'] + info = link._asdict(extended=True) + info.update(link.canonical_outputs()) - to_date_str = lambda ts: datetime.fromtimestamp(float(ts)).strftime('%Y-%m-%d %H:%M') + return info - extended_info = { - **link, - 'link_dir': '{}/{}'.format(ARCHIVE_DIR_NAME, link['timestamp']), - 'bookmarked_date': to_date_str(link['timestamp']), - 'updated_date': to_date_str(link['updated']) if 'updated' in link else None, - 'domain': domain(url), - 'path': path(url), - 'basename': basename(url), - 'extension': extension(url), - 'base_url': base_url(url), - 'is_static': is_static_file(url), - 'is_archived': os.path.exists(os.path.join( - ARCHIVE_DIR, - link['timestamp'], - domain(url), - )), - 'num_outputs': len([entry for entry in latest_output(link).values() if entry]), - } - - # Archive Method Output URLs - extended_info.update({ - 'index_url': 'index.html', - 'favicon_url': 'favicon.ico', - 'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**extended_info), - 'archive_url': wget_output_path(link), - 'warc_url': 'warc', - 'pdf_url': 'output.pdf', - 'screenshot_url': 'screenshot.png', - 'dom_url': 'output.html', - 'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**extended_info), - 'git_url': 'git', - 'media_url': 'media', - }) - # static binary files like PDF and images are handled slightly differently. - # they're just downloaded once and aren't archived separately multiple times, - # so the wget, screenshot, & pdf urls should all point to the same file - if is_static_file(url): - extended_info.update({ - 'title': basename(url), - 'archive_url': base_url(url), - 'pdf_url': base_url(url), - 'screenshot_url': base_url(url), - 'dom_url': base_url(url), - }) - - return extended_info - - -def latest_output(link: Link, status: str=None) -> Dict[str, Optional[str]]: - """get the latest output that each archive method produced for link""" - - latest = { - 'title': None, - 'favicon': None, - 'wget': None, - 'warc': None, - 'pdf': None, - 'screenshot': None, - 'dom': None, - 'git': None, - 'media': None, - 'archive_org': None, - } - for archive_method in latest.keys(): - # get most recent succesful result in history for each archive method - history = link.get('history', {}).get(archive_method) or [] - history = filter(lambda result: result['output'], reversed(history)) - if status is not None: - history = filter(lambda result: result['status'] == status, history) - - history = list(history) - if history: - latest[archive_method] = history[0]['output'] - - return latest ### Python / System Helpers @@ -466,21 +414,13 @@ class TimedProgress: self.p = Process(target=progress_bar, args=(seconds, prefix)) self.p.start() - self.stats = { - 'start_ts': datetime.now(), - 'end_ts': None, - 'duration': None, - } + self.stats = {'start_ts': datetime.now(), 'end_ts': None} def end(self): """immediately end progress, clear the progressbar line, and save end_ts""" end_ts = datetime.now() - self.stats.update({ - 'end_ts': end_ts, - 'duration': (end_ts - self.stats['start_ts']).seconds, - }) - + self.stats['end_ts'] = end_ts if SHOW_PROGRESS: # protect from double termination #if p is None or not hasattr(p, 'kill'): From 0d8a076c1fc95d8896beb6a431bd65f35c2775c0 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Tue, 26 Mar 2019 19:21:48 -0400 Subject: [PATCH 008/365] add base32 crockford dependency --- archivebox/base32_crockford.py | 172 +++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 archivebox/base32_crockford.py diff --git a/archivebox/base32_crockford.py b/archivebox/base32_crockford.py new file mode 100644 index 00000000..bafb69b4 --- /dev/null +++ b/archivebox/base32_crockford.py @@ -0,0 +1,172 @@ +""" +base32-crockford +================ + +A Python module implementing the alternate base32 encoding as described +by Douglas Crockford at: http://www.crockford.com/wrmg/base32.html. + +He designed the encoding to: + + * Be human and machine readable + * Be compact + * Be error resistant + * Be pronounceable + +It uses a symbol set of 10 digits and 22 letters, excluding I, L O and +U. Decoding is not case sensitive, and 'i' and 'l' are converted to '1' +and 'o' is converted to '0'. Encoding uses only upper-case characters. + +Hyphens may be present in symbol strings to improve readability, and +are removed when decoding. + +A check symbol can be appended to a symbol string to detect errors +within the string. + +""" + +import re +import sys + +PY3 = sys.version_info[0] == 3 + +if not PY3: + import string as str + + +__all__ = ["encode", "decode", "normalize"] + + +if PY3: + string_types = str, +else: + string_types = basestring, + +# The encoded symbol space does not include I, L, O or U +symbols = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' +# These five symbols are exclusively for checksum values +check_symbols = '*~$=U' + +encode_symbols = dict((i, ch) for (i, ch) in enumerate(symbols + check_symbols)) +decode_symbols = dict((ch, i) for (i, ch) in enumerate(symbols + check_symbols)) +normalize_symbols = str.maketrans('IiLlOo', '111100') +valid_symbols = re.compile('^[%s]+[%s]?$' % (symbols, + re.escape(check_symbols))) + +base = len(symbols) +check_base = len(symbols + check_symbols) + + +def encode(number, checksum=False, split=0): + """Encode an integer into a symbol string. + + A ValueError is raised on invalid input. + + If checksum is set to True, a check symbol will be + calculated and appended to the string. + + If split is specified, the string will be divided into + clusters of that size separated by hyphens. + + The encoded string is returned. + """ + number = int(number) + if number < 0: + raise ValueError("number '%d' is not a positive integer" % number) + + split = int(split) + if split < 0: + raise ValueError("split '%d' is not a positive integer" % split) + + check_symbol = '' + if checksum: + check_symbol = encode_symbols[number % check_base] + + if number == 0: + return '0' + check_symbol + + symbol_string = '' + while number > 0: + remainder = number % base + number //= base + symbol_string = encode_symbols[remainder] + symbol_string + symbol_string = symbol_string + check_symbol + + if split: + chunks = [] + for pos in range(0, len(symbol_string), split): + chunks.append(symbol_string[pos:pos + split]) + symbol_string = '-'.join(chunks) + + return symbol_string + + +def decode(symbol_string, checksum=False, strict=False): + """Decode an encoded symbol string. + + If checksum is set to True, the string is assumed to have a + trailing check symbol which will be validated. If the + checksum validation fails, a ValueError is raised. + + If strict is set to True, a ValueError is raised if the + normalization step requires changes to the string. + + The decoded string is returned. + """ + symbol_string = normalize(symbol_string, strict=strict) + if checksum: + symbol_string, check_symbol = symbol_string[:-1], symbol_string[-1] + + number = 0 + for symbol in symbol_string: + number = number * base + decode_symbols[symbol] + + if checksum: + check_value = decode_symbols[check_symbol] + modulo = number % check_base + if check_value != modulo: + raise ValueError("invalid check symbol '%s' for string '%s'" % + (check_symbol, symbol_string)) + + return number + + +def normalize(symbol_string, strict=False): + """Normalize an encoded symbol string. + + Normalization provides error correction and prepares the + string for decoding. These transformations are applied: + + 1. Hyphens are removed + 2. 'I', 'i', 'L' or 'l' are converted to '1' + 3. 'O' or 'o' are converted to '0' + 4. All characters are converted to uppercase + + A TypeError is raised if an invalid string type is provided. + + A ValueError is raised if the normalized string contains + invalid characters. + + If the strict parameter is set to True, a ValueError is raised + if any of the above transformations are applied. + + The normalized string is returned. + """ + if isinstance(symbol_string, string_types): + if not PY3: + try: + symbol_string = symbol_string.encode('ascii') + except UnicodeEncodeError: + raise ValueError("string should only contain ASCII characters") + else: + raise TypeError("string is of invalid type %s" % + symbol_string.__class__.__name__) + + norm_string = symbol_string.replace('-', '').translate(normalize_symbols).upper() + + if not valid_symbols.match(norm_string): + raise ValueError("string '%s' contains invalid characters" % norm_string) + + if strict and norm_string != symbol_string: + raise ValueError("string '%s' requires normalization" % symbol_string) + + return norm_string From ab09560f14703f1aa1e5ebc797920ec214489b9a Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Tue, 26 Mar 2019 22:26:21 -0400 Subject: [PATCH 009/365] working runtime type casting and enforcement for a wide range of types --- archivebox/index.py | 29 ++++--- archivebox/schema.py | 17 ++++- archivebox/util.py | 177 ++++++++++++++++++++++++++++++++----------- 3 files changed, 162 insertions(+), 61 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index 0a60dd23..f0cd46af 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -1,7 +1,6 @@ import os import json -from itertools import chain from datetime import datetime from string import Template from typing import List, Tuple, Iterator, Optional @@ -20,13 +19,13 @@ from config import ( FOOTER_INFO, ) from util import ( + merge_links, chmod_file, urlencode, derived_link_info, wget_output_path, ExtendedEncoder, - check_link_structure, - check_links_structure, + enforce_types, ) from parse import parse_links from links import validate_links @@ -43,6 +42,7 @@ TITLE_LOADING_MSG = 'Not yet archived...' ### Homepage index for all the links +@enforce_types def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: """create index.html file for a given list of links""" @@ -55,8 +55,9 @@ def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> log_indexing_started(out_dir, 'index.html') write_html_links_index(out_dir, links, finished=finished) log_indexing_finished(out_dir, 'index.html') - + +@enforce_types def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[List[Link], List[Link]]: """parse and load existing index with any new links from import_path merged in""" @@ -81,6 +82,7 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[Li return all_links, new_links +@enforce_types def write_json_links_index(out_dir: str, links: List[Link]) -> None: """write the json link index to a given path""" @@ -114,6 +116,7 @@ def write_json_links_index(out_dir: str, links: List[Link]) -> None: chmod_file(path) +@enforce_types def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: """parse a archive index json file and return the list of links""" @@ -121,13 +124,13 @@ def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: if os.path.exists(index_path): with open(index_path, 'r', encoding='utf-8') as f: links = json.load(f)['links'] - check_links_structure(links) for link in links: yield Link(**link) return () +@enforce_types def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: """write the html link index to a given path""" @@ -151,6 +154,7 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False link.title or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), + 'tags': link.tags or '', 'favicon_url': ( os.path.join('archive', link.timestamp, 'favicon.ico') # if link['is_archived'] else 'data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=' @@ -179,6 +183,7 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False chmod_file(path) +@enforce_types def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: """hack to in-place update one row's info in the generated index html""" @@ -218,11 +223,13 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: ### Individual link index +@enforce_types def write_link_index(out_dir: str, link: Link) -> None: write_json_link_index(out_dir, link) write_html_link_index(out_dir, link) +@enforce_types def write_json_link_index(out_dir: str, link: Link) -> None: """write a json file with some info about the link""" @@ -234,29 +241,29 @@ def write_json_link_index(out_dir: str, link: Link) -> None: chmod_file(path) +@enforce_types def parse_json_link_index(out_dir: str) -> Optional[Link]: """load the json link index from a given directory""" existing_index = os.path.join(out_dir, 'index.json') if os.path.exists(existing_index): with open(existing_index, 'r', encoding='utf-8') as f: link_json = json.load(f) - check_link_structure(link_json) return Link(**link_json) return None +@enforce_types def load_json_link_index(out_dir: str, link: Link) -> Link: """check for an existing link archive in the given directory, and load+merge it into the given link dict """ - existing_link = parse_json_link_index(out_dir) - existing_link = existing_link._asdict() if existing_link else {} - new_link = link._asdict() - - return Link(**{**existing_link, **new_link}) + if existing_link: + return merge_links(existing_link, link) + return link +@enforce_types def write_html_link_index(out_dir: str, link: Link) -> None: with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: link_html = f.read() diff --git a/archivebox/schema.py b/archivebox/schema.py index b92d1779..e02d69c7 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -39,15 +39,24 @@ class Link: tags: Optional[str] sources: List[str] history: Dict[str, List[ArchiveResult]] = field(default_factory=lambda: {}) - updated: Optional[str] = None + updated: Optional[datetime] = None - def __hash__(self): - return self.urlhash + def __post_init__(self): + """fix any history result items to be type-checked ArchiveResults""" + cast_history = {} + for method, method_history in self.history.items(): + cast_history[method] = [] + for result in method_history: + if isinstance(result, dict): + result = ArchiveResult(**result) + cast_history[method].append(result) + + object.__setattr__(self, 'history', cast_history) def __eq__(self, other): if not isinstance(other, Link): return NotImplemented - return self.urlhash == other.urlhash + return self.url == other.url def __gt__(self, other): if not isinstance(other, Link): diff --git a/archivebox/util.py b/archivebox/util.py index ef0b8fe6..5097ec76 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -4,7 +4,9 @@ import sys import time from json import JSONEncoder -from typing import List, Optional, Iterable +from typing import List, Optional, Any +from inspect import signature, _empty +from functools import wraps from hashlib import sha256 from urllib.request import Request, urlopen from urllib.parse import urlparse, quote, unquote @@ -22,7 +24,7 @@ from subprocess import ( from base32_crockford import encode as base32_encode -from schema import Link, LinkDict, ArchiveResult +from schema import Link from config import ( ANSI, TERM_WIDTH, @@ -55,26 +57,13 @@ fragment = lambda url: urlparse(url).fragment extension = lambda url: basename(url).rsplit('.', 1)[-1].lower() if '.' in basename(url) else '' base_url = lambda url: without_scheme(url) # uniq base url used to dedupe links - without_www = lambda url: url.replace('://www.', '://', 1) without_trailing_slash = lambda url: url[:-1] if url[-1] == '/' else url.replace('/?', '?') fuzzy_url = lambda url: without_trailing_slash(without_www(without_scheme(url.lower()))) -short_ts = lambda ts: ( - str(ts.timestamp()).split('.')[0] - if isinstance(ts, datetime) else - str(ts).split('.')[0] -) -ts_to_date = lambda ts: ( - ts.strftime('%Y-%m-%d %H:%M') - if isinstance(ts, datetime) else - datetime.fromtimestamp(float(ts)).strftime('%Y-%m-%d %H:%M') -) -ts_to_iso = lambda ts: ( - ts.isoformat() - if isinstance(ts, datetime) else - datetime.fromtimestamp(float(ts)).isoformat() -) +short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0] +ts_to_date = lambda ts: parse_date(ts).strftime('%Y-%m-%d %H:%M') +ts_to_iso = lambda ts: parse_date(ts).isoformat() urlencode = lambda s: s and quote(s, encoding='utf-8', errors='replace') urldecode = lambda s: s and unquote(s) @@ -122,23 +111,46 @@ STATICFILE_EXTENSIONS = { ### Checks & Tests -def check_link_structure(link: LinkDict) -> None: - """basic sanity check invariants to make sure the data is valid""" - assert isinstance(link, dict) - assert isinstance(link.get('url'), str) - assert len(link['url']) > 2 - assert len(re.findall(URL_REGEX, link['url'])) == 1 - if 'history' in link: - assert isinstance(link['history'], dict), 'history must be a Dict' - for key, val in link['history'].items(): - assert isinstance(key, str) - assert isinstance(val, list), 'history must be a Dict[str, List], got: {}'.format(link['history']) - -def check_links_structure(links: Iterable[LinkDict]) -> None: - """basic sanity check invariants to make sure the data is valid""" - assert isinstance(links, list) - if links: - check_link_structure(links[0]) +def enforce_types(func): + """ + Checks parameters type signatures against arg and kwarg type hints. + """ + + @wraps(func) + def typechecked_function(*args, **kwargs): + sig = signature(func) + + def check_argument_type(arg_key, arg_val): + try: + annotation = sig.parameters[arg_key].annotation + except KeyError: + annotation = _empty + + if annotation is not _empty and annotation.__class__ is type: + if not isinstance(arg_val, annotation): + raise TypeError( + '{}(..., {}: {}) got unexpected {} argument {}={}'.format( + func.__name__, + arg_key, + annotation.__name__, + type(arg_val).__name__, + arg_key, + arg_val, + ) + ) + + # check args + for arg_val, arg_key in zip(args, sig.parameters): + check_argument_type(arg_key, arg_val) + + # check kwargs + for arg_key, arg_val in kwargs.items(): + check_argument_type(arg_key, arg_val) + + return func(*args, **kwargs) + + return typechecked_function + def check_url_parsing_invariants() -> None: """Check that plain text regex URL parsing works as expected""" @@ -329,25 +341,98 @@ def str_between(string: str, start: str, end: str=None) -> str: return content +def parse_date(date: Any) -> Optional[datetime]: + """Parse unix timestamps, iso format, and human-readable strings""" + + if isinstance(date, datetime): + return date + + if date is None: + return None + + if isinstance(date, (float, int)): + date = str(date) + + if isinstance(date, str): + if date.replace('.', '').isdigit(): + timestamp = float(date) + + EARLIEST_POSSIBLE = 473403600.0 # 1985 + LATEST_POSSIBLE = 1735707600.0 # 2025 + + if EARLIEST_POSSIBLE < timestamp < LATEST_POSSIBLE: + # number is seconds + return datetime.fromtimestamp(timestamp) + elif EARLIEST_POSSIBLE * 1000 < timestamp < LATEST_POSSIBLE * 1000: + # number is milliseconds + return datetime.fromtimestamp(timestamp / 1000) + + elif EARLIEST_POSSIBLE * 1000*1000 < timestamp < LATEST_POSSIBLE * 1000*1000: + # number is microseconds + return datetime.fromtimestamp(timestamp / (1000*1000)) + + if '-' in date: + try: + return datetime.fromisoformat(date) + except Exception: + try: + return datetime.strptime(date, '%Y-%m-%d %H:%M') + except Exception: + pass + + raise ValueError('Tried to parse invalid date! {}'.format(date)) + + + ### Link Helpers +@enforce_types def merge_links(a: Link, b: Link) -> Link: """deterministially merge two links, favoring longer field values over shorter, and "cleaner" values over worse ones. """ - a, b = a._asdict(), b._asdict() - longer = lambda key: (a[key] if len(a[key]) > len(b[key]) else b[key]) if (a[key] and b[key]) else (a[key] or b[key]) - earlier = lambda key: a[key] if a[key] < b[key] else b[key] - - url = longer('url') - longest_title = longer('title') - cleanest_title = a['title'] if '://' not in (a['title'] or '') else b['title'] + assert a.base_url == b.base_url, 'Cannot merge two links with different URLs' + + url = a.url if len(a.url) > len(b.url) else b.url + + possible_titles = [ + title + for title in (a.title, b.title) + if title and title.strip() and '://' not in title + ] + title = None + if len(possible_titles) == 2: + title = max(possible_titles, key=lambda t: len(t)) + elif len(possible_titles) == 1: + title = possible_titles[0] + + timestamp = ( + a.timestamp + if float(a.timestamp or 0) < float(b.timestamp or 0) else + b.timestamp + ) + + tags_set = ( + set(tag.strip() for tag in (a.tags or '').split(',')) + | set(tag.strip() for tag in (b.tags or '').split(',')) + ) + tags = ','.join(tags_set) or None + + sources = list(set(a.sources + b.sources)) + + all_methods = (set(a.history.keys()) | set(a.history.keys())) + history = { + method: (a.history.get(method) or []) + (b.history.get(method) or []) + for method in all_methods + } + return Link( url=url, - timestamp=earlier('timestamp'), - title=longest_title if '://' not in (longest_title or '') else cleanest_title, - tags=longer('tags'), - sources=list(set(a.get('sources', []) + b.get('sources', []))), + timestamp=timestamp, + title=title, + tags=tags, + sources=sources, + history=history, ) def is_static_file(url: str) -> bool: From c9c5b04df055f6dfaec7468b0cbfa1fa7c7295f1 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Tue, 26 Mar 2019 23:25:07 -0400 Subject: [PATCH 010/365] full type-hinting coverage --- archivebox/archive.py | 4 +++- archivebox/archive_methods.py | 23 ++++++++++++++++++++++- archivebox/config.py | 30 ++++++++++++++++++++++-------- archivebox/index.py | 2 +- archivebox/parse.py | 10 ++++++++++ archivebox/schema.py | 13 +++++++++++++ archivebox/util.py | 25 +++++++++++++++++++++++-- 7 files changed, 94 insertions(+), 13 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index c6e10bd2..ff4128c9 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -24,6 +24,7 @@ from config import ( GIT_SHA, ) from util import ( + enforce_types, save_remote_source, save_stdin_source, ) @@ -100,7 +101,8 @@ def main(*args) -> List[Link]: return update_archive_data(import_path=import_path, resume=resume) -def update_archive_data(import_path: str=None, resume: float=None) -> List[Link]: +@enforce_types +def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]=None) -> List[Link]: """The main ArchiveBox entrancepoint. Everything starts here.""" # Step 1: Load list of links from the existing index diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 76153e70..3bfc15a7 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -42,6 +42,7 @@ from config import ( YOUTUBEDL_VERSION, ) from util import ( + enforce_types, domain, extension, without_query, @@ -63,6 +64,7 @@ from logs import ( ) +@enforce_types def archive_link(link: Link, page=None) -> Link: """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" @@ -126,6 +128,7 @@ def archive_link(link: Link, page=None) -> Link: ### Archive Method Functions +@enforce_types def should_fetch_title(link_dir: str, link: Link) -> bool: # if link already has valid title, skip it if link.title and not link.title.lower().startswith('http'): @@ -136,6 +139,7 @@ def should_fetch_title(link_dir: str, link: Link) -> bool: return FETCH_TITLE +@enforce_types def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """try to guess the page's title from its content""" @@ -169,12 +173,14 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul ) +@enforce_types def should_fetch_favicon(link_dir: str, link: Link) -> bool: if os.path.exists(os.path.join(link_dir, 'favicon.ico')): return False return FETCH_FAVICON - + +@enforce_types def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """download site favicon from google's favicon api""" @@ -207,6 +213,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes **timer.stats, ) +@enforce_types def should_fetch_wget(link_dir: str, link: Link) -> bool: output_path = wget_output_path(link) if output_path and os.path.exists(os.path.join(link_dir, output_path)): @@ -215,6 +222,7 @@ def should_fetch_wget(link_dir: str, link: Link) -> bool: return FETCH_WGET +@enforce_types def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using wget""" @@ -294,6 +302,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult **timer.stats, ) +@enforce_types def should_fetch_pdf(link_dir: str, link: Link) -> bool: if is_static_file(link.url): return False @@ -304,6 +313,7 @@ def should_fetch_pdf(link_dir: str, link: Link) -> bool: return FETCH_PDF +@enforce_types def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """print PDF of site to file using chrome --headless""" @@ -338,6 +348,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: **timer.stats, ) +@enforce_types def should_fetch_screenshot(link_dir: str, link: Link) -> bool: if is_static_file(link.url): return False @@ -347,6 +358,7 @@ def should_fetch_screenshot(link_dir: str, link: Link) -> bool: return FETCH_SCREENSHOT +@enforce_types def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """take screenshot of site using chrome --headless""" @@ -381,6 +393,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive **timer.stats, ) +@enforce_types def should_fetch_dom(link_dir: str, link: Link) -> bool: if is_static_file(link.url): return False @@ -390,6 +403,7 @@ def should_fetch_dom(link_dir: str, link: Link) -> bool: return FETCH_DOM +@enforce_types def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """print HTML of site to file using chrome --dump-html""" @@ -426,6 +440,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: **timer.stats, ) +@enforce_types def should_fetch_git(link_dir: str, link: Link) -> bool: if is_static_file(link.url): return False @@ -443,6 +458,7 @@ def should_fetch_git(link_dir: str, link: Link) -> bool: return FETCH_GIT +@enforce_types def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using git""" @@ -485,6 +501,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) +@enforce_types def should_fetch_media(link_dir: str, link: Link) -> bool: if is_static_file(link.url): return False @@ -494,6 +511,7 @@ def should_fetch_media(link_dir: str, link: Link) -> bool: return FETCH_MEDIA +@enforce_types def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: """Download playlists or individual video, audio, and subtitles using youtube-dl""" @@ -557,6 +575,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv ) +@enforce_types def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: if is_static_file(link.url): return False @@ -567,6 +586,7 @@ def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: return SUBMIT_ARCHIVE_DOT_ORG +@enforce_types def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: """submit site to archive.org for archiving via their service, save returned archive url""" @@ -622,6 +642,7 @@ def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveR **timer.stats, ) +@enforce_types def parse_archive_dot_org_response(response: bytes) -> Tuple[List[str], List[str]]: # Parse archive.org response headers headers: Dict[str, List[str]] = defaultdict(list) diff --git a/archivebox/config.py b/archivebox/config.py index ec38b367..38a12d4a 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -46,6 +46,10 @@ CHROME_USER_DATA_DIR = os.getenv('CHROME_USER_DATA_DIR', None) CHROME_HEADLESS = os.getenv('CHROME_HEADLESS', 'True' ).lower() == 'true' CHROME_USER_AGENT = os.getenv('CHROME_USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36') +USE_CURL = os.getenv('USE_CURL', 'True' ).lower() == 'true' +USE_WGET = os.getenv('USE_WGET', 'True' ).lower() == 'true' +USE_CHROME = os.getenv('USE_CHROME', 'True' ).lower() == 'true' + CURL_BINARY = os.getenv('CURL_BINARY', 'curl') GIT_BINARY = os.getenv('GIT_BINARY', 'git') WGET_BINARY = os.getenv('WGET_BINARY', 'wget') @@ -195,13 +199,19 @@ try: print(' env PYTHONIOENCODING=UTF-8 ./archive.py export.html') ### Make sure curl is installed - USE_CURL = FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG + if USE_CURL: + USE_CURL = FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG + else: + FETCH_FAVICON = SUBMIT_ARCHIVE_DOT_ORG = False CURL_VERSION = None if USE_CURL: CURL_VERSION = check_version(CURL_BINARY) ### Make sure wget is installed and calculate version - USE_WGET = FETCH_WGET or FETCH_WARC + if USE_WGET: + USE_WGET = FETCH_WGET or FETCH_WARC + else: + FETCH_WGET = FETCH_WARC = False WGET_VERSION = None if USE_WGET: WGET_VERSION = check_version(WGET_BINARY) @@ -222,17 +232,21 @@ try: check_version(YOUTUBEDL_BINARY) ### Make sure chrome is installed and calculate version - USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM + if USE_CHROME: + USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM + else: + FETCH_PDF = FETCH_SCREENSHOT = FETCH_DOM = False CHROME_VERSION = None if USE_CHROME: if CHROME_BINARY is None: CHROME_BINARY = find_chrome_binary() - CHROME_VERSION = check_version(CHROME_BINARY) - # print('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY)) + if CHROME_BINARY: + CHROME_VERSION = check_version(CHROME_BINARY) + # print('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY)) - if CHROME_USER_DATA_DIR is None: - CHROME_USER_DATA_DIR = find_chrome_data_dir() - # print('[i] Using Chrome data dir: {}'.format(os.path.abspath(CHROME_USER_DATA_DIR))) + if CHROME_USER_DATA_DIR is None: + CHROME_USER_DATA_DIR = find_chrome_data_dir() + # print('[i] Using Chrome data dir: {}'.format(os.path.abspath(CHROME_USER_DATA_DIR))) CHROME_OPTIONS = { 'TIMEOUT': TIMEOUT, diff --git a/archivebox/index.py b/archivebox/index.py index f0cd46af..2bf2b5eb 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -58,7 +58,7 @@ def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> @enforce_types -def load_links_index(out_dir: str=OUTPUT_DIR, import_path: str=None) -> Tuple[List[Link], List[Link]]: +def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) -> Tuple[List[Link], List[Link]]: """parse and load existing index with any new links from import_path merged in""" existing_links: List[Link] = [] diff --git a/archivebox/parse.py b/archivebox/parse.py index ba200ff3..093d4a92 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -32,9 +32,11 @@ from util import ( check_url_parsing_invariants, TimedProgress, Link, + enforce_types, ) +@enforce_types def parse_links(source_file: str) -> Tuple[List[Link], str]: """parse a list of URLs with their metadata from an RSS feed, bookmarks export, or text file @@ -77,6 +79,7 @@ def parse_links(source_file: str) -> Tuple[List[Link], str]: ### Import Parser Functions +@enforce_types def parse_pocket_html_export(html_file: IO[str]) -> Iterable[Link]: """Parse Pocket-format bookmarks export files (produced by getpocket.com/export/)""" @@ -101,6 +104,7 @@ def parse_pocket_html_export(html_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_json_export(json_file: IO[str]) -> Iterable[Link]: """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)""" @@ -153,6 +157,7 @@ def parse_json_export(json_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse RSS XML-format files into links""" @@ -190,6 +195,7 @@ def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_shaarli_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Shaarli-specific RSS XML-format files into links""" @@ -227,6 +233,7 @@ def parse_shaarli_rss_export(rss_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: """Parse netscape-format bookmarks export files (produced by all browsers)""" @@ -251,6 +258,7 @@ def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Pinboard RSS feed files into links""" @@ -282,6 +290,7 @@ def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Medium RSS feed files into links""" @@ -303,6 +312,7 @@ def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: ) +@enforce_types def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: """Parse raw links from each line in a text file""" diff --git a/archivebox/schema.py b/archivebox/schema.py index e02d69c7..fa110653 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -23,6 +23,10 @@ class ArchiveResult: status: str start_ts: datetime end_ts: datetime + schema: str = 'ArchiveResult' + + def __post_init__(self): + assert self.schema == self.__class__.__name__ def _asdict(self): return asdict(self) @@ -40,9 +44,11 @@ class Link: sources: List[str] history: Dict[str, List[ArchiveResult]] = field(default_factory=lambda: {}) updated: Optional[datetime] = None + schema: str = 'Link' def __post_init__(self): """fix any history result items to be type-checked ArchiveResults""" + assert self.schema == self.__class__.__name__ cast_history = {} for method, method_history in self.history.items(): cast_history[method] = [] @@ -67,6 +73,7 @@ class Link: def _asdict(self, extended=False): info = { + 'schema': 'Link', 'url': self.url, 'title': self.title or None, 'timestamp': self.timestamp, @@ -234,12 +241,18 @@ class ArchiveIndex: num_links: int updated: str links: List[Link] + schema: str = 'ArchiveIndex' + + def __post_init__(self): + assert self.schema == self.__class__.__name__ def _asdict(self): return asdict(self) @dataclass class RuntimeStats: + """mutable stats counter for logging archiving timing info to CLI output""" + skipped: int succeeded: int failed: int diff --git a/archivebox/util.py b/archivebox/util.py index 5097ec76..dc5590c5 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -91,7 +91,7 @@ STATICFILE_EXTENSIONS = { 'gif', 'jpeg', 'jpg', 'png', 'tif', 'tiff', 'wbmp', 'ico', 'jng', 'bmp', 'svg', 'svgz', 'webp', 'ps', 'eps', 'ai', 'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v', - 'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8' + 'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8', 'pdf', 'txt', 'rtf', 'rtfd', 'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'atom', 'rss', 'css', 'js', 'json', 'dmg', 'iso', 'img', @@ -113,8 +113,9 @@ STATICFILE_EXTENSIONS = { def enforce_types(func): """ - Checks parameters type signatures against arg and kwarg type hints. + Enforce function arg and kwarg types at runtime using its python3 type hints """ + # TODO: check return type as well @wraps(func) def typechecked_function(*args, **kwargs): @@ -183,6 +184,7 @@ def check_url_parsing_invariants() -> None: ### Random Helpers +@enforce_types def save_stdin_source(raw_text: str) -> str: if not os.path.exists(SOURCES_DIR): os.makedirs(SOURCES_DIR) @@ -196,6 +198,8 @@ def save_stdin_source(raw_text: str) -> str: return source_path + +@enforce_types def save_remote_source(url: str, timeout: int=TIMEOUT) -> str: """download a given url's content into output/sources/domain-<timestamp>.txt""" @@ -233,6 +237,8 @@ def save_remote_source(url: str, timeout: int=TIMEOUT) -> str: return source_path + +@enforce_types def fetch_page_title(url: str, timeout: int=10, progress: bool=SHOW_PROGRESS) -> Optional[str]: """Attempt to guess a page's title by downloading the html""" @@ -255,6 +261,8 @@ def fetch_page_title(url: str, timeout: int=10, progress: bool=SHOW_PROGRESS) -> # )) return None + +@enforce_types def wget_output_path(link: Link) -> Optional[str]: """calculate the path to the wgetted .html file, since wget may adjust some paths to be different than the base_url path. @@ -323,14 +331,17 @@ def wget_output_path(link: Link) -> Optional[str]: return None +@enforce_types def read_js_script(script_name: str) -> str: script_path = os.path.join(PYTHON_PATH, 'scripts', script_name) with open(script_path, 'r') as f: return f.read().split('// INFO BELOW HERE')[0].strip() + ### String Manipulation & Logging Helpers +@enforce_types def str_between(string: str, start: str, end: str=None) -> str: """(<abc>12345</def>, <abc>, </def>) -> 12345""" @@ -341,6 +352,7 @@ def str_between(string: str, start: str, end: str=None) -> str: return content +@enforce_types def parse_date(date: Any) -> Optional[datetime]: """Parse unix timestamps, iso format, and human-readable strings""" @@ -435,6 +447,8 @@ def merge_links(a: Link, b: Link) -> Link: history=history, ) + +@enforce_types def is_static_file(url: str) -> bool: """Certain URLs just point to a single static file, and don't need to be re-archived in many formats @@ -443,6 +457,8 @@ def is_static_file(url: str) -> bool: # TODO: the proper way is with MIME type detection, not using extension return extension(url) in STATICFILE_EXTENSIONS + +@enforce_types def derived_link_info(link: Link) -> dict: """extend link info with the archive urls and other derived data""" @@ -518,6 +534,7 @@ class TimedProgress: sys.stdout.flush() +@enforce_types def progress_bar(seconds: int, prefix: str='') -> None: """show timer in the form of progress bar, with percentage and seconds remaining""" chunk = '█' if sys.stdout.encoding == 'UTF-8' else '#' @@ -557,6 +574,7 @@ def progress_bar(seconds: int, prefix: str='') -> None: pass +@enforce_types def download_url(url: str, timeout: int=TIMEOUT) -> str: """Download the contents of a remote url and return the text""" @@ -572,6 +590,8 @@ def download_url(url: str, timeout: int=TIMEOUT) -> str: encoding = resp.headers.get_content_charset() or 'utf-8' return resp.read().decode(encoding) + +@enforce_types def chmod_file(path: str, cwd: str='.', permissions: str=OUTPUT_PERMISSIONS, timeout: int=30) -> None: """chmod -R <permissions> <cwd>/<path>""" @@ -584,6 +604,7 @@ def chmod_file(path: str, cwd: str='.', permissions: str=OUTPUT_PERMISSIONS, tim raise Exception('Failed to chmod {}/{}'.format(cwd, path)) +@enforce_types def chrome_args(**options) -> List[str]: """helper to build up a chrome shell command with arguments""" From 2ed91fe42981ab8f774eb5eece80d2b2e5e1bd3f Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 03:49:39 -0400 Subject: [PATCH 011/365] new details page design --- archivebox/archive_methods.py | 42 +- archivebox/index.py | 15 +- archivebox/logs.py | 2 +- archivebox/schema.py | 38 ++ archivebox/templates/index.html | 123 ++--- archivebox/templates/index_row.html | 2 +- archivebox/templates/link_index.html | 440 +++++++++++------- archivebox/templates/static/bootstrap.min.css | 6 + archivebox/util.py | 2 +- 9 files changed, 419 insertions(+), 251 deletions(-) create mode 100644 archivebox/templates/static/bootstrap.min.css diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 3bfc15a7..db48f41c 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -40,6 +40,7 @@ from config import ( CHROME_VERSION, GIT_VERSION, YOUTUBEDL_VERSION, + ONLY_NEW, ) from util import ( enforce_types, @@ -87,33 +88,40 @@ def archive_link(link: Link, page=None) -> Link: link = load_json_link_index(link.link_dir, link) log_link_archiving_started(link.link_dir, link, is_new) + link = link.overwrite(updated=datetime.now()) stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} for method_name, should_run, method_function in ARCHIVE_METHODS: - if method_name not in link.history: - link.history[method_name] = [] - - if should_run(link.link_dir, link): - log_archive_method_started(method_name) + try: + if method_name not in link.history: + link.history[method_name] = [] + + if should_run(link.link_dir, link): + log_archive_method_started(method_name) - result = method_function(link.link_dir, link) + result = method_function(link.link_dir, link) - link.history[method_name].append(result) + link.history[method_name].append(result) - stats[result.status] += 1 - log_archive_method_finished(result) - else: - stats['skipped'] += 1 + stats[result.status] += 1 + log_archive_method_finished(result) + else: + stats['skipped'] += 1 + except Exception as e: + raise Exception('Exception in archive_methods.fetch_{}(Link(url={}))'.format( + method_name, + link.url, + )) from e # print(' ', stats) - link = Link(**{ - **link._asdict(), - 'updated': datetime.now(), - }) - + # If any changes were made, update the link index json and html write_link_index(link.link_dir, link) - patch_links_index(link) + + was_changed = stats['succeeded'] or stats['failed'] + if was_changed: + patch_links_index(link) + log_link_archiving_finished(link.link_dir, link, is_new, stats) except KeyboardInterrupt: diff --git a/archivebox/index.py b/archivebox/index.py index 2bf2b5eb..74e7dd42 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -154,7 +154,7 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False link.title or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), - 'tags': link.tags or '', + 'tags': (link.tags or '') + (' {}'.format(link.extension) if link.is_static else ''), 'favicon_url': ( os.path.join('archive', link.timestamp, 'favicon.ico') # if link['is_archived'] else 'data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=' @@ -196,12 +196,11 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: patched_links = [] for saved_link in json_file_links: if saved_link.url == link.url: - patched_links.append(Link(**{ - **saved_link._asdict(), - 'title': title, - 'history': link.history, - 'updated': link.updated, - })) + patched_links.append(saved_link.overwrite( + title=title, + history=link.history, + updated=link.updated, + )) else: patched_links.append(saved_link) @@ -283,7 +282,7 @@ def write_html_link_index(out_dir: str, link: Link) -> None: ), 'extension': link.extension or 'html', 'tags': link.tags or 'untagged', - 'status': 'Archived' if link.is_archived else 'Not yet archived', + 'status': 'archived' if link.is_archived else 'not yet archived', 'status_color': 'success' if link.is_archived else 'danger', })) diff --git a/archivebox/logs.py b/archivebox/logs.py index fd1f0bc5..b2913c18 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -131,7 +131,7 @@ def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format( symbol_color=ANSI['green' if is_new else 'black'], - symbol='+' if is_new else '*', + symbol='+' if is_new else '√', now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), title=link.title or link.base_url, **ANSI, diff --git a/archivebox/schema.py b/archivebox/schema.py index fa110653..434f9dc5 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -59,6 +59,10 @@ class Link: object.__setattr__(self, 'history', cast_history) + def overwrite(self, **kwargs): + """pure functional version of dict.update that returns a new instance""" + return Link(**{**self._asdict(), **kwargs}) + def __eq__(self, other): if not isinstance(other, Link): return NotImplemented @@ -96,6 +100,9 @@ class Link: 'is_static': self.is_static, 'is_archived': self.is_archived, 'num_outputs': self.num_outputs, + 'num_failures': self.num_failures, + 'oldest_archive_date': self.oldest_archive_date, + 'newest_archive_date': self.newest_archive_date, }) return info @@ -152,11 +159,42 @@ class Link: from util import ts_to_date return ts_to_date(self.updated) if self.updated else None + @property + def oldest_archive_date(self) -> Optional[datetime]: + from util import ts_to_date + + most_recent = min( + (result.start_ts + for method in self.history.keys() + for result in self.history[method]), + default=None, + ) + return ts_to_date(most_recent) if most_recent else None + + @property + def newest_archive_date(self) -> Optional[datetime]: + from util import ts_to_date + + most_recent = max( + (result.start_ts + for method in self.history.keys() + for result in self.history[method]), + default=None, + ) + return ts_to_date(most_recent) if most_recent else None + ### Archive Status Helpers @property def num_outputs(self) -> int: return len(tuple(filter(None, self.latest_outputs().values()))) + @property + def num_failures(self) -> int: + return sum(1 + for method in self.history.keys() + for result in self.history[method] + if result.status == 'failed') + @property def is_static(self) -> bool: from util import is_static_file diff --git a/archivebox/templates/index.html b/archivebox/templates/index.html index 264deb4d..f5cf2785 100644 --- a/archivebox/templates/index.html +++ b/archivebox/templates/index.html @@ -13,51 +13,64 @@ padding: 0px; font-family: "Gill Sans", Helvetica, sans-serif; } - header { - background-color: #aa1e55; - color: #1a1a1a; - padding: 10px; - padding-top: 0px; - padding-bottom: 15px; - /*height: 40px;*/ + .header-top small { + font-weight: 200; + color: #efefef; } - header h1 { - margin: 7px 0px; - font-size: 35px; - font-weight: 300; - color: #1a1a1a; - } - header h1 img { - height: 44px; - vertical-align: bottom; - } - header a { - text-decoration: none !important; - color: #1a1a1a; - } - .header-center { - margin: auto; - float: none; + + .header-top { + width: 100%; + height: auto; + min-height: 40px; + margin: 0px; text-align: center; - padding-top: 6px; + color: white; + font-size: calc(11px + 0.86vw); + font-weight: 200; + padding: 4px 4px; + border-bottom: 3px solid #aa1e55; + background-color: #aa1e55; } - .header-center small { - color: #eaeaea; - opacity: 0.7; + input[type=search] { + width: 22vw; + border-radius: 4px; + border: 1px solid #aeaeae; + padding: 3px 5px; } - .header-left { - float: left; + .nav > div { + min-height: 30px; } - .header-right { - float: right; - padding-top: 17px; - padding-right: 10px; + .header-top a { + text-decoration: none; + color: rgba(0,0,0,0.6); } - header + div { - margin-top: 10px; + .header-top a:hover { + text-decoration: none; + color: rgba(0,0,0,0.9); } + .header-top .col-lg-4 { + text-align: center; + padding-top: 4px; + padding-bottom: 4px; + } + .header-archivebox img { + display: inline-block; + margin-right: 3px; + height: 30px; + margin-left: 12px; + margin-top: -4px; + margin-bottom: 2px; + } + .header-archivebox img:hover { + opacity: 0.5; + } + #table-bookmarks_length, #table-bookmarks_filter { - padding: 0px 15px; + padding-top: 12px; + opacity: 0.8; + padding-left: 24px; + padding-right: 22px; + margin-bottom: -16px; } table { padding: 6px; @@ -98,6 +111,9 @@ overflow-y: scroll; table-layout: fixed; } + .dataTables_wrapper { + background-color: #fafafa; + } table tr a span[data-archived~=False] { opacity: 0.4; } @@ -131,7 +147,11 @@ border-radius: 4px; float:right } + input[type=search]::-webkit-search-cancel-button { + -webkit-appearance: searchfield-cancel-button; + } </style> + <link rel="stylesheet" href="static/bootstrap.min.css"> <link rel="stylesheet" href="static/jquery.dataTables.min.css"/> <script src="static/jquery.min.js"></script> <script src="static/jquery.dataTables.min.js"></script> @@ -151,21 +171,20 @@ </head> <body data-status="$status"> <header> - <div class="header-right"> - <a href="https://github.com/pirate/ArchiveBox/wiki">Documentation</a>   |   - <a href="https://github.com/pirate/ArchiveBox">Source</a>   |   - <a href="https://archivebox.io">Website</a> - </div> - <div class="header-left"> - <a href="?" title="Reload..."> - <h1><img src="static/archive.png"/> ArchiveBox: Index</h1> - </a> - </div> - <div class="header-center"> - Archived Sites - <!--<span class="in-progress">(Currently Updating)</span>--> - <br/> - <small>Last updated $time_updated</small> + <div class="header-top container-fluid"> + <div class="row nav"> + <div class="col-lg-6"> + <a href="?" class="header-archivebox" title="Last updated: $time_updated"> + <img src="static/archive.png" alt="Logo"/> + ArchiveBox: Index + </a> + </div> + <div class="col-lg-6"> + <a href="https://github.com/pirate/ArchiveBox/wiki">Documentation</a>   |   + <a href="https://github.com/pirate/ArchiveBox">Source</a>   |   + <a href="https://archivebox.io">Website</a> + </div> + </div> </div> </header> <table id="table-bookmarks"> diff --git a/archivebox/templates/index_row.html b/archivebox/templates/index_row.html index 766f8038..41c6e1ea 100644 --- a/archivebox/templates/index_row.html +++ b/archivebox/templates/index_row.html @@ -4,7 +4,7 @@ <a href="$archive_path/$index_url"><img src="$favicon_url" class="link-favicon" decoding="async"></a> <a href="$archive_path/$archive_url" title="$title"> <span data-title-for="$url" data-archived="$is_archived">$title</span> - <small>$tags</small> + <small style="float:right">$tags</small> </a> </td> <td> diff --git a/archivebox/templates/link_index.html b/archivebox/templates/link_index.html index 95aa6bb1..6309fb14 100644 --- a/archivebox/templates/link_index.html +++ b/archivebox/templates/link_index.html @@ -6,69 +6,72 @@ html, body { width: 100%; height: 100%; - } - body { background-color: #ddd; } header { - width: 100%; - height: 90px; background-color: #aa1e55; + padding-bottom: 12px; + } + small { + font-weight: 200; + } + .header-top { + width: 100%; + height: auto; + min-height: 40px; margin: 0px; text-align: center; color: white; - } - header h1 { - padding-top: 5px; - padding-bottom: 5px; - margin: 0px; + font-size: calc(11px + 0.86vw); font-weight: 200; - font-family: "Gill Sans", Helvetica, sans-serif; - font-size: calc(16px + 1vw); + padding: 4px 4px; + background-color: #aa1e55; } - .collapse-icon { - float: right; - color: black; - width: 126px; - font-size: 0.8em; - margin-top: 20px; - margin-right: 0px; - margin-left: -35px; + .nav > div { + min-height: 30px; + margin: 8px 0px; } - .nav-icon img { - float: left; - display: block; - margin-right: 13px; - color: black; - height: 53px; - margin-top: 12px; - margin-left: 10px; + .header-top a { + text-decoration: none; + color: rgba(0,0,0,0.6); } - .nav-icon img:hover { + .header-top a:hover { + text-decoration: none; + color: rgba(0,0,0,0.9); + } + .header-top .col-lg-4 { + text-align: center; + padding-top: 4px; + padding-bottom: 4px; + } + .header-archivebox img { + display: inline-block; + margin-right: 3px; + height: 30px; + margin-left: 12px; + margin-top: -4px; + margin-bottom: 2px; + } + .header-archivebox img:hover { opacity: 0.5; } - .title-url { - color: black; - display: block; - width: 75%; + .header-url small { white-space: nowrap; - overflow: hidden; - margin: auto; + font-weight: 200; } - .archive-page-header { - margin-top: 5px; + .header-url img { + height: 20px; + vertical-align: -2px; + margin-right: 4px; + } + + .info-row { + margin-top: 2px; margin-bottom: 5px; } - .archive-page-header .alert { + .info-row .alert { margin-bottom: 0px; } - h1 small { - opacity: 0.4; - font-size: 0.6em; - } - h1 small:hover { - opacity: 0.8; - } .card { overflow: hidden; box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02); @@ -87,18 +90,24 @@ max-height: 102px; overflow: hidden; } + .card-title { + margin-bottom: 4px; + } .card-img-top { border: 0px; padding: 0px; margin: 0px; overflow: hidden; opacity: 0.8; - border-top: 1px solid gray; - border-radius: 3px; - border-bottom: 1px solid #ddd; + border-top: 1px solid rgba(0,0,0,0); + border-radius: 4px; + border-bottom: 1px solid rgba(0,0,0,0); height: 430px; - width: 400%; + width: 405%; margin-bottom: -330px; + background-color: #333; + margin-left: -1%; + margin-right: -1%; transform: scale(0.25); transform-origin: 0 0; @@ -116,8 +125,7 @@ box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05); } .iframe-large { - height: 93%; - margin-top: -10px; + height: calc(100% - 40px); } .pdf-frame { transform: none; @@ -125,6 +133,9 @@ height: 160px; margin-top: -60px; margin-bottom: 0px; + transform: scale(1.1); + width: 100%; + margin-left: -10%; } img.external { height: 30px; @@ -138,13 +149,61 @@ border: 4px solid green; } .screenshot { + background-color: #333; transform: none; width: 100%; - height: auto; + min-height: 100px; max-height: 100px; margin-bottom: 0px; object-fit: cover; - object-position: top; + object-position: top center; + } + .header-bottom { + border-top: 1px solid rgba(170, 30, 85, 0.9); + padding-bottom: 12px; + border-bottom: 5px solid rgb(170, 30, 85); + margin-bottom: -1px; + + border-radius: 4px; + background-color: rgba(23, 22, 22, 0.88); + width: 98%; + border: 1px solid rgba(0,0,0,0.2); + box-shadow: 4px 4px 4px rgba(0,0,0,0.2); + margin-top: 5px; + } + .header-bottom-info { + color: #6f6f6f; + padding-top: 8px; + padding-bottom: 13px; + } + + .header-bottom-info > div { + text-align: center; + } + .header-bottom-info h5 { + font-size: 1.1em; + font-weight: 200; + margin-top: 3px; + margin-bottom: 3px; + color: rgba(255, 255, 255, 0.74); + } + .info-chunk { + width: auto; + display:inline-block; + text-align: center; + margin: 10px 10px; + vertical-align: top; + } + .info-chunk .badge { + margin-top: 5px; + } + .header-bottom-frames .card-title { + padding-bottom: 0px; + font-size: 1.2vw; + margin-bottom: 5px; + } + .header-bottom-frames .card-text { + font-size: 0.9em; } @media(max-width: 1092px) { @@ -164,131 +223,170 @@ .card { margin-bottom: 5px; } - header > h1 > a.collapse-icon, header > h1 > a.nav-icon { + header > h1 > a.header-url, header > h1 > a.header-archivebox { display: none; } } </style> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" integrity="sha384-rwoIResjU2yc3z8GV/NPeZWAv56rSmLldC3R/AZzGRnGxQQKnKkoFVhFQhNUwEyJ" crossorigin="anonymous"> + <link rel="stylesheet" href="../../static/bootstrap.min.css"> </head> <body> <header> - <h1 class="page-title"> - <a href="../../index.html" class="nav-icon" title="Go to Main Index..."> - <img src="../../static/archive.png" alt="Archive Icon"> - </a> - <a href="#" class="collapse-icon" style="text-decoration: none" title="Toggle info panel..."> - ▾ - </a> - <img src="$favicon_url" height="20px"> $title<br/> - <a href="$url" class="title-url"> - <small>$base_url</small> - </a> - </h1> - </header> - <div class="site-header container-fluid"> - <div class="row archive-page-header"> - <div class="col-lg-4 alert well"> - Bookmarked: <small title="Timestamp: $timestamp">$bookmarked_date</small> -   |   - Last updated: <small title="Timestamp: $updated">$updated_date</small> -   |   - Total files: <small title="Archive methods">🗃 $num_outputs</small> - </div> - <div class="col-lg-4 alert well"> - Type: - <span class="badge badge-default">$extension</span> -   |   - Tags: - <span class="badge badge-warning">$tags</span> -   |   - Status: - <span class="badge badge-$status_color">$status</span> - </div> - <div class="col-lg-4 alert well"> - Archive Methods: - <a href="index.json" title="JSON summary of archived link.">JSON</a> | - <a href="warc/" title="Any WARC archives for the page">WARC</a> | - <a href="media/" title="Audio, Video, and Subtitle files.">Media</a> | - <a href="git/" title="Any git repos at the url">Git Repos</a> | - <a href="favicon.ico" title="Any git repos at the url">Favicon</a> | - <a href="." title="Webserver-provided index of files directory.">See all files...</a> - </div> - <hr/> - <div class="col-lg-2"> - <div class="card selected-card"> - <iframe class="card-img-top" src="$archive_url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> - <div class="card-body"> - <a href="$archive_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> - <img src="../../static/external.png" class="external"/> + <div class="header-top container-fluid"> + <div class="row nav"> + <div class="col-lg-6"> + <a href="../../index.html" class="header-archivebox" title="Go to Main Index..."> + <img src="../../static/archive.png" alt="Archive Icon"> + ArchiveBox: </a> - <a href="$archive_url" target="preview"><h4 class="card-title">Local Archive</h4></a> - <p class="card-text">archive/$domain</p> - </div> + <a href="#">Page Details</a><br/> + <small style="margin-top: 5px; display: block"> + <a href="../../index.html">Home</a>   |   + <a href="https://github.com/pirate/ArchiveBox">Github</a>   |   + <a href="https://github.com/pirate/ArchiveBox/wiki">Documentation</a> + </small> </div> - </div> - <div class="col-lg-2"> - <div class="card"> - <iframe class="card-img-top" src="$dom_url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> - <div class="card-body"> - <a href="$dom_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> - <img src="../../static/external.png" class="external"/> - </a> - <a href="$dom_url" target="preview"><h4 class="card-title">HTML</h4></a> - <p class="card-text">archive/output.html</p> - </div> - </div> - </div> - <div class="col-lg-2"> - <div class="card"> - <iframe class="card-img-top pdf-frame" src="$pdf_url" scrolling="no"></iframe> - <div class="card-body"> - <a href="$pdf_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> - <img src="../../static/external.png" class="external"/> - </a> - <a href="$pdf_url" target="preview" id="pdf-btn"><h4 class="card-title">PDF</h4></a> - <p class="card-text">archive/output.pdf</p> - </div> - </div> - </div> - <div class="col-lg-2"> - <div class="card"> - <img class="card-img-top screenshot" src="$screenshot_url"></iframe> - <div class="card-body"> - <a href="$screenshot_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> - <img src="../../static/external.png" class="external"/> - </a> - <a href="$screenshot_url" target="preview"><h4 class="card-title">Screenshot</h4></a> - <p class="card-text">archive/screenshot.png</p> - </div> - </div> - </div> - <div class="col-lg-2"> - <div class="card"> - <iframe class="card-img-top" src="$url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> - <div class="card-body"> - <a href="$url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> - <img src="../../static/external.png" class="external"/> - </a> - <a href="$url" target="preview"><h4 class="card-title">Original</h4></a> - <p class="card-text">$domain</p> - </div> - </div> - </div> - <div class="col-lg-2"> - <div class="card"> - <iframe class="card-img-top" src="$archive_org_url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> - <div class="card-body"> - <a href="$archive_org_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> - <img src="../../static/external.png" class="external"/> - </a> - <a href="$archive_org_url" target="preview"><h4 class="card-title">Archive.Org</h4></a> - <p class="card-text">web.archive.org/web/...</p> - </div> + <div class="col-lg-6"> + <img src="$link_dir/$favicon_url" alt="Favicon"> +     + $title +     + <small> + <a href="$url" title="Toggle info panel..." class="header-url" title="$url"> + $base_url + </a> + </small> +     + <a href="#" class="header-toggle">▾</a> </div> </div> </div> - </div> + <div class="header-bottom container-fluid"> + <div class="row header-bottom-info"> + <div class="col-lg-4"> + <div title="Date bookmarked or imported" class="info-chunk"> + <h5>Added</h5> + $bookmarked_date + </div> + <div title="Date first archived" class="info-chunk"> + <h5>First Archived</h5> + $oldest_archive_date + </div> + <div title="Date last checked" class="info-chunk"> + <h5>Last Checked</h5> + $updated_date + </div> + </div> + <div class="col-lg-4"> + <div class="info-chunk"> + <h5>Type</h5> + <div class="badge badge-default">$extension</div> + </div> + <div class="info-chunk"> + <h5>Tags</h5> + <div class="badge badge-warning">$tags</div> + </div> + <div class="info-chunk"> + <h5>Status</h5> + <div class="badge badge-$status_color">$status</div> + </div> + <div class="info-chunk"> + <h5>Saved</h5> + ✅ $num_outputs + </div> + <div class="info-chunk"> + <h5>Errors</h5> + ❌ $num_failures + </div> + </div> + <div class="col-lg-4"> + <div class="info-chunk"> + <h5>🗃 Files</h5> + <a href="index.json" title="JSON summary of archived link.">JSON</a> | + <a href="warc/" title="Any WARC archives for the page">WARC</a> | + <a href="media/" title="Audio, Video, and Subtitle files.">Media</a> | + <a href="git/" title="Any git repos at the url">Git</a> | + <a href="favicon.ico" title="Any git repos at the url">Favicon</a> | + <a href="." title="Webserver-provided index of files directory.">See all...</a> + </div> + </div> + </div> + <div class="row header-bottom-frames"> + <div class="col-lg-2"> + <div class="card selected-card"> + <iframe class="card-img-top" src="$archive_url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> + <div class="card-body"> + <a href="$archive_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> + <img src="../../static/external.png" class="external"/> + </a> + <a href="$archive_url" target="preview"><h4 class="card-title">Local Archive</h4></a> + <p class="card-text">archive/$domain</p> + </div> + </div> + </div> + <div class="col-lg-2"> + <div class="card"> + <iframe class="card-img-top" src="$dom_url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> + <div class="card-body"> + <a href="$dom_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> + <img src="../../static/external.png" class="external"/> + </a> + <a href="$dom_url" target="preview"><h4 class="card-title">HTML</h4></a> + <p class="card-text">archive/output.html</p> + </div> + </div> + </div> + <div class="col-lg-2"> + <div class="card"> + <iframe class="card-img-top pdf-frame" src="$pdf_url" scrolling="no"></iframe> + <div class="card-body"> + <a href="$pdf_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> + <img src="../../static/external.png" class="external"/> + </a> + <a href="$pdf_url" target="preview" id="pdf-btn"><h4 class="card-title">PDF</h4></a> + <p class="card-text">archive/output.pdf</p> + </div> + </div> + </div> + <div class="col-lg-2"> + <div class="card"> + <img class="card-img-top screenshot" src="$screenshot_url"></iframe> + <div class="card-body"> + <a href="$screenshot_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> + <img src="../../static/external.png" class="external"/> + </a> + <a href="$screenshot_url" target="preview"><h4 class="card-title">Screenshot</h4></a> + <p class="card-text">archive/screenshot.png</p> + </div> + </div> + </div> + <div class="col-lg-2"> + <div class="card"> + <iframe class="card-img-top" src="$url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> + <div class="card-body"> + <a href="$url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> + <img src="../../static/external.png" class="external"/> + </a> + <a href="$url" target="preview"><h4 class="card-title">Original</h4></a> + <p class="card-text">$domain</p> + </div> + </div> + </div> + <div class="col-lg-2"> + <div class="card"> + <iframe class="card-img-top" src="$archive_org_url" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe> + <div class="card-body"> + <a href="$archive_org_url" style="float:right" title="Open in new tab..." target="_blank" rel="noopener"> + <img src="../../static/external.png" class="external"/> + </a> + <a href="$archive_org_url" target="preview"><h4 class="card-title">Archive.Org</h4></a> + <p class="card-text">web.archive.org/web/...</p> + </div> + </div> + </div> + </div> + </div> + </header> <iframe sandbox="allow-same-origin allow-scripts allow-forms" class="full-page-iframe" src="$archive_url" name="preview"></iframe> <script @@ -321,14 +419,14 @@ }) // hide header when collapse icon is clicked - jQuery('.collapse-icon').on('click', function() { - if (jQuery('.collapse-icon').text().includes('▾')) { - jQuery('.collapse-icon').text('▸') - jQuery('.site-header').hide() + jQuery('.header-toggle').on('click', function() { + if (jQuery('.header-toggle').text().includes('▾')) { + jQuery('.header-toggle').text('▸') + jQuery('.header-bottom').hide() jQuery('.full-page-iframe').addClass('iframe-large') } else { - jQuery('.collapse-icon').text('▾') - jQuery('.site-header').show() + jQuery('.header-toggle').text('▾') + jQuery('.header-bottom').show() jQuery('.full-page-iframe').removeClass('iframe-large') } return true diff --git a/archivebox/templates/static/bootstrap.min.css b/archivebox/templates/static/bootstrap.min.css new file mode 100644 index 00000000..a8da0748 --- /dev/null +++ b/archivebox/templates/static/bootstrap.min.css @@ -0,0 +1,6 @@ +/*! + * Bootstrap v4.0.0-alpha.6 (https://getbootstrap.com) + * Copyright 2011-2017 The Bootstrap Authors + * Copyright 2011-2017 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + *//*! normalize.css v5.0.0 | MIT License | github.com/necolas/normalize.css */html{font-family:sans-serif;line-height:1.15;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,footer,header,nav,section{display:block}h1{font-size:2em;margin:.67em 0}figcaption,figure,main{display:block}figure{margin:1em 40px}hr{-webkit-box-sizing:content-box;box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent;-webkit-text-decoration-skip:objects}a:active,a:hover{outline-width:0}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:inherit}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}dfn{font-style:italic}mark{background-color:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}audio,video{display:inline-block}audio:not([controls]){display:none;height:0}img{border-style:none}svg:not(:root){overflow:hidden}button,input,optgroup,select,textarea{font-family:sans-serif;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{border-style:none;padding:0}[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring,button:-moz-focusring{outline:1px dotted ButtonText}fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}legend{-webkit-box-sizing:border-box;box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{display:inline-block;vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{-webkit-box-sizing:border-box;box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details,menu{display:block}summary{display:list-item}canvas{display:inline-block}template{display:none}[hidden]{display:none}@media print{*,::after,::before,blockquote::first-letter,blockquote::first-line,div::first-letter,div::first-line,li::first-letter,li::first-line,p::first-letter,p::first-line{text-shadow:none!important;-webkit-box-shadow:none!important;box-shadow:none!important}a,a:visited{text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #ddd!important}}html{-webkit-box-sizing:border-box;box-sizing:border-box}*,::after,::before{-webkit-box-sizing:inherit;box-sizing:inherit}@-ms-viewport{width:device-width}html{-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}body{font-family:-apple-system,system-ui,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-size:1rem;font-weight:400;line-height:1.5;color:#292b2c;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{cursor:help}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}a{color:#0275d8;text-decoration:none}a:focus,a:hover{color:#014c8c;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle}[role=button]{cursor:pointer}[role=button],a,area,button,input,label,select,summary,textarea{-ms-touch-action:manipulation;touch-action:manipulation}table{border-collapse:collapse;background-color:transparent}caption{padding-top:.75rem;padding-bottom:.75rem;color:#636c72;text-align:left;caption-side:bottom}th{text-align:left}label{display:inline-block;margin-bottom:.5rem}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,select,textarea{line-height:inherit}input[type=checkbox]:disabled,input[type=radio]:disabled{cursor:not-allowed}input[type=date],input[type=time],input[type=datetime-local],input[type=month]{-webkit-appearance:listbox}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit}input[type=search]{-webkit-appearance:none}output{display:inline-block}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-family:inherit;font-weight:500;line-height:1.1;color:inherit}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.1}.display-2{font-size:5.5rem;font-weight:300;line-height:1.1}.display-3{font-size:4.5rem;font-weight:300;line-height:1.1}.display-4{font-size:3.5rem;font-weight:300;line-height:1.1}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:5px}.initialism{font-size:90%;text-transform:uppercase}.blockquote{padding:.5rem 1rem;margin-bottom:1rem;font-size:1.25rem;border-left:.25rem solid #eceeef}.blockquote-footer{display:block;font-size:80%;color:#636c72}.blockquote-footer::before{content:"\2014 \00A0"}.blockquote-reverse{padding-right:1rem;padding-left:0;text-align:right;border-right:.25rem solid #eceeef;border-left:0}.blockquote-reverse .blockquote-footer::before{content:""}.blockquote-reverse .blockquote-footer::after{content:"\00A0 \2014"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #ddd;border-radius:.25rem;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#636c72}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}code{padding:.2rem .4rem;font-size:90%;color:#bd4147;background-color:#f7f7f9;border-radius:.25rem}a>code{padding:0;color:inherit;background-color:inherit}kbd{padding:.2rem .4rem;font-size:90%;color:#fff;background-color:#292b2c;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;margin-top:0;margin-bottom:1rem;font-size:90%;color:#292b2c}pre code{padding:0;font-size:inherit;color:inherit;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{position:relative;margin-left:auto;margin-right:auto;padding-right:15px;padding-left:15px}@media (min-width:576px){.container{padding-right:15px;padding-left:15px}}@media (min-width:768px){.container{padding-right:15px;padding-left:15px}}@media (min-width:992px){.container{padding-right:15px;padding-left:15px}}@media (min-width:1200px){.container{padding-right:15px;padding-left:15px}}@media (min-width:576px){.container{width:540px;max-width:100%}}@media (min-width:768px){.container{width:720px;max-width:100%}}@media (min-width:992px){.container{width:960px;max-width:100%}}@media (min-width:1200px){.container{width:1140px;max-width:100%}}.container-fluid{position:relative;margin-left:auto;margin-right:auto;padding-right:15px;padding-left:15px}@media (min-width:576px){.container-fluid{padding-right:15px;padding-left:15px}}@media (min-width:768px){.container-fluid{padding-right:15px;padding-left:15px}}@media (min-width:992px){.container-fluid{padding-right:15px;padding-left:15px}}@media (min-width:1200px){.container-fluid{padding-right:15px;padding-left:15px}}.row{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:wrap;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}@media (min-width:576px){.row{margin-right:-15px;margin-left:-15px}}@media (min-width:768px){.row{margin-right:-15px;margin-left:-15px}}@media (min-width:992px){.row{margin-right:-15px;margin-left:-15px}}@media (min-width:1200px){.row{margin-right:-15px;margin-left:-15px}}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{position:relative;width:100%;min-height:1px;padding-right:15px;padding-left:15px}@media (min-width:576px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}@media (min-width:768px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}@media (min-width:992px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}@media (min-width:1200px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}.col{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-0{right:auto}.pull-1{right:8.333333%}.pull-2{right:16.666667%}.pull-3{right:25%}.pull-4{right:33.333333%}.pull-5{right:41.666667%}.pull-6{right:50%}.pull-7{right:58.333333%}.pull-8{right:66.666667%}.pull-9{right:75%}.pull-10{right:83.333333%}.pull-11{right:91.666667%}.pull-12{right:100%}.push-0{left:auto}.push-1{left:8.333333%}.push-2{left:16.666667%}.push-3{left:25%}.push-4{left:33.333333%}.push-5{left:41.666667%}.push-6{left:50%}.push-7{left:58.333333%}.push-8{left:66.666667%}.push-9{left:75%}.push-10{left:83.333333%}.push-11{left:91.666667%}.push-12{left:100%}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-sm-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-sm-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-sm-0{right:auto}.pull-sm-1{right:8.333333%}.pull-sm-2{right:16.666667%}.pull-sm-3{right:25%}.pull-sm-4{right:33.333333%}.pull-sm-5{right:41.666667%}.pull-sm-6{right:50%}.pull-sm-7{right:58.333333%}.pull-sm-8{right:66.666667%}.pull-sm-9{right:75%}.pull-sm-10{right:83.333333%}.pull-sm-11{right:91.666667%}.pull-sm-12{right:100%}.push-sm-0{left:auto}.push-sm-1{left:8.333333%}.push-sm-2{left:16.666667%}.push-sm-3{left:25%}.push-sm-4{left:33.333333%}.push-sm-5{left:41.666667%}.push-sm-6{left:50%}.push-sm-7{left:58.333333%}.push-sm-8{left:66.666667%}.push-sm-9{left:75%}.push-sm-10{left:83.333333%}.push-sm-11{left:91.666667%}.push-sm-12{left:100%}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-md-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-md-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-md-0{right:auto}.pull-md-1{right:8.333333%}.pull-md-2{right:16.666667%}.pull-md-3{right:25%}.pull-md-4{right:33.333333%}.pull-md-5{right:41.666667%}.pull-md-6{right:50%}.pull-md-7{right:58.333333%}.pull-md-8{right:66.666667%}.pull-md-9{right:75%}.pull-md-10{right:83.333333%}.pull-md-11{right:91.666667%}.pull-md-12{right:100%}.push-md-0{left:auto}.push-md-1{left:8.333333%}.push-md-2{left:16.666667%}.push-md-3{left:25%}.push-md-4{left:33.333333%}.push-md-5{left:41.666667%}.push-md-6{left:50%}.push-md-7{left:58.333333%}.push-md-8{left:66.666667%}.push-md-9{left:75%}.push-md-10{left:83.333333%}.push-md-11{left:91.666667%}.push-md-12{left:100%}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-lg-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-lg-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-lg-0{right:auto}.pull-lg-1{right:8.333333%}.pull-lg-2{right:16.666667%}.pull-lg-3{right:25%}.pull-lg-4{right:33.333333%}.pull-lg-5{right:41.666667%}.pull-lg-6{right:50%}.pull-lg-7{right:58.333333%}.pull-lg-8{right:66.666667%}.pull-lg-9{right:75%}.pull-lg-10{right:83.333333%}.pull-lg-11{right:91.666667%}.pull-lg-12{right:100%}.push-lg-0{left:auto}.push-lg-1{left:8.333333%}.push-lg-2{left:16.666667%}.push-lg-3{left:25%}.push-lg-4{left:33.333333%}.push-lg-5{left:41.666667%}.push-lg-6{left:50%}.push-lg-7{left:58.333333%}.push-lg-8{left:66.666667%}.push-lg-9{left:75%}.push-lg-10{left:83.333333%}.push-lg-11{left:91.666667%}.push-lg-12{left:100%}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-xl-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-xl-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-xl-0{right:auto}.pull-xl-1{right:8.333333%}.pull-xl-2{right:16.666667%}.pull-xl-3{right:25%}.pull-xl-4{right:33.333333%}.pull-xl-5{right:41.666667%}.pull-xl-6{right:50%}.pull-xl-7{right:58.333333%}.pull-xl-8{right:66.666667%}.pull-xl-9{right:75%}.pull-xl-10{right:83.333333%}.pull-xl-11{right:91.666667%}.pull-xl-12{right:100%}.push-xl-0{left:auto}.push-xl-1{left:8.333333%}.push-xl-2{left:16.666667%}.push-xl-3{left:25%}.push-xl-4{left:33.333333%}.push-xl-5{left:41.666667%}.push-xl-6{left:50%}.push-xl-7{left:58.333333%}.push-xl-8{left:66.666667%}.push-xl-9{left:75%}.push-xl-10{left:83.333333%}.push-xl-11{left:91.666667%}.push-xl-12{left:100%}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;max-width:100%;margin-bottom:1rem}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #eceeef}.table thead th{vertical-align:bottom;border-bottom:2px solid #eceeef}.table tbody+tbody{border-top:2px solid #eceeef}.table .table{background-color:#fff}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #eceeef}.table-bordered td,.table-bordered th{border:1px solid #eceeef}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{background-color:rgba(0,0,0,.075)}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table-success,.table-success>td,.table-success>th{background-color:#dff0d8}.table-hover .table-success:hover{background-color:#d0e9c6}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#d0e9c6}.table-info,.table-info>td,.table-info>th{background-color:#d9edf7}.table-hover .table-info:hover{background-color:#c4e3f3}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#c4e3f3}.table-warning,.table-warning>td,.table-warning>th{background-color:#fcf8e3}.table-hover .table-warning:hover{background-color:#faf2cc}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#faf2cc}.table-danger,.table-danger>td,.table-danger>th{background-color:#f2dede}.table-hover .table-danger:hover{background-color:#ebcccc}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#ebcccc}.thead-inverse th{color:#fff;background-color:#292b2c}.thead-default th{color:#464a4c;background-color:#eceeef}.table-inverse{color:#fff;background-color:#292b2c}.table-inverse td,.table-inverse th,.table-inverse thead th{border-color:#fff}.table-inverse.table-bordered{border:0}.table-responsive{display:block;width:100%;overflow-x:auto;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive.table-bordered{border:0}.form-control{display:block;width:100%;padding:.5rem .75rem;font-size:1rem;line-height:1.25;color:#464a4c;background-color:#fff;background-image:none;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem;-webkit-transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s,-webkit-box-shadow ease-in-out .15s}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:focus{color:#464a4c;background-color:#fff;border-color:#5cb3fd;outline:0}.form-control::-webkit-input-placeholder{color:#636c72;opacity:1}.form-control::-moz-placeholder{color:#636c72;opacity:1}.form-control:-ms-input-placeholder{color:#636c72;opacity:1}.form-control::placeholder{color:#636c72;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#eceeef;opacity:1}.form-control:disabled{cursor:not-allowed}select.form-control:not([size]):not([multiple]){height:calc(2.25rem + 2px)}select.form-control:focus::-ms-value{color:#464a4c;background-color:#fff}.form-control-file,.form-control-range{display:block}.col-form-label{padding-top:calc(.5rem - 1px * 2);padding-bottom:calc(.5rem - 1px * 2);margin-bottom:0}.col-form-label-lg{padding-top:calc(.75rem - 1px * 2);padding-bottom:calc(.75rem - 1px * 2);font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem - 1px * 2);padding-bottom:calc(.25rem - 1px * 2);font-size:.875rem}.col-form-legend{padding-top:.5rem;padding-bottom:.5rem;margin-bottom:0;font-size:1rem}.form-control-static{padding-top:.5rem;padding-bottom:.5rem;margin-bottom:0;line-height:1.25;border:solid transparent;border-width:1px 0}.form-control-static.form-control-lg,.form-control-static.form-control-sm,.input-group-lg>.form-control-static.form-control,.input-group-lg>.form-control-static.input-group-addon,.input-group-lg>.input-group-btn>.form-control-static.btn,.input-group-sm>.form-control-static.form-control,.input-group-sm>.form-control-static.input-group-addon,.input-group-sm>.input-group-btn>.form-control-static.btn{padding-right:0;padding-left:0}.form-control-sm,.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-sm>.input-group-btn>select.btn:not([size]):not([multiple]),.input-group-sm>select.form-control:not([size]):not([multiple]),.input-group-sm>select.input-group-addon:not([size]):not([multiple]),select.form-control-sm:not([size]):not([multiple]){height:1.8125rem}.form-control-lg,.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{padding:.75rem 1.5rem;font-size:1.25rem;border-radius:.3rem}.input-group-lg>.input-group-btn>select.btn:not([size]):not([multiple]),.input-group-lg>select.form-control:not([size]):not([multiple]),.input-group-lg>select.input-group-addon:not([size]):not([multiple]),select.form-control-lg:not([size]):not([multiple]){height:3.166667rem}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-check{position:relative;display:block;margin-bottom:.5rem}.form-check.disabled .form-check-label{color:#636c72;cursor:not-allowed}.form-check-label{padding-left:1.25rem;margin-bottom:0;cursor:pointer}.form-check-input{position:absolute;margin-top:.25rem;margin-left:-1.25rem}.form-check-input:only-child{position:static}.form-check-inline{display:inline-block}.form-check-inline .form-check-label{vertical-align:middle}.form-check-inline+.form-check-inline{margin-left:.75rem}.form-control-feedback{margin-top:.25rem}.form-control-danger,.form-control-success,.form-control-warning{padding-right:2.25rem;background-repeat:no-repeat;background-position:center right .5625rem;-webkit-background-size:1.125rem 1.125rem;background-size:1.125rem 1.125rem}.has-success .col-form-label,.has-success .custom-control,.has-success .form-check-label,.has-success .form-control-feedback,.has-success .form-control-label{color:#5cb85c}.has-success .form-control{border-color:#5cb85c}.has-success .input-group-addon{color:#5cb85c;border-color:#5cb85c;background-color:#eaf6ea}.has-success .form-control-success{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%235cb85c' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3E%3C/svg%3E")}.has-warning .col-form-label,.has-warning .custom-control,.has-warning .form-check-label,.has-warning .form-control-feedback,.has-warning .form-control-label{color:#f0ad4e}.has-warning .form-control{border-color:#f0ad4e}.has-warning .input-group-addon{color:#f0ad4e;border-color:#f0ad4e;background-color:#fff}.has-warning .form-control-warning{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%23f0ad4e' d='M4.4 5.324h-.8v-2.46h.8zm0 1.42h-.8V5.89h.8zM3.76.63L.04 7.075c-.115.2.016.425.26.426h7.397c.242 0 .372-.226.258-.426C6.726 4.924 5.47 2.79 4.253.63c-.113-.174-.39-.174-.494 0z'/%3E%3C/svg%3E")}.has-danger .col-form-label,.has-danger .custom-control,.has-danger .form-check-label,.has-danger .form-control-feedback,.has-danger .form-control-label{color:#d9534f}.has-danger .form-control{border-color:#d9534f}.has-danger .input-group-addon{color:#d9534f;border-color:#d9534f;background-color:#fdf7f7}.has-danger .form-control-danger{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23d9534f' viewBox='-2 -2 7 7'%3E%3Cpath stroke='%23d9534f' d='M0 0l3 3m0-3L0 3'/%3E%3Ccircle r='.5'/%3E%3Ccircle cx='3' r='.5'/%3E%3Ccircle cy='3' r='.5'/%3E%3Ccircle cx='3' cy='3' r='.5'/%3E%3C/svg%3E")}.form-inline{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{width:auto}.form-inline .form-control-label{margin-bottom:0;vertical-align:middle}.form-inline .form-check{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;width:auto;margin-top:0;margin-bottom:0}.form-inline .form-check-label{padding-left:0}.form-inline .form-check-input{position:relative;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;padding-left:0}.form-inline .custom-control-indicator{position:static;display:inline-block;margin-right:.25rem;vertical-align:text-bottom}.form-inline .has-feedback .form-control-feedback{top:0}}.btn{display:inline-block;font-weight:400;line-height:1.25;text-align:center;white-space:nowrap;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;border:1px solid transparent;padding:.5rem 1rem;font-size:1rem;border-radius:.25rem;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.btn:focus,.btn:hover{text-decoration:none}.btn.focus,.btn:focus{outline:0;-webkit-box-shadow:0 0 0 2px rgba(2,117,216,.25);box-shadow:0 0 0 2px rgba(2,117,216,.25)}.btn.disabled,.btn:disabled{cursor:not-allowed;opacity:.65}.btn.active,.btn:active{background-image:none}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#0275d8;border-color:#0275d8}.btn-primary:hover{color:#fff;background-color:#025aa5;border-color:#01549b}.btn-primary.focus,.btn-primary:focus{-webkit-box-shadow:0 0 0 2px rgba(2,117,216,.5);box-shadow:0 0 0 2px rgba(2,117,216,.5)}.btn-primary.disabled,.btn-primary:disabled{background-color:#0275d8;border-color:#0275d8}.btn-primary.active,.btn-primary:active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#025aa5;background-image:none;border-color:#01549b}.btn-secondary{color:#292b2c;background-color:#fff;border-color:#ccc}.btn-secondary:hover{color:#292b2c;background-color:#e6e6e6;border-color:#adadad}.btn-secondary.focus,.btn-secondary:focus{-webkit-box-shadow:0 0 0 2px rgba(204,204,204,.5);box-shadow:0 0 0 2px rgba(204,204,204,.5)}.btn-secondary.disabled,.btn-secondary:disabled{background-color:#fff;border-color:#ccc}.btn-secondary.active,.btn-secondary:active,.show>.btn-secondary.dropdown-toggle{color:#292b2c;background-color:#e6e6e6;background-image:none;border-color:#adadad}.btn-info{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#2aabd2}.btn-info.focus,.btn-info:focus{-webkit-box-shadow:0 0 0 2px rgba(91,192,222,.5);box-shadow:0 0 0 2px rgba(91,192,222,.5)}.btn-info.disabled,.btn-info:disabled{background-color:#5bc0de;border-color:#5bc0de}.btn-info.active,.btn-info:active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#31b0d5;background-image:none;border-color:#2aabd2}.btn-success{color:#fff;background-color:#5cb85c;border-color:#5cb85c}.btn-success:hover{color:#fff;background-color:#449d44;border-color:#419641}.btn-success.focus,.btn-success:focus{-webkit-box-shadow:0 0 0 2px rgba(92,184,92,.5);box-shadow:0 0 0 2px rgba(92,184,92,.5)}.btn-success.disabled,.btn-success:disabled{background-color:#5cb85c;border-color:#5cb85c}.btn-success.active,.btn-success:active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#449d44;background-image:none;border-color:#419641}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#f0ad4e}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#eb9316}.btn-warning.focus,.btn-warning:focus{-webkit-box-shadow:0 0 0 2px rgba(240,173,78,.5);box-shadow:0 0 0 2px rgba(240,173,78,.5)}.btn-warning.disabled,.btn-warning:disabled{background-color:#f0ad4e;border-color:#f0ad4e}.btn-warning.active,.btn-warning:active,.show>.btn-warning.dropdown-toggle{color:#fff;background-color:#ec971f;background-image:none;border-color:#eb9316}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d9534f}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#c12e2a}.btn-danger.focus,.btn-danger:focus{-webkit-box-shadow:0 0 0 2px rgba(217,83,79,.5);box-shadow:0 0 0 2px rgba(217,83,79,.5)}.btn-danger.disabled,.btn-danger:disabled{background-color:#d9534f;border-color:#d9534f}.btn-danger.active,.btn-danger:active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#c9302c;background-image:none;border-color:#c12e2a}.btn-outline-primary{color:#0275d8;background-image:none;background-color:transparent;border-color:#0275d8}.btn-outline-primary:hover{color:#fff;background-color:#0275d8;border-color:#0275d8}.btn-outline-primary.focus,.btn-outline-primary:focus{-webkit-box-shadow:0 0 0 2px rgba(2,117,216,.5);box-shadow:0 0 0 2px rgba(2,117,216,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#0275d8;background-color:transparent}.btn-outline-primary.active,.btn-outline-primary:active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#0275d8;border-color:#0275d8}.btn-outline-secondary{color:#ccc;background-image:none;background-color:transparent;border-color:#ccc}.btn-outline-secondary:hover{color:#fff;background-color:#ccc;border-color:#ccc}.btn-outline-secondary.focus,.btn-outline-secondary:focus{-webkit-box-shadow:0 0 0 2px rgba(204,204,204,.5);box-shadow:0 0 0 2px rgba(204,204,204,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#ccc;background-color:transparent}.btn-outline-secondary.active,.btn-outline-secondary:active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#ccc;border-color:#ccc}.btn-outline-info{color:#5bc0de;background-image:none;background-color:transparent;border-color:#5bc0de}.btn-outline-info:hover{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.btn-outline-info.focus,.btn-outline-info:focus{-webkit-box-shadow:0 0 0 2px rgba(91,192,222,.5);box-shadow:0 0 0 2px rgba(91,192,222,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#5bc0de;background-color:transparent}.btn-outline-info.active,.btn-outline-info:active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.btn-outline-success{color:#5cb85c;background-image:none;background-color:transparent;border-color:#5cb85c}.btn-outline-success:hover{color:#fff;background-color:#5cb85c;border-color:#5cb85c}.btn-outline-success.focus,.btn-outline-success:focus{-webkit-box-shadow:0 0 0 2px rgba(92,184,92,.5);box-shadow:0 0 0 2px rgba(92,184,92,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#5cb85c;background-color:transparent}.btn-outline-success.active,.btn-outline-success:active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#5cb85c;border-color:#5cb85c}.btn-outline-warning{color:#f0ad4e;background-image:none;background-color:transparent;border-color:#f0ad4e}.btn-outline-warning:hover{color:#fff;background-color:#f0ad4e;border-color:#f0ad4e}.btn-outline-warning.focus,.btn-outline-warning:focus{-webkit-box-shadow:0 0 0 2px rgba(240,173,78,.5);box-shadow:0 0 0 2px rgba(240,173,78,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#f0ad4e;background-color:transparent}.btn-outline-warning.active,.btn-outline-warning:active,.show>.btn-outline-warning.dropdown-toggle{color:#fff;background-color:#f0ad4e;border-color:#f0ad4e}.btn-outline-danger{color:#d9534f;background-image:none;background-color:transparent;border-color:#d9534f}.btn-outline-danger:hover{color:#fff;background-color:#d9534f;border-color:#d9534f}.btn-outline-danger.focus,.btn-outline-danger:focus{-webkit-box-shadow:0 0 0 2px rgba(217,83,79,.5);box-shadow:0 0 0 2px rgba(217,83,79,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#d9534f;background-color:transparent}.btn-outline-danger.active,.btn-outline-danger:active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#d9534f;border-color:#d9534f}.btn-link{font-weight:400;color:#0275d8;border-radius:0}.btn-link,.btn-link.active,.btn-link:active,.btn-link:disabled{background-color:transparent}.btn-link,.btn-link:active,.btn-link:focus{border-color:transparent}.btn-link:hover{border-color:transparent}.btn-link:focus,.btn-link:hover{color:#014c8c;text-decoration:underline;background-color:transparent}.btn-link:disabled{color:#636c72}.btn-link:disabled:focus,.btn-link:disabled:hover{text-decoration:none}.btn-group-lg>.btn,.btn-lg{padding:.75rem 1.5rem;font-size:1.25rem;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.show{opacity:1}.collapse{display:none}.collapse.show{display:block}tr.collapse.show{display:table-row}tbody.collapse.show{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.dropdown,.dropup{position:relative}.dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:.3em;vertical-align:middle;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-left:.3em solid transparent}.dropdown-toggle:focus{outline:0}.dropup .dropdown-toggle::after{border-top:0;border-bottom:.3em solid}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#292b2c;text-align:left;list-style:none;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-divider{height:1px;margin:.5rem 0;overflow:hidden;background-color:#eceeef}.dropdown-item{display:block;width:100%;padding:3px 1.5rem;clear:both;font-weight:400;color:#292b2c;text-align:inherit;white-space:nowrap;background:0 0;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#1d1e1f;text-decoration:none;background-color:#f7f7f9}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#0275d8}.dropdown-item.disabled,.dropdown-item:disabled{color:#636c72;cursor:not-allowed;background-color:transparent}.show>.dropdown-menu{display:block}.show>a{outline:0}.dropdown-menu-right{right:0;left:auto}.dropdown-menu-left{right:auto;left:0}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#636c72;white-space:nowrap}.dropdown-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:990}.dropup .dropdown-menu{top:auto;bottom:100%;margin-bottom:.125rem}.btn-group,.btn-group-vertical{position:relative;display:-webkit-inline-box;display:-webkit-inline-flex;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-webkit-box-flex:0;-webkit-flex:0 1 auto;-ms-flex:0 1 auto;flex:0 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:2}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group,.btn-group-vertical .btn+.btn,.btn-group-vertical .btn+.btn-group,.btn-group-vertical .btn-group+.btn,.btn-group-vertical .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-bottom-left-radius:0;border-top-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn+.dropdown-toggle-split::after{margin-left:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:1.125rem;padding-left:1.125rem}.btn-group-vertical{display:-webkit-inline-box;display:-webkit-inline-flex;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-box-align:start;-webkit-align-items:flex-start;-ms-flex-align:start;align-items:flex-start;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center}.btn-group-vertical .btn,.btn-group-vertical .btn-group{width:100%}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-right-radius:0;border-top-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-right-radius:0;border-top-left-radius:0}[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;width:100%}.input-group .form-control{position:relative;z-index:2;-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;margin-bottom:0}.input-group .form-control:active,.input-group .form-control:focus,.input-group .form-control:hover{z-index:3}.input-group .form-control,.input-group-addon,.input-group-btn{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center}.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{white-space:nowrap;vertical-align:middle}.input-group-addon{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.25;color:#464a4c;text-align:center;background-color:#eceeef;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.input-group-addon.form-control-sm,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.input-group-addon.btn{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-addon.form-control-lg,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.input-group-addon.btn{padding:.75rem 1.5rem;font-size:1.25rem;border-radius:.3rem}.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0}.input-group .form-control:not(:last-child),.input-group-addon:not(:last-child),.input-group-btn:not(:first-child)>.btn-group:not(:last-child)>.btn,.input-group-btn:not(:first-child)>.btn:not(:last-child):not(.dropdown-toggle),.input-group-btn:not(:last-child)>.btn,.input-group-btn:not(:last-child)>.btn-group>.btn,.input-group-btn:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.input-group-addon:not(:last-child){border-right:0}.input-group .form-control:not(:first-child),.input-group-addon:not(:first-child),.input-group-btn:not(:first-child)>.btn,.input-group-btn:not(:first-child)>.btn-group>.btn,.input-group-btn:not(:first-child)>.dropdown-toggle,.input-group-btn:not(:last-child)>.btn-group:not(:first-child)>.btn,.input-group-btn:not(:last-child)>.btn:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.form-control+.input-group-addon:not(:first-child){border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative;-webkit-box-flex:1;-webkit-flex:1 1 0%;-ms-flex:1 1 0%;flex:1 1 0%}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:3}.input-group-btn:not(:last-child)>.btn,.input-group-btn:not(:last-child)>.btn-group{margin-right:-1px}.input-group-btn:not(:first-child)>.btn,.input-group-btn:not(:first-child)>.btn-group{z-index:2;margin-left:-1px}.input-group-btn:not(:first-child)>.btn-group:active,.input-group-btn:not(:first-child)>.btn-group:focus,.input-group-btn:not(:first-child)>.btn-group:hover,.input-group-btn:not(:first-child)>.btn:active,.input-group-btn:not(:first-child)>.btn:focus,.input-group-btn:not(:first-child)>.btn:hover{z-index:3}.custom-control{position:relative;display:-webkit-inline-box;display:-webkit-inline-flex;display:-ms-inline-flexbox;display:inline-flex;min-height:1.5rem;padding-left:1.5rem;margin-right:1rem;cursor:pointer}.custom-control-input{position:absolute;z-index:-1;opacity:0}.custom-control-input:checked~.custom-control-indicator{color:#fff;background-color:#0275d8}.custom-control-input:focus~.custom-control-indicator{-webkit-box-shadow:0 0 0 1px #fff,0 0 0 3px #0275d8;box-shadow:0 0 0 1px #fff,0 0 0 3px #0275d8}.custom-control-input:active~.custom-control-indicator{color:#fff;background-color:#8fcafe}.custom-control-input:disabled~.custom-control-indicator{cursor:not-allowed;background-color:#eceeef}.custom-control-input:disabled~.custom-control-description{color:#636c72;cursor:not-allowed}.custom-control-indicator{position:absolute;top:.25rem;left:0;display:block;width:1rem;height:1rem;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#ddd;background-repeat:no-repeat;background-position:center center;-webkit-background-size:50% 50%;background-size:50% 50%}.custom-checkbox .custom-control-indicator{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-indicator{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26 2.974 7.25 8 2.193z'/%3E%3C/svg%3E")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-indicator{background-color:#0275d8;background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 4'%3E%3Cpath stroke='%23fff' d='M0 2h4'/%3E%3C/svg%3E")}.custom-radio .custom-control-indicator{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-indicator{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3E%3Ccircle r='3' fill='%23fff'/%3E%3C/svg%3E")}.custom-controls-stacked{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column}.custom-controls-stacked .custom-control{margin-bottom:.25rem}.custom-controls-stacked .custom-control+.custom-control{margin-left:0}.custom-select{display:inline-block;max-width:100%;height:calc(2.25rem + 2px);padding:.375rem 1.75rem .375rem .75rem;line-height:1.25;color:#464a4c;vertical-align:middle;background:#fff url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3E%3Cpath fill='%23333' d='M2 0L0 2h4zm0 5L0 3h4z'/%3E%3C/svg%3E") no-repeat right .75rem center;-webkit-background-size:8px 10px;background-size:8px 10px;border:1px solid rgba(0,0,0,.15);border-radius:.25rem;-moz-appearance:none;-webkit-appearance:none}.custom-select:focus{border-color:#5cb3fd;outline:0}.custom-select:focus::-ms-value{color:#464a4c;background-color:#fff}.custom-select:disabled{color:#636c72;cursor:not-allowed;background-color:#eceeef}.custom-select::-ms-expand{opacity:0}.custom-select-sm{padding-top:.375rem;padding-bottom:.375rem;font-size:75%}.custom-file{position:relative;display:inline-block;max-width:100%;height:2.5rem;margin-bottom:0;cursor:pointer}.custom-file-input{min-width:14rem;max-width:100%;height:2.5rem;margin:0;filter:alpha(opacity=0);opacity:0}.custom-file-control{position:absolute;top:0;right:0;left:0;z-index:5;height:2.5rem;padding:.5rem 1rem;line-height:1.5;color:#464a4c;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#fff;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.custom-file-control:lang(en)::after{content:"Choose file..."}.custom-file-control::before{position:absolute;top:-1px;right:-1px;bottom:-1px;z-index:6;display:block;height:2.5rem;padding:.5rem 1rem;line-height:1.5;color:#464a4c;background-color:#eceeef;border:1px solid rgba(0,0,0,.15);border-radius:0 .25rem .25rem 0}.custom-file-control:lang(en)::before{content:"Browse"}.nav{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5em 1em}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#636c72;cursor:not-allowed}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-right-radius:.25rem;border-top-left-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#eceeef #eceeef #ddd}.nav-tabs .nav-link.disabled{color:#636c72;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#464a4c;background-color:#fff;border-color:#ddd #ddd #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-right-radius:0;border-top-left-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-item.show .nav-link,.nav-pills .nav-link.active{color:#fff;cursor:default;background-color:#0275d8}.nav-fill .nav-item{-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-webkit-box-flex:1;-webkit-flex:1 1 100%;-ms-flex:1 1 100%;flex:1 1 100%;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;padding:.5rem 1rem}.navbar-brand{display:inline-block;padding-top:.25rem;padding-bottom:.25rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-text{display:inline-block;padding-top:.425rem;padding-bottom:.425rem}.navbar-toggler{-webkit-align-self:flex-start;-ms-flex-item-align:start;align-self:flex-start;padding:.25rem .75rem;font-size:1.25rem;line-height:1;background:0 0;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;-webkit-background-size:100% 100%;background-size:100% 100%}.navbar-toggler-left{position:absolute;left:1rem}.navbar-toggler-right{position:absolute;right:1rem}@media (max-width:575px){.navbar-toggleable .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable>.container{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-toggleable{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable .navbar-toggler{display:none}}@media (max-width:767px){.navbar-toggleable-sm .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-sm>.container{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-toggleable-sm{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-sm .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-sm>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-sm .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-sm .navbar-toggler{display:none}}@media (max-width:991px){.navbar-toggleable-md .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-md>.container{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-toggleable-md{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-md .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-md>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-md .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-md .navbar-toggler{display:none}}@media (max-width:1199px){.navbar-toggleable-lg .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-lg>.container{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-toggleable-lg{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-lg .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-lg>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-lg .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-lg .navbar-toggler{display:none}}.navbar-toggleable-xl{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-xl .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-xl>.container{padding-right:0;padding-left:0}.navbar-toggleable-xl .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-xl>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-xl .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-xl .navbar-toggler{display:none}.navbar-light .navbar-brand,.navbar-light .navbar-toggler{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover,.navbar-light .navbar-toggler:focus,.navbar-light .navbar-toggler:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.open,.navbar-light .navbar-nav .open>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 32 32' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(0, 0, 0, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 8h24M4 16h24M4 24h24'/%3E%3C/svg%3E")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-inverse .navbar-brand,.navbar-inverse .navbar-toggler{color:#fff}.navbar-inverse .navbar-brand:focus,.navbar-inverse .navbar-brand:hover,.navbar-inverse .navbar-toggler:focus,.navbar-inverse .navbar-toggler:hover{color:#fff}.navbar-inverse .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-inverse .navbar-nav .nav-link:focus,.navbar-inverse .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-inverse .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-inverse .navbar-nav .active>.nav-link,.navbar-inverse .navbar-nav .nav-link.active,.navbar-inverse .navbar-nav .nav-link.open,.navbar-inverse .navbar-nav .open>.nav-link{color:#fff}.navbar-inverse .navbar-toggler{border-color:rgba(255,255,255,.1)}.navbar-inverse .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 32 32' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(255, 255, 255, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 8h24M4 16h24M4 24h24'/%3E%3C/svg%3E")}.navbar-inverse .navbar-text{color:rgba(255,255,255,.5)}.card{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;background-color:#fff;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card-block{-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card>.list-group:first-child .list-group-item:first-child{border-top-right-radius:.25rem;border-top-left-radius:.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:#f7f7f9;border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-footer{padding:.75rem 1.25rem;background-color:#f7f7f9;border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-primary{background-color:#0275d8;border-color:#0275d8}.card-primary .card-footer,.card-primary .card-header{background-color:transparent}.card-success{background-color:#5cb85c;border-color:#5cb85c}.card-success .card-footer,.card-success .card-header{background-color:transparent}.card-info{background-color:#5bc0de;border-color:#5bc0de}.card-info .card-footer,.card-info .card-header{background-color:transparent}.card-warning{background-color:#f0ad4e;border-color:#f0ad4e}.card-warning .card-footer,.card-warning .card-header{background-color:transparent}.card-danger{background-color:#d9534f;border-color:#d9534f}.card-danger .card-footer,.card-danger .card-header{background-color:transparent}.card-outline-primary{background-color:transparent;border-color:#0275d8}.card-outline-secondary{background-color:transparent;border-color:#ccc}.card-outline-info{background-color:transparent;border-color:#5bc0de}.card-outline-success{background-color:transparent;border-color:#5cb85c}.card-outline-warning{background-color:transparent;border-color:#f0ad4e}.card-outline-danger{background-color:transparent;border-color:#d9534f}.card-inverse{color:rgba(255,255,255,.65)}.card-inverse .card-footer,.card-inverse .card-header{background-color:transparent;border-color:rgba(255,255,255,.2)}.card-inverse .card-blockquote,.card-inverse .card-footer,.card-inverse .card-header,.card-inverse .card-title{color:#fff}.card-inverse .card-blockquote .blockquote-footer,.card-inverse .card-link,.card-inverse .card-subtitle,.card-inverse .card-text{color:rgba(255,255,255,.65)}.card-inverse .card-link:focus,.card-inverse .card-link:hover{color:#fff}.card-blockquote{padding:0;margin-bottom:0;border-left:0}.card-img{border-radius:calc(.25rem - 1px)}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img-top{border-top-right-radius:calc(.25rem - 1px);border-top-left-radius:calc(.25rem - 1px)}.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}@media (min-width:576px){.card-deck{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-deck .card{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-flex:1;-webkit-flex:1 0 0%;-ms-flex:1 0 0%;flex:1 0 0%;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column}.card-deck .card:not(:first-child){margin-left:15px}.card-deck .card:not(:last-child){margin-right:15px}}@media (min-width:576px){.card-group{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group .card{-webkit-box-flex:1;-webkit-flex:1 0 0%;-ms-flex:1 0 0%;flex:1 0 0%}.card-group .card+.card{margin-left:0;border-left:0}.card-group .card:first-child{border-bottom-right-radius:0;border-top-right-radius:0}.card-group .card:first-child .card-img-top{border-top-right-radius:0}.card-group .card:first-child .card-img-bottom{border-bottom-right-radius:0}.card-group .card:last-child{border-bottom-left-radius:0;border-top-left-radius:0}.card-group .card:last-child .card-img-top{border-top-left-radius:0}.card-group .card:last-child .card-img-bottom{border-bottom-left-radius:0}.card-group .card:not(:first-child):not(:last-child){border-radius:0}.card-group .card:not(:first-child):not(:last-child) .card-img-bottom,.card-group .card:not(:first-child):not(:last-child) .card-img-top{border-radius:0}}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem}.card-columns .card{display:inline-block;width:100%;margin-bottom:.75rem}}.breadcrumb{padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#eceeef;border-radius:.25rem}.breadcrumb::after{display:block;content:"";clear:both}.breadcrumb-item{float:left}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;padding-left:.5rem;color:#636c72;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#636c72}.pagination{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-item:first-child .page-link{margin-left:0;border-bottom-left-radius:.25rem;border-top-left-radius:.25rem}.page-item:last-child .page-link{border-bottom-right-radius:.25rem;border-top-right-radius:.25rem}.page-item.active .page-link{z-index:2;color:#fff;background-color:#0275d8;border-color:#0275d8}.page-item.disabled .page-link{color:#636c72;pointer-events:none;cursor:not-allowed;background-color:#fff;border-color:#ddd}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#0275d8;background-color:#fff;border:1px solid #ddd}.page-link:focus,.page-link:hover{color:#014c8c;text-decoration:none;background-color:#eceeef;border-color:#ddd}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem}.pagination-lg .page-item:first-child .page-link{border-bottom-left-radius:.3rem;border-top-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-bottom-right-radius:.3rem;border-top-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem}.pagination-sm .page-item:first-child .page-link{border-bottom-left-radius:.2rem;border-top-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-bottom-right-radius:.2rem;border-top-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}a.badge:focus,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-default{background-color:#636c72}.badge-default[href]:focus,.badge-default[href]:hover{background-color:#4b5257}.badge-primary{background-color:#0275d8}.badge-primary[href]:focus,.badge-primary[href]:hover{background-color:#025aa5}.badge-success{background-color:#5cb85c}.badge-success[href]:focus,.badge-success[href]:hover{background-color:#449d44}.badge-info{background-color:#5bc0de}.badge-info[href]:focus,.badge-info[href]:hover{background-color:#31b0d5}.badge-warning{background-color:#f0ad4e}.badge-warning[href]:focus,.badge-warning[href]:hover{background-color:#ec971f}.badge-danger{background-color:#d9534f}.badge-danger[href]:focus,.badge-danger[href]:hover{background-color:#c9302c}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#eceeef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-hr{border-top-color:#d0d5d8}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible .close{position:relative;top:-.75rem;right:-1.25rem;padding:.75rem 1.25rem;color:inherit}.alert-success{background-color:#dff0d8;border-color:#d0e9c6;color:#3c763d}.alert-success hr{border-top-color:#c1e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{background-color:#d9edf7;border-color:#bcdff1;color:#31708f}.alert-info hr{border-top-color:#a6d5ec}.alert-info .alert-link{color:#245269}.alert-warning{background-color:#fcf8e3;border-color:#faf2cc;color:#8a6d3b}.alert-warning hr{border-top-color:#f7ecb5}.alert-warning .alert-link{color:#66512c}.alert-danger{background-color:#f2dede;border-color:#ebcccc;color:#a94442}.alert-danger hr{border-top-color:#e4b9b9}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;overflow:hidden;font-size:.75rem;line-height:1rem;text-align:center;background-color:#eceeef;border-radius:.25rem}.progress-bar{height:1rem;color:#fff;background-color:#0275d8}.progress-bar-striped{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);-webkit-background-size:1rem 1rem;background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;-o-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}.media{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:start;-webkit-align-items:flex-start;-ms-flex-align:start;align-items:flex-start}.media-body{-webkit-box-flex:1;-webkit-flex:1 1 0%;-ms-flex:1 1 0%;flex:1 1 0%}.list-group{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#464a4c;text-align:inherit}.list-group-item-action .list-group-item-heading{color:#292b2c}.list-group-item-action:focus,.list-group-item-action:hover{color:#464a4c;text-decoration:none;background-color:#f7f7f9}.list-group-item-action:active{color:#292b2c;background-color:#eceeef}.list-group-item{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;padding:.75rem 1.25rem;margin-bottom:-1px;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-right-radius:.25rem;border-top-left-radius:.25rem}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.list-group-item:focus,.list-group-item:hover{text-decoration:none}.list-group-item.disabled,.list-group-item:disabled{color:#636c72;cursor:not-allowed;background-color:#fff}.list-group-item.disabled .list-group-item-heading,.list-group-item:disabled .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item:disabled .list-group-item-text{color:#636c72}.list-group-item.active{z-index:2;color:#fff;background-color:#0275d8;border-color:#0275d8}.list-group-item.active .list-group-item-heading,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active .list-group-item-heading>small{color:inherit}.list-group-item.active .list-group-item-text{color:#daeeff}.list-group-flush .list-group-item{border-right:0;border-left:0;border-radius:0}.list-group-flush:first-child .list-group-item:first-child{border-top:0}.list-group-flush:last-child .list-group-item:last-child{border-bottom:0}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#3c763d;background-color:#d0e9c6}a.list-group-item-success.active,button.list-group-item-success.active{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,button.list-group-item-info.active{color:#fff;background-color:#31708f;border-color:#31708f}.list-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,button.list-group-item-warning.active{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,button.list-group-item-danger.active{color:#fff;background-color:#a94442;border-color:#a94442}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:focus,.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.75}button.close{padding:0;cursor:pointer;background:0 0;border:0;-webkit-appearance:none}.modal-open{overflow:hidden}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;outline:0}.modal.fade .modal-dialog{-webkit-transition:-webkit-transform .3s ease-out;transition:-webkit-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out,-o-transform .3s ease-out;-webkit-transform:translate(0,-25%);-o-transform:translate(0,-25%);transform:translate(0,-25%)}.modal.show .modal-dialog{-webkit-transform:translate(0,0);-o-transform:translate(0,0);transform:translate(0,0)}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;padding:15px;border-bottom:1px solid #eceeef}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;padding:15px}.modal-footer{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:end;-webkit-justify-content:flex-end;-ms-flex-pack:end;justify-content:flex-end;padding:15px;border-top:1px solid #eceeef}.modal-footer>:not(:first-child){margin-left:.25rem}.modal-footer>:not(:last-child){margin-right:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:30px auto}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg{max-width:800px}}.tooltip{position:absolute;z-index:1070;display:block;font-family:-apple-system,system-ui,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-style:normal;font-weight:400;letter-spacing:normal;line-break:auto;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;white-space:normal;word-break:normal;word-spacing:normal;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip.bs-tether-element-attached-bottom,.tooltip.tooltip-top{padding:5px 0;margin-top:-3px}.tooltip.bs-tether-element-attached-bottom .tooltip-inner::before,.tooltip.tooltip-top .tooltip-inner::before{bottom:0;left:50%;margin-left:-5px;content:"";border-width:5px 5px 0;border-top-color:#000}.tooltip.bs-tether-element-attached-left,.tooltip.tooltip-right{padding:0 5px;margin-left:3px}.tooltip.bs-tether-element-attached-left .tooltip-inner::before,.tooltip.tooltip-right .tooltip-inner::before{top:50%;left:0;margin-top:-5px;content:"";border-width:5px 5px 5px 0;border-right-color:#000}.tooltip.bs-tether-element-attached-top,.tooltip.tooltip-bottom{padding:5px 0;margin-top:3px}.tooltip.bs-tether-element-attached-top .tooltip-inner::before,.tooltip.tooltip-bottom .tooltip-inner::before{top:0;left:50%;margin-left:-5px;content:"";border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bs-tether-element-attached-right,.tooltip.tooltip-left{padding:0 5px;margin-left:-3px}.tooltip.bs-tether-element-attached-right .tooltip-inner::before,.tooltip.tooltip-left .tooltip-inner::before{top:50%;right:0;margin-top:-5px;content:"";border-width:5px 0 5px 5px;border-left-color:#000}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.tooltip-inner::before{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;padding:1px;font-family:-apple-system,system-ui,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-style:normal;font-weight:400;letter-spacing:normal;line-break:auto;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;white-space:normal;word-break:normal;word-spacing:normal;font-size:.875rem;word-wrap:break-word;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover.bs-tether-element-attached-bottom,.popover.popover-top{margin-top:-10px}.popover.bs-tether-element-attached-bottom::after,.popover.bs-tether-element-attached-bottom::before,.popover.popover-top::after,.popover.popover-top::before{left:50%;border-bottom-width:0}.popover.bs-tether-element-attached-bottom::before,.popover.popover-top::before{bottom:-11px;margin-left:-11px;border-top-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-bottom::after,.popover.popover-top::after{bottom:-10px;margin-left:-10px;border-top-color:#fff}.popover.bs-tether-element-attached-left,.popover.popover-right{margin-left:10px}.popover.bs-tether-element-attached-left::after,.popover.bs-tether-element-attached-left::before,.popover.popover-right::after,.popover.popover-right::before{top:50%;border-left-width:0}.popover.bs-tether-element-attached-left::before,.popover.popover-right::before{left:-11px;margin-top:-11px;border-right-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-left::after,.popover.popover-right::after{left:-10px;margin-top:-10px;border-right-color:#fff}.popover.bs-tether-element-attached-top,.popover.popover-bottom{margin-top:10px}.popover.bs-tether-element-attached-top::after,.popover.bs-tether-element-attached-top::before,.popover.popover-bottom::after,.popover.popover-bottom::before{left:50%;border-top-width:0}.popover.bs-tether-element-attached-top::before,.popover.popover-bottom::before{top:-11px;margin-left:-11px;border-bottom-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-top::after,.popover.popover-bottom::after{top:-10px;margin-left:-10px;border-bottom-color:#f7f7f7}.popover.bs-tether-element-attached-top .popover-title::before,.popover.popover-bottom .popover-title::before{position:absolute;top:0;left:50%;display:block;width:20px;margin-left:-10px;content:"";border-bottom:1px solid #f7f7f7}.popover.bs-tether-element-attached-right,.popover.popover-left{margin-left:-10px}.popover.bs-tether-element-attached-right::after,.popover.bs-tether-element-attached-right::before,.popover.popover-left::after,.popover.popover-left::before{top:50%;border-right-width:0}.popover.bs-tether-element-attached-right::before,.popover.popover-left::before{right:-11px;margin-top:-11px;border-left-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-right::after,.popover.popover-left::after{right:-10px;margin-top:-10px;border-left-color:#fff}.popover-title{padding:8px 14px;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-right-radius:calc(.3rem - 1px);border-top-left-radius:calc(.3rem - 1px)}.popover-title:empty{display:none}.popover-content{padding:9px 14px}.popover::after,.popover::before{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid}.popover::before{content:"";border-width:11px}.popover::after{content:"";border-width:10px}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-item{position:relative;display:none;width:100%}@media (-webkit-transform-3d){.carousel-item{-webkit-transition:-webkit-transform .6s ease-in-out;transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out,-o-transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}}@supports ((-webkit-transform:translate3d(0,0,0)) or (transform:translate3d(0,0,0))){.carousel-item{-webkit-transition:-webkit-transform .6s ease-in-out;transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out,-o-transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex}.carousel-item-next,.carousel-item-prev{position:absolute;top:0}@media (-webkit-transform-3d){.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.active.carousel-item-right,.carousel-item-next{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.active.carousel-item-left,.carousel-item-prev{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}}@supports ((-webkit-transform:translate3d(0,0,0)) or (transform:translate3d(0,0,0))){.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.active.carousel-item-right,.carousel-item-next{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.active.carousel-item-left,.carousel-item-prev{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:transparent no-repeat center center;-webkit-background-size:100% 100%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M4 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3E%3C/svg%3E")}.carousel-control-next-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M1.5 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3E%3C/svg%3E")}.carousel-indicators{position:absolute;right:0;bottom:10px;left:0;z-index:15;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{position:relative;-webkit-box-flex:1;-webkit-flex:1 0 auto;-ms-flex:1 0 auto;flex:1 0 auto;max-width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:rgba(255,255,255,.5)}.carousel-indicators li::before{position:absolute;top:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators li::after{position:absolute;bottom:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators .active{background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-faded{background-color:#f7f7f7}.bg-primary{background-color:#0275d8!important}a.bg-primary:focus,a.bg-primary:hover{background-color:#025aa5!important}.bg-success{background-color:#5cb85c!important}a.bg-success:focus,a.bg-success:hover{background-color:#449d44!important}.bg-info{background-color:#5bc0de!important}a.bg-info:focus,a.bg-info:hover{background-color:#31b0d5!important}.bg-warning{background-color:#f0ad4e!important}a.bg-warning:focus,a.bg-warning:hover{background-color:#ec971f!important}.bg-danger{background-color:#d9534f!important}a.bg-danger:focus,a.bg-danger:hover{background-color:#c9302c!important}.bg-inverse{background-color:#292b2c!important}a.bg-inverse:focus,a.bg-inverse:hover{background-color:#101112!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.rounded{border-radius:.25rem}.rounded-top{border-top-right-radius:.25rem;border-top-left-radius:.25rem}.rounded-right{border-bottom-right-radius:.25rem;border-top-right-radius:.25rem}.rounded-bottom{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.rounded-left{border-bottom-left-radius:.25rem;border-top-left-radius:.25rem}.rounded-circle{border-radius:50%}.rounded-0{border-radius:0}.clearfix::after{display:block;content:"";clear:both}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-cell{display:table-cell!important}.d-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}.flex-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-sm-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-sm-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-sm-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-sm-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-sm-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-sm-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-md-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-md-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-md-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-md-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-md-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-md-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-lg-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-lg-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-lg-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-lg-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-lg-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-lg-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-xl-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-xl-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-xl-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-xl-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-xl-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-xl-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1030}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.m-0{margin:0 0!important}.mt-0{margin-top:0!important}.mr-0{margin-right:0!important}.mb-0{margin-bottom:0!important}.ml-0{margin-left:0!important}.mx-0{margin-right:0!important;margin-left:0!important}.my-0{margin-top:0!important;margin-bottom:0!important}.m-1{margin:.25rem .25rem!important}.mt-1{margin-top:.25rem!important}.mr-1{margin-right:.25rem!important}.mb-1{margin-bottom:.25rem!important}.ml-1{margin-left:.25rem!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-2{margin:.5rem .5rem!important}.mt-2{margin-top:.5rem!important}.mr-2{margin-right:.5rem!important}.mb-2{margin-bottom:.5rem!important}.ml-2{margin-left:.5rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-3{margin:1rem 1rem!important}.mt-3{margin-top:1rem!important}.mr-3{margin-right:1rem!important}.mb-3{margin-bottom:1rem!important}.ml-3{margin-left:1rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-4{margin:1.5rem 1.5rem!important}.mt-4{margin-top:1.5rem!important}.mr-4{margin-right:1.5rem!important}.mb-4{margin-bottom:1.5rem!important}.ml-4{margin-left:1.5rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-5{margin:3rem 3rem!important}.mt-5{margin-top:3rem!important}.mr-5{margin-right:3rem!important}.mb-5{margin-bottom:3rem!important}.ml-5{margin-left:3rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-0{padding:0 0!important}.pt-0{padding-top:0!important}.pr-0{padding-right:0!important}.pb-0{padding-bottom:0!important}.pl-0{padding-left:0!important}.px-0{padding-right:0!important;padding-left:0!important}.py-0{padding-top:0!important;padding-bottom:0!important}.p-1{padding:.25rem .25rem!important}.pt-1{padding-top:.25rem!important}.pr-1{padding-right:.25rem!important}.pb-1{padding-bottom:.25rem!important}.pl-1{padding-left:.25rem!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-2{padding:.5rem .5rem!important}.pt-2{padding-top:.5rem!important}.pr-2{padding-right:.5rem!important}.pb-2{padding-bottom:.5rem!important}.pl-2{padding-left:.5rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-3{padding:1rem 1rem!important}.pt-3{padding-top:1rem!important}.pr-3{padding-right:1rem!important}.pb-3{padding-bottom:1rem!important}.pl-3{padding-left:1rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-4{padding:1.5rem 1.5rem!important}.pt-4{padding-top:1.5rem!important}.pr-4{padding-right:1.5rem!important}.pb-4{padding-bottom:1.5rem!important}.pl-4{padding-left:1.5rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-5{padding:3rem 3rem!important}.pt-5{padding-top:3rem!important}.pr-5{padding-right:3rem!important}.pb-5{padding-bottom:3rem!important}.pl-5{padding-left:3rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-auto{margin:auto!important}.mt-auto{margin-top:auto!important}.mr-auto{margin-right:auto!important}.mb-auto{margin-bottom:auto!important}.ml-auto{margin-left:auto!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}@media (min-width:576px){.m-sm-0{margin:0 0!important}.mt-sm-0{margin-top:0!important}.mr-sm-0{margin-right:0!important}.mb-sm-0{margin-bottom:0!important}.ml-sm-0{margin-left:0!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.m-sm-1{margin:.25rem .25rem!important}.mt-sm-1{margin-top:.25rem!important}.mr-sm-1{margin-right:.25rem!important}.mb-sm-1{margin-bottom:.25rem!important}.ml-sm-1{margin-left:.25rem!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-sm-2{margin:.5rem .5rem!important}.mt-sm-2{margin-top:.5rem!important}.mr-sm-2{margin-right:.5rem!important}.mb-sm-2{margin-bottom:.5rem!important}.ml-sm-2{margin-left:.5rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-sm-3{margin:1rem 1rem!important}.mt-sm-3{margin-top:1rem!important}.mr-sm-3{margin-right:1rem!important}.mb-sm-3{margin-bottom:1rem!important}.ml-sm-3{margin-left:1rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-sm-4{margin:1.5rem 1.5rem!important}.mt-sm-4{margin-top:1.5rem!important}.mr-sm-4{margin-right:1.5rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.ml-sm-4{margin-left:1.5rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-sm-5{margin:3rem 3rem!important}.mt-sm-5{margin-top:3rem!important}.mr-sm-5{margin-right:3rem!important}.mb-sm-5{margin-bottom:3rem!important}.ml-sm-5{margin-left:3rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-sm-0{padding:0 0!important}.pt-sm-0{padding-top:0!important}.pr-sm-0{padding-right:0!important}.pb-sm-0{padding-bottom:0!important}.pl-sm-0{padding-left:0!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.p-sm-1{padding:.25rem .25rem!important}.pt-sm-1{padding-top:.25rem!important}.pr-sm-1{padding-right:.25rem!important}.pb-sm-1{padding-bottom:.25rem!important}.pl-sm-1{padding-left:.25rem!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-sm-2{padding:.5rem .5rem!important}.pt-sm-2{padding-top:.5rem!important}.pr-sm-2{padding-right:.5rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pl-sm-2{padding-left:.5rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-sm-3{padding:1rem 1rem!important}.pt-sm-3{padding-top:1rem!important}.pr-sm-3{padding-right:1rem!important}.pb-sm-3{padding-bottom:1rem!important}.pl-sm-3{padding-left:1rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-sm-4{padding:1.5rem 1.5rem!important}.pt-sm-4{padding-top:1.5rem!important}.pr-sm-4{padding-right:1.5rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pl-sm-4{padding-left:1.5rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-sm-5{padding:3rem 3rem!important}.pt-sm-5{padding-top:3rem!important}.pr-sm-5{padding-right:3rem!important}.pb-sm-5{padding-bottom:3rem!important}.pl-sm-5{padding-left:3rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto{margin-top:auto!important}.mr-sm-auto{margin-right:auto!important}.mb-sm-auto{margin-bottom:auto!important}.ml-sm-auto{margin-left:auto!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:768px){.m-md-0{margin:0 0!important}.mt-md-0{margin-top:0!important}.mr-md-0{margin-right:0!important}.mb-md-0{margin-bottom:0!important}.ml-md-0{margin-left:0!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.m-md-1{margin:.25rem .25rem!important}.mt-md-1{margin-top:.25rem!important}.mr-md-1{margin-right:.25rem!important}.mb-md-1{margin-bottom:.25rem!important}.ml-md-1{margin-left:.25rem!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-md-2{margin:.5rem .5rem!important}.mt-md-2{margin-top:.5rem!important}.mr-md-2{margin-right:.5rem!important}.mb-md-2{margin-bottom:.5rem!important}.ml-md-2{margin-left:.5rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-md-3{margin:1rem 1rem!important}.mt-md-3{margin-top:1rem!important}.mr-md-3{margin-right:1rem!important}.mb-md-3{margin-bottom:1rem!important}.ml-md-3{margin-left:1rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-md-4{margin:1.5rem 1.5rem!important}.mt-md-4{margin-top:1.5rem!important}.mr-md-4{margin-right:1.5rem!important}.mb-md-4{margin-bottom:1.5rem!important}.ml-md-4{margin-left:1.5rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-md-5{margin:3rem 3rem!important}.mt-md-5{margin-top:3rem!important}.mr-md-5{margin-right:3rem!important}.mb-md-5{margin-bottom:3rem!important}.ml-md-5{margin-left:3rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-md-0{padding:0 0!important}.pt-md-0{padding-top:0!important}.pr-md-0{padding-right:0!important}.pb-md-0{padding-bottom:0!important}.pl-md-0{padding-left:0!important}.px-md-0{padding-right:0!important;padding-left:0!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.p-md-1{padding:.25rem .25rem!important}.pt-md-1{padding-top:.25rem!important}.pr-md-1{padding-right:.25rem!important}.pb-md-1{padding-bottom:.25rem!important}.pl-md-1{padding-left:.25rem!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-md-2{padding:.5rem .5rem!important}.pt-md-2{padding-top:.5rem!important}.pr-md-2{padding-right:.5rem!important}.pb-md-2{padding-bottom:.5rem!important}.pl-md-2{padding-left:.5rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-md-3{padding:1rem 1rem!important}.pt-md-3{padding-top:1rem!important}.pr-md-3{padding-right:1rem!important}.pb-md-3{padding-bottom:1rem!important}.pl-md-3{padding-left:1rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-md-4{padding:1.5rem 1.5rem!important}.pt-md-4{padding-top:1.5rem!important}.pr-md-4{padding-right:1.5rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pl-md-4{padding-left:1.5rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-md-5{padding:3rem 3rem!important}.pt-md-5{padding-top:3rem!important}.pr-md-5{padding-right:3rem!important}.pb-md-5{padding-bottom:3rem!important}.pl-md-5{padding-left:3rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto{margin-top:auto!important}.mr-md-auto{margin-right:auto!important}.mb-md-auto{margin-bottom:auto!important}.ml-md-auto{margin-left:auto!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:992px){.m-lg-0{margin:0 0!important}.mt-lg-0{margin-top:0!important}.mr-lg-0{margin-right:0!important}.mb-lg-0{margin-bottom:0!important}.ml-lg-0{margin-left:0!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.m-lg-1{margin:.25rem .25rem!important}.mt-lg-1{margin-top:.25rem!important}.mr-lg-1{margin-right:.25rem!important}.mb-lg-1{margin-bottom:.25rem!important}.ml-lg-1{margin-left:.25rem!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-lg-2{margin:.5rem .5rem!important}.mt-lg-2{margin-top:.5rem!important}.mr-lg-2{margin-right:.5rem!important}.mb-lg-2{margin-bottom:.5rem!important}.ml-lg-2{margin-left:.5rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-lg-3{margin:1rem 1rem!important}.mt-lg-3{margin-top:1rem!important}.mr-lg-3{margin-right:1rem!important}.mb-lg-3{margin-bottom:1rem!important}.ml-lg-3{margin-left:1rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-lg-4{margin:1.5rem 1.5rem!important}.mt-lg-4{margin-top:1.5rem!important}.mr-lg-4{margin-right:1.5rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.ml-lg-4{margin-left:1.5rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-lg-5{margin:3rem 3rem!important}.mt-lg-5{margin-top:3rem!important}.mr-lg-5{margin-right:3rem!important}.mb-lg-5{margin-bottom:3rem!important}.ml-lg-5{margin-left:3rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-lg-0{padding:0 0!important}.pt-lg-0{padding-top:0!important}.pr-lg-0{padding-right:0!important}.pb-lg-0{padding-bottom:0!important}.pl-lg-0{padding-left:0!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.p-lg-1{padding:.25rem .25rem!important}.pt-lg-1{padding-top:.25rem!important}.pr-lg-1{padding-right:.25rem!important}.pb-lg-1{padding-bottom:.25rem!important}.pl-lg-1{padding-left:.25rem!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-lg-2{padding:.5rem .5rem!important}.pt-lg-2{padding-top:.5rem!important}.pr-lg-2{padding-right:.5rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pl-lg-2{padding-left:.5rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-lg-3{padding:1rem 1rem!important}.pt-lg-3{padding-top:1rem!important}.pr-lg-3{padding-right:1rem!important}.pb-lg-3{padding-bottom:1rem!important}.pl-lg-3{padding-left:1rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-lg-4{padding:1.5rem 1.5rem!important}.pt-lg-4{padding-top:1.5rem!important}.pr-lg-4{padding-right:1.5rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pl-lg-4{padding-left:1.5rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-lg-5{padding:3rem 3rem!important}.pt-lg-5{padding-top:3rem!important}.pr-lg-5{padding-right:3rem!important}.pb-lg-5{padding-bottom:3rem!important}.pl-lg-5{padding-left:3rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto{margin-top:auto!important}.mr-lg-auto{margin-right:auto!important}.mb-lg-auto{margin-bottom:auto!important}.ml-lg-auto{margin-left:auto!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0 0!important}.mt-xl-0{margin-top:0!important}.mr-xl-0{margin-right:0!important}.mb-xl-0{margin-bottom:0!important}.ml-xl-0{margin-left:0!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.m-xl-1{margin:.25rem .25rem!important}.mt-xl-1{margin-top:.25rem!important}.mr-xl-1{margin-right:.25rem!important}.mb-xl-1{margin-bottom:.25rem!important}.ml-xl-1{margin-left:.25rem!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-xl-2{margin:.5rem .5rem!important}.mt-xl-2{margin-top:.5rem!important}.mr-xl-2{margin-right:.5rem!important}.mb-xl-2{margin-bottom:.5rem!important}.ml-xl-2{margin-left:.5rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-xl-3{margin:1rem 1rem!important}.mt-xl-3{margin-top:1rem!important}.mr-xl-3{margin-right:1rem!important}.mb-xl-3{margin-bottom:1rem!important}.ml-xl-3{margin-left:1rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-xl-4{margin:1.5rem 1.5rem!important}.mt-xl-4{margin-top:1.5rem!important}.mr-xl-4{margin-right:1.5rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.ml-xl-4{margin-left:1.5rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-xl-5{margin:3rem 3rem!important}.mt-xl-5{margin-top:3rem!important}.mr-xl-5{margin-right:3rem!important}.mb-xl-5{margin-bottom:3rem!important}.ml-xl-5{margin-left:3rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-xl-0{padding:0 0!important}.pt-xl-0{padding-top:0!important}.pr-xl-0{padding-right:0!important}.pb-xl-0{padding-bottom:0!important}.pl-xl-0{padding-left:0!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.p-xl-1{padding:.25rem .25rem!important}.pt-xl-1{padding-top:.25rem!important}.pr-xl-1{padding-right:.25rem!important}.pb-xl-1{padding-bottom:.25rem!important}.pl-xl-1{padding-left:.25rem!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-xl-2{padding:.5rem .5rem!important}.pt-xl-2{padding-top:.5rem!important}.pr-xl-2{padding-right:.5rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pl-xl-2{padding-left:.5rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-xl-3{padding:1rem 1rem!important}.pt-xl-3{padding-top:1rem!important}.pr-xl-3{padding-right:1rem!important}.pb-xl-3{padding-bottom:1rem!important}.pl-xl-3{padding-left:1rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-xl-4{padding:1.5rem 1.5rem!important}.pt-xl-4{padding-top:1.5rem!important}.pr-xl-4{padding-right:1.5rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pl-xl-4{padding-left:1.5rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-xl-5{padding:3rem 3rem!important}.pt-xl-5{padding-top:3rem!important}.pr-xl-5{padding-right:3rem!important}.pb-xl-5{padding-bottom:3rem!important}.pl-xl-5{padding-left:3rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto{margin-top:auto!important}.mr-xl-auto{margin-right:auto!important}.mb-xl-auto{margin-bottom:auto!important}.ml-xl-auto{margin-left:auto!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}}.text-justify{text-align:justify!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-normal{font-weight:400}.font-weight-bold{font-weight:700}.font-italic{font-style:italic}.text-white{color:#fff!important}.text-muted{color:#636c72!important}a.text-muted:focus,a.text-muted:hover{color:#4b5257!important}.text-primary{color:#0275d8!important}a.text-primary:focus,a.text-primary:hover{color:#025aa5!important}.text-success{color:#5cb85c!important}a.text-success:focus,a.text-success:hover{color:#449d44!important}.text-info{color:#5bc0de!important}a.text-info:focus,a.text-info:hover{color:#31b0d5!important}.text-warning{color:#f0ad4e!important}a.text-warning:focus,a.text-warning:hover{color:#ec971f!important}.text-danger{color:#d9534f!important}a.text-danger:focus,a.text-danger:hover{color:#c9302c!important}.text-gray-dark{color:#292b2c!important}a.text-gray-dark:focus,a.text-gray-dark:hover{color:#101112!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.invisible{visibility:hidden!important}.hidden-xs-up{display:none!important}@media (max-width:575px){.hidden-xs-down{display:none!important}}@media (min-width:576px){.hidden-sm-up{display:none!important}}@media (max-width:767px){.hidden-sm-down{display:none!important}}@media (min-width:768px){.hidden-md-up{display:none!important}}@media (max-width:991px){.hidden-md-down{display:none!important}}@media (min-width:992px){.hidden-lg-up{display:none!important}}@media (max-width:1199px){.hidden-lg-down{display:none!important}}@media (min-width:1200px){.hidden-xl-up{display:none!important}}.hidden-xl-down{display:none!important}.visible-print-block{display:none!important}@media print{.visible-print-block{display:block!important}}.visible-print-inline{display:none!important}@media print{.visible-print-inline{display:inline!important}}.visible-print-inline-block{display:none!important}@media print{.visible-print-inline-block{display:inline-block!important}}@media print{.hidden-print{display:none!important}}/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/archivebox/util.py b/archivebox/util.py index dc5590c5..970085ea 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -528,10 +528,10 @@ class TimedProgress: # return if self.p is not None: self.p.terminate() + self.p = None sys.stdout.write('\r{}{}\r'.format((' ' * TERM_WIDTH()), ANSI['reset'])) # clear whole terminal line - sys.stdout.flush() @enforce_types From d2cf260e7c2fe46cb2c7e63711d78bb24669b605 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 03:52:38 -0400 Subject: [PATCH 012/365] css tweaks --- archivebox/templates/link_index.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archivebox/templates/link_index.html b/archivebox/templates/link_index.html index 6309fb14..ea5d3e71 100644 --- a/archivebox/templates/link_index.html +++ b/archivebox/templates/link_index.html @@ -235,11 +235,13 @@ <div class="header-top container-fluid"> <div class="row nav"> <div class="col-lg-6"> + <a href="../../index.html" class="header-archivebox" title="Go to Main Index..."> <img src="../../static/archive.png" alt="Archive Icon"> ArchiveBox: </a> - <a href="#">Page Details</a><br/> + <a href="#">Page Details</a>   + <br/> <small style="margin-top: 5px; display: block"> <a href="../../index.html">Home</a>   |   <a href="https://github.com/pirate/ArchiveBox">Github</a>   |   @@ -249,8 +251,7 @@ <div class="col-lg-6"> <img src="$link_dir/$favicon_url" alt="Favicon">     - $title -     + $title<br/> <small> <a href="$url" title="Toggle info panel..." class="header-url" title="$url"> $base_url From f7aa082dbfdbb3f74c7bf6f0ab103ec1a8727c3c Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 03:52:50 -0400 Subject: [PATCH 013/365] css tweaks From 98870ba428ce2bcde67294140b8e50eb5a8c91b1 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 03:56:38 -0400 Subject: [PATCH 014/365] fit bigger titles with tighter spacing --- archivebox/templates/index.html | 2 +- archivebox/templates/link_index.html | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/archivebox/templates/index.html b/archivebox/templates/index.html index f5cf2785..f78e89a6 100644 --- a/archivebox/templates/index.html +++ b/archivebox/templates/index.html @@ -25,7 +25,7 @@ margin: 0px; text-align: center; color: white; - font-size: calc(11px + 0.86vw); + font-size: calc(11px + 0.84vw); font-weight: 200; padding: 4px 4px; border-bottom: 3px solid #aa1e55; diff --git a/archivebox/templates/link_index.html b/archivebox/templates/link_index.html index ea5d3e71..a450a525 100644 --- a/archivebox/templates/link_index.html +++ b/archivebox/templates/link_index.html @@ -15,6 +15,9 @@ small { font-weight: 200; } + header a:hover { + text-decoration: none; + } .header-top { width: 100%; height: auto; @@ -22,7 +25,7 @@ margin: 0px; text-align: center; color: white; - font-size: calc(11px + 0.86vw); + font-size: calc(11px + 0.84vw); font-weight: 200; padding: 4px 4px; background-color: #aa1e55; @@ -242,7 +245,7 @@ </a> <a href="#">Page Details</a>   <br/> - <small style="margin-top: 5px; display: block"> + <small style="margin-top: 5px; display: block; opacity: 0.7"> <a href="../../index.html">Home</a>   |   <a href="https://github.com/pirate/ArchiveBox">Github</a>   |   <a href="https://github.com/pirate/ArchiveBox/wiki">Documentation</a> @@ -250,15 +253,16 @@ </div> <div class="col-lg-6"> <img src="$link_dir/$favicon_url" alt="Favicon"> -     - $title<br/> +    + $title +    + <a href="#" class="header-toggle">▾</a> + <br/> <small> <a href="$url" title="Toggle info panel..." class="header-url" title="$url"> $base_url </a> </small> -     - <a href="#" class="header-toggle">▾</a> </div> </div> </div> From 19e1c35f1a771f9bbca93dc0e7f0420ab27a0968 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 10:31:07 -0400 Subject: [PATCH 015/365] support loading link indexes with extra keys --- archivebox/index.py | 10 ++++++++-- archivebox/schema.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index 74e7dd42..eab2197b 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -120,12 +120,18 @@ def write_json_links_index(out_dir: str, links: List[Link]) -> None: def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: """parse a archive index json file and return the list of links""" + allowed_fields = {f.name for f in fields(Link)} + index_path = os.path.join(out_dir, 'index.json') if os.path.exists(index_path): with open(index_path, 'r', encoding='utf-8') as f: links = json.load(f)['links'] - for link in links: - yield Link(**link) + for link_json in links: + yield Link(**{ + key: val + for key, val in link_json.items() + if key in allowed_fields + }) return () diff --git a/archivebox/schema.py b/archivebox/schema.py index 434f9dc5..5aa629d7 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -164,7 +164,7 @@ class Link: from util import ts_to_date most_recent = min( - (result.start_ts + (ts_to_date(result.start_ts) for method in self.history.keys() for result in self.history[method]), default=None, @@ -176,7 +176,7 @@ class Link: from util import ts_to_date most_recent = max( - (result.start_ts + (ts_to_date(result.start_ts) for method in self.history.keys() for result in self.history[method]), default=None, From 407484b91e63d535431dc437a82297aa56767e79 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 10:31:22 -0400 Subject: [PATCH 016/365] show progress during main index writing --- archivebox/index.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/archivebox/index.py b/archivebox/index.py index eab2197b..116de5a7 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -4,6 +4,7 @@ import json from datetime import datetime from string import Template from typing import List, Tuple, Iterator, Optional +from dataclasses import fields try: from distutils.dir_util import copy_tree @@ -17,6 +18,7 @@ from config import ( TEMPLATES_DIR, GIT_SHA, FOOTER_INFO, + TIMEOUT, ) from util import ( merge_links, @@ -26,6 +28,7 @@ from util import ( wget_output_path, ExtendedEncoder, enforce_types, + TimedProgress, ) from parse import parse_links from links import validate_links @@ -49,11 +52,15 @@ def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> log_indexing_process_started() log_indexing_started(out_dir, 'index.json') + timer = TimedProgress(TIMEOUT * 2, prefix=' ') write_json_links_index(out_dir, links) + timer.end() log_indexing_finished(out_dir, 'index.json') log_indexing_started(out_dir, 'index.html') + timer = TimedProgress(TIMEOUT * 2, prefix=' ') write_html_links_index(out_dir, links, finished=finished) + timer.end() log_indexing_finished(out_dir, 'index.html') From 0835baf4359e759eafbf7348d3499979ef996c31 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 10:31:44 -0400 Subject: [PATCH 017/365] show link titles in black --- archivebox/templates/index.html | 8 +++++++- archivebox/templates/index_row.html | 2 +- archivebox/templates/link_index.html | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/archivebox/templates/index.html b/archivebox/templates/index.html index f78e89a6..8436a412 100644 --- a/archivebox/templates/index.html +++ b/archivebox/templates/index.html @@ -150,6 +150,12 @@ input[type=search]::-webkit-search-cancel-button { -webkit-appearance: searchfield-cancel-button; } + .title-col { + text-align: left; + } + .title-col a { + color: black; + } </style> <link rel="stylesheet" href="static/bootstrap.min.css"> <link rel="stylesheet" href="static/jquery.dataTables.min.css"/> @@ -190,7 +196,7 @@ <table id="table-bookmarks"> <thead> <tr> - <th style="width: 80px;">Bookmarked</th> + <th style="width: 100px;">Bookmarked</th> <th style="width: 26vw;">Saved Link ($num_links)</th> <th style="width: 50px">Files</th> <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th> diff --git a/archivebox/templates/index_row.html b/archivebox/templates/index_row.html index 41c6e1ea..ffda7a19 100644 --- a/archivebox/templates/index_row.html +++ b/archivebox/templates/index_row.html @@ -1,6 +1,6 @@ <tr> <td title="$timestamp">$bookmarked_date</td> - <td style="text-align:left"> + <td class="title-col"> <a href="$archive_path/$index_url"><img src="$favicon_url" class="link-favicon" decoding="async"></a> <a href="$archive_path/$archive_url" title="$title"> <span data-title-for="$url" data-archived="$is_archived">$title</span> diff --git a/archivebox/templates/link_index.html b/archivebox/templates/link_index.html index a450a525..807a4fdc 100644 --- a/archivebox/templates/link_index.html +++ b/archivebox/templates/link_index.html @@ -246,7 +246,7 @@ <a href="#">Page Details</a>   <br/> <small style="margin-top: 5px; display: block; opacity: 0.7"> - <a href="../../index.html">Home</a>   |   + <a href="../../index.html">Index</a>   |   <a href="https://github.com/pirate/ArchiveBox">Github</a>   |   <a href="https://github.com/pirate/ArchiveBox/wiki">Documentation</a> </small> From e1a1ea25dd6dfaea950b49a05b204d21c6b58560 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 11:39:28 -0400 Subject: [PATCH 018/365] add readability script --- archivebox/scripts/readability.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 archivebox/scripts/readability.js diff --git a/archivebox/scripts/readability.js b/archivebox/scripts/readability.js new file mode 100644 index 00000000..ce937f54 --- /dev/null +++ b/archivebox/scripts/readability.js @@ -0,0 +1,27 @@ +() => { + function Readability(e,t){if(t&&t.documentElement)e=t,t=arguments[2];else if(!e||!e.documentElement)throw new Error("First argument to Readability constructor should be a document object.");var i;t=t||{},this._doc=e,this._articleTitle=null,this._articleByline=null,this._articleDir=null,this._articleSiteName=null,this._attempts=[],this._debug=!!t.debug,this._maxElemsToParse=t.maxElemsToParse||this.DEFAULT_MAX_ELEMS_TO_PARSE,this._nbTopCandidates=t.nbTopCandidates||this.DEFAULT_N_TOP_CANDIDATES,this._charThreshold=t.charThreshold||this.DEFAULT_CHAR_THRESHOLD,this._classesToPreserve=this.CLASSES_TO_PRESERVE.concat(t.classesToPreserve||[]),this._flags=this.FLAG_STRIP_UNLIKELYS|this.FLAG_WEIGHT_CLASSES|this.FLAG_CLEAN_CONDITIONALLY,this._debug?(i=function(e){var t=e.nodeName+" ";if(e.nodeType==e.TEXT_NODE)return t+'("'+e.textContent+'")';var i=e.className&&"."+e.className.replace(/ /g,"."),a="";return e.id?a="(#"+e.id+i+")":i&&(a="("+i+")"),t+a},this.log=function(){if("undefined"!=typeof dump){var e=Array.prototype.map.call(arguments,function(e){return e&&e.nodeName?i(e):e}).join(" ");dump("Reader: (Readability) "+e+"\n")}else if("undefined"!=typeof console){var t=["Reader: (Readability) "].concat(arguments);console.log.apply(console,t)}}):this.log=function(){}}Readability.prototype={FLAG_STRIP_UNLIKELYS:1,FLAG_WEIGHT_CLASSES:2,FLAG_CLEAN_CONDITIONALLY:4,ELEMENT_NODE:1,TEXT_NODE:3,DEFAULT_MAX_ELEMS_TO_PARSE:0,DEFAULT_N_TOP_CANDIDATES:5,DEFAULT_TAGS_TO_SCORE:"section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),DEFAULT_CHAR_THRESHOLD:500,REGEXPS:{unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|main|shadow/i,positive:/article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,negative:/hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i,extraneous:/print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,byline:/byline|author|dateline|writtenby|p-author/i,replaceFonts:/<(\/?)font[^>]*>/gi,normalize:/\s{2,}/g,videos:/\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,nextLink:/(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,prevLink:/(prev|earl|old|new|<|«)/i,whitespace:/^\s*$/,hasContent:/\S$/},DIV_TO_P_ELEMS:["A","BLOCKQUOTE","DL","DIV","IMG","OL","P","PRE","TABLE","UL","SELECT"],ALTER_TO_DIV_EXCEPTIONS:["DIV","ARTICLE","SECTION","P"],PRESENTATIONAL_ATTRIBUTES:["align","background","bgcolor","border","cellpadding","cellspacing","frame","hspace","rules","style","valign","vspace"],DEPRECATED_SIZE_ATTRIBUTE_ELEMS:["TABLE","TH","TD","HR","PRE"],PHRASING_ELEMS:["ABBR","AUDIO","B","BDO","BR","BUTTON","CITE","CODE","DATA","DATALIST","DFN","EM","EMBED","I","IMG","INPUT","KBD","LABEL","MARK","MATH","METER","NOSCRIPT","OBJECT","OUTPUT","PROGRESS","Q","RUBY","SAMP","SCRIPT","SELECT","SMALL","SPAN","STRONG","SUB","SUP","TEXTAREA","TIME","VAR","WBR"],CLASSES_TO_PRESERVE:["page"],_postProcessContent:function(e){this._fixRelativeUris(e),this._cleanClasses(e)},_removeNodes:function(e,t){for(var i=e.length-1;0<=i;i--){var a=e[i],n=a.parentNode;n&&(t&&!t.call(this,a,i,e)||n.removeChild(a))}},_replaceNodeTags:function(e,t){for(var i=e.length-1;0<=i;i--){var a=e[i];this._setNodeTag(a,t)}},_forEachNode:function(e,t){Array.prototype.forEach.call(e,t,this)},_someNode:function(e,t){return Array.prototype.some.call(e,t,this)},_everyNode:function(e,t){return Array.prototype.every.call(e,t,this)},_concatNodeLists:function(){var t=Array.prototype.slice,e=t.call(arguments).map(function(e){return t.call(e)});return Array.prototype.concat.apply([],e)},_getAllNodesWithTag:function(i,e){return i.querySelectorAll?i.querySelectorAll(e.join(",")):[].concat.apply([],e.map(function(e){var t=i.getElementsByTagName(e);return Array.isArray(t)?t:Array.from(t)}))},_cleanClasses:function(e){var t=this._classesToPreserve,i=(e.getAttribute("class")||"").split(/\s+/).filter(function(e){return-1!=t.indexOf(e)}).join(" ");for(i?e.setAttribute("class",i):e.removeAttribute("class"),e=e.firstElementChild;e;e=e.nextElementSibling)this._cleanClasses(e)},_fixRelativeUris:function(e){var t=this._doc.baseURI,i=this._doc.documentURI;function a(e){if(t==i&&"#"==e.charAt(0))return e;try{return new URL(e,t).href}catch(e){}return e}var n=this._getAllNodesWithTag(e,["a"]);this._forEachNode(n,function(e){var t=e.getAttribute("href");if(t)if(0===t.indexOf("javascript:")){var i=this._doc.createTextNode(e.textContent);e.parentNode.replaceChild(i,e)}else e.setAttribute("href",a(t))});var r=this._getAllNodesWithTag(e,["img"]);this._forEachNode(r,function(e){var t=e.getAttribute("src");t&&e.setAttribute("src",a(t))})},_getArticleTitle:function(){var e=this._doc,t="",i="";try{"string"!=typeof(t=i=e.title.trim())&&(t=i=this._getInnerText(e.getElementsByTagName("title")[0]))}catch(e){}var a=!1;function n(e){return e.split(/\s+/).length}if(/ [\|\-\\\/>»] /.test(t))a=/ [\\\/>»] /.test(t),n(t=i.replace(/(.*)[\|\-\\\/>»] .*/gi,"$1"))<3&&(t=i.replace(/[^\|\-\\\/>»]*[\|\-\\\/>»](.*)/gi,"$1"));else if(-1!==t.indexOf(": ")){var r=this._concatNodeLists(e.getElementsByTagName("h1"),e.getElementsByTagName("h2")),s=t.trim();this._someNode(r,function(e){return e.textContent.trim()===s})||(n(t=i.substring(i.lastIndexOf(":")+1))<3?t=i.substring(i.indexOf(":")+1):5<n(i.substr(0,i.indexOf(":")))&&(t=i))}else if(150<t.length||t.length<15){var o=e.getElementsByTagName("h1");1===o.length&&(t=this._getInnerText(o[0]))}var l=n(t=t.trim().replace(this.REGEXPS.normalize," "));return l<=4&&(!a||l!=n(i.replace(/[\|\-\\\/>»]+/g,""))-1)&&(t=i),t},_prepDocument:function(){var e=this._doc;this._removeNodes(e.getElementsByTagName("style")),e.body&&this._replaceBrs(e.body),this._replaceNodeTags(e.getElementsByTagName("font"),"SPAN")},_nextElement:function(e){for(var t=e;t&&t.nodeType!=this.ELEMENT_NODE&&this.REGEXPS.whitespace.test(t.textContent);)t=t.nextSibling;return t},_replaceBrs:function(e){this._forEachNode(this._getAllNodesWithTag(e,["br"]),function(e){for(var t=e.nextSibling,i=!1;(t=this._nextElement(t))&&"BR"==t.tagName;){i=!0;var a=t.nextSibling;t.parentNode.removeChild(t),t=a}if(i){var n=this._doc.createElement("p");for(e.parentNode.replaceChild(n,e),t=n.nextSibling;t;){if("BR"==t.tagName){var r=this._nextElement(t.nextSibling);if(r&&"BR"==r.tagName)break}if(!this._isPhrasingContent(t))break;var s=t.nextSibling;n.appendChild(t),t=s}for(;n.lastChild&&this._isWhitespace(n.lastChild);)n.removeChild(n.lastChild);"P"===n.parentNode.tagName&&this._setNodeTag(n.parentNode,"DIV")}})},_setNodeTag:function(e,t){if(this.log("_setNodeTag",e,t),e.__JSDOMParser__)return e.localName=t.toLowerCase(),e.tagName=t.toUpperCase(),e;for(var i=e.ownerDocument.createElement(t);e.firstChild;)i.appendChild(e.firstChild);e.parentNode.replaceChild(i,e),e.readability&&(i.readability=e.readability);for(var a=0;a<e.attributes.length;a++)try{i.setAttribute(e.attributes[a].name,e.attributes[a].value)}catch(e){}return i},_prepArticle:function(e){this._cleanStyles(e),this._markDataTables(e),this._cleanConditionally(e,"form"),this._cleanConditionally(e,"fieldset"),this._clean(e,"object"),this._clean(e,"embed"),this._clean(e,"h1"),this._clean(e,"footer"),this._clean(e,"link"),this._clean(e,"aside");var i=this.DEFAULT_CHAR_THRESHOLD;this._forEachNode(e.children,function(e){this._cleanMatchedNodes(e,function(e,t){return/share/.test(t)&&e.textContent.length<i})});var t=e.getElementsByTagName("h2");if(1===t.length){var a=(t[0].textContent.length-this._articleTitle.length)/this._articleTitle.length;if(Math.abs(a)<.5){(0<a?t[0].textContent.includes(this._articleTitle):this._articleTitle.includes(t[0].textContent))&&this._clean(e,"h2")}}this._clean(e,"iframe"),this._clean(e,"input"),this._clean(e,"textarea"),this._clean(e,"select"),this._clean(e,"button"),this._cleanHeaders(e),this._cleanConditionally(e,"table"),this._cleanConditionally(e,"ul"),this._cleanConditionally(e,"div"),this._removeNodes(e.getElementsByTagName("p"),function(e){return 0===e.getElementsByTagName("img").length+e.getElementsByTagName("embed").length+e.getElementsByTagName("object").length+e.getElementsByTagName("iframe").length&&!this._getInnerText(e,!1)}),this._forEachNode(this._getAllNodesWithTag(e,["br"]),function(e){var t=this._nextElement(e.nextSibling);t&&"P"==t.tagName&&e.parentNode.removeChild(e)}),this._forEachNode(this._getAllNodesWithTag(e,["table"]),function(e){var t=this._hasSingleTagInsideElement(e,"TBODY")?e.firstElementChild:e;if(this._hasSingleTagInsideElement(t,"TR")){var i=t.firstElementChild;if(this._hasSingleTagInsideElement(i,"TD")){var a=i.firstElementChild;a=this._setNodeTag(a,this._everyNode(a.childNodes,this._isPhrasingContent)?"P":"DIV"),e.parentNode.replaceChild(a,e)}}})},_initializeNode:function(e){switch(e.readability={contentScore:0},e.tagName){case"DIV":e.readability.contentScore+=5;break;case"PRE":case"TD":case"BLOCKQUOTE":e.readability.contentScore+=3;break;case"ADDRESS":case"OL":case"UL":case"DL":case"DD":case"DT":case"LI":case"FORM":e.readability.contentScore-=3;break;case"H1":case"H2":case"H3":case"H4":case"H5":case"H6":case"TH":e.readability.contentScore-=5}e.readability.contentScore+=this._getClassWeight(e)},_removeAndGetNext:function(e){var t=this._getNextNode(e,!0);return e.parentNode.removeChild(e),t},_getNextNode:function(e,t){if(!t&&e.firstElementChild)return e.firstElementChild;if(e.nextElementSibling)return e.nextElementSibling;for(;(e=e.parentNode)&&!e.nextElementSibling;);return e&&e.nextElementSibling},_checkByline:function(e,t){if(this._articleByline)return!1;if(void 0!==e.getAttribute)var i=e.getAttribute("rel"),a=e.getAttribute("itemprop");return!(!("author"===i||a&&-1!==a.indexOf("author")||this.REGEXPS.byline.test(t))||!this._isValidByline(e.textContent))&&(this._articleByline=e.textContent.trim(),!0)},_getNodeAncestors:function(e,t){t=t||0;for(var i=0,a=[];e.parentNode&&(a.push(e.parentNode),!t||++i!==t);)e=e.parentNode;return a},_grabArticle:function(e){this.log("**** grabArticle ****");var t=this._doc,i=null!==e;if(!(e=e||this._doc.body))return this.log("No body found in document. Abort."),null;for(var a=e.innerHTML;;){for(var n=this._flagIsActive(this.FLAG_STRIP_UNLIKELYS),r=[],s=this._doc.documentElement;s;){var o=s.className+" "+s.id;if(this._isProbablyVisible(s))if(this._checkByline(s,o))s=this._removeAndGetNext(s);else if(!n||!this.REGEXPS.unlikelyCandidates.test(o)||this.REGEXPS.okMaybeItsACandidate.test(o)||this._hasAncestorTag(s,"table")||"BODY"===s.tagName||"A"===s.tagName)if("DIV"!==s.tagName&&"SECTION"!==s.tagName&&"HEADER"!==s.tagName&&"H1"!==s.tagName&&"H2"!==s.tagName&&"H3"!==s.tagName&&"H4"!==s.tagName&&"H5"!==s.tagName&&"H6"!==s.tagName||!this._isElementWithoutContent(s)){if(-1!==this.DEFAULT_TAGS_TO_SCORE.indexOf(s.tagName)&&r.push(s),"DIV"===s.tagName){for(var l=null,h=s.firstChild;h;){var c=h.nextSibling;if(this._isPhrasingContent(h))null!==l?l.appendChild(h):this._isWhitespace(h)||(l=t.createElement("p"),s.replaceChild(l,h),l.appendChild(h));else if(null!==l){for(;l.lastChild&&this._isWhitespace(l.lastChild);)l.removeChild(l.lastChild);l=null}h=c}if(this._hasSingleTagInsideElement(s,"P")&&this._getLinkDensity(s)<.25){var d=s.children[0];s.parentNode.replaceChild(d,s),s=d,r.push(s)}else this._hasChildBlockElement(s)||(s=this._setNodeTag(s,"P"),r.push(s))}s=this._getNextNode(s)}else s=this._removeAndGetNext(s);else this.log("Removing unlikely candidate - "+o),s=this._removeAndGetNext(s);else this.log("Removing hidden node - "+o),s=this._removeAndGetNext(s)}var g=[];this._forEachNode(r,function(e){if(e.parentNode&&void 0!==e.parentNode.tagName){var t=this._getInnerText(e);if(!(t.length<25)){var i=this._getNodeAncestors(e,3);if(0!==i.length){var a=0;a+=1,a+=t.split(",").length,a+=Math.min(Math.floor(t.length/100),3),this._forEachNode(i,function(e,t){if(e.tagName&&e.parentNode&&void 0!==e.parentNode.tagName){if(void 0===e.readability&&(this._initializeNode(e),g.push(e)),0===t)var i=1;else i=1===t?2:3*t;e.readability.contentScore+=a/i}})}}}});for(var _=[],m=0,u=g.length;m<u;m+=1){var f=g[m],N=f.readability.contentScore*(1-this._getLinkDensity(f));f.readability.contentScore=N,this.log("Candidate:",f,"with score "+N);for(var E=0;E<this._nbTopCandidates;E++){var p=_[E];if(!p||N>p.readability.contentScore){_.splice(E,0,f),_.length>this._nbTopCandidates&&_.pop();break}}}var T,b=_[0]||null,y=!1;if(null===b||"BODY"===b.tagName){b=t.createElement("DIV"),y=!0;for(var v=e.childNodes;v.length;)this.log("Moving child out:",v[0]),b.appendChild(v[0]);e.appendChild(b),this._initializeNode(b)}else if(b){for(var A=[],C=1;C<_.length;C++).75<=_[C].readability.contentScore/b.readability.contentScore&&A.push(this._getNodeAncestors(_[C]));if(3<=A.length)for(T=b.parentNode;"BODY"!==T.tagName;){for(var S=0,L=0;L<A.length&&S<3;L++)S+=Number(A[L].includes(T));if(3<=S){b=T;break}T=T.parentNode}b.readability||this._initializeNode(b),T=b.parentNode;for(var x=b.readability.contentScore,I=x/3;"BODY"!==T.tagName;)if(T.readability){var D=T.readability.contentScore;if(D<I)break;if(x<D){b=T;break}x=T.readability.contentScore,T=T.parentNode}else T=T.parentNode;for(T=b.parentNode;"BODY"!=T.tagName&&1==T.children.length;)T=(b=T).parentNode;b.readability||this._initializeNode(b)}var R=t.createElement("DIV");i&&(R.id="readability-content");for(var B=Math.max(10,.2*b.readability.contentScore),O=(T=b.parentNode).children,P=0,G=O.length;P<G;P++){var M=O[P],w=!1;if(this.log("Looking at sibling node:",M,M.readability?"with score "+M.readability.contentScore:""),this.log("Sibling has score",M.readability?M.readability.contentScore:"Unknown"),M===b)w=!0;else{var H=0;if(M.className===b.className&&""!==b.className&&(H+=.2*b.readability.contentScore),M.readability&&M.readability.contentScore+H>=B)w=!0;else if("P"===M.nodeName){var U=this._getLinkDensity(M),k=this._getInnerText(M),F=k.length;80<F&&U<.25?w=!0:F<80&&0<F&&0===U&&-1!==k.search(/\.( |$)/)&&(w=!0)}}w&&(this.log("Appending node:",M),-1===this.ALTER_TO_DIV_EXCEPTIONS.indexOf(M.nodeName)&&(this.log("Altering sibling:",M,"to div."),M=this._setNodeTag(M,"DIV")),R.appendChild(M),P-=1,G-=1)}if(this._debug&&this.log("Article content pre-prep: "+R.innerHTML),this._prepArticle(R),this._debug&&this.log("Article content post-prep: "+R.innerHTML),y)b.id="readability-page-1",b.className="page";else{var X=t.createElement("DIV");X.id="readability-page-1",X.className="page";for(var V=R.childNodes;V.length;)X.appendChild(V[0]);R.appendChild(X)}this._debug&&this.log("Article content after paging: "+R.innerHTML);var W=!0,Y=this._getInnerText(R,!0).length;if(Y<this._charThreshold)if(W=!1,e.innerHTML=a,this._flagIsActive(this.FLAG_STRIP_UNLIKELYS))this._removeFlag(this.FLAG_STRIP_UNLIKELYS),this._attempts.push({articleContent:R,textLength:Y});else if(this._flagIsActive(this.FLAG_WEIGHT_CLASSES))this._removeFlag(this.FLAG_WEIGHT_CLASSES),this._attempts.push({articleContent:R,textLength:Y});else if(this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY))this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY),this._attempts.push({articleContent:R,textLength:Y});else{if(this._attempts.push({articleContent:R,textLength:Y}),this._attempts.sort(function(e,t){return t.textLength-e.textLength}),!this._attempts[0].textLength)return null;R=this._attempts[0].articleContent,W=!0}if(W){var j=[T,b].concat(this._getNodeAncestors(T));return this._someNode(j,function(e){if(!e.tagName)return!1;var t=e.getAttribute("dir");return!!t&&(this._articleDir=t,!0)}),R}}},_isValidByline:function(e){return("string"==typeof e||e instanceof String)&&(0<(e=e.trim()).length&&e.length<100)},_getArticleMetadata:function(){var e={},o={},t=this._doc.getElementsByTagName("meta"),l=/\s*(dc|dcterm|og|twitter)\s*:\s*(author|creator|description|title|site_name)\s*/gi,h=/^\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\s*[\.:]\s*)?(author|creator|description|title|site_name)\s*$/i;return this._forEachNode(t,function(e){var t=e.getAttribute("name"),i=e.getAttribute("property"),a=e.getAttribute("content");if(a){var n=null,r=null;if(i&&(n=i.match(l)))for(var s=n.length-1;0<=s;s--)r=n[s].toLowerCase().replace(/\s/g,""),o[r]=a.trim();!n&&t&&h.test(t)&&(r=t,a&&(r=r.toLowerCase().replace(/\s/g,"").replace(/\./g,":"),o[r]=a.trim()))}}),e.title=o["dc:title"]||o["dcterm:title"]||o["og:title"]||o["weibo:article:title"]||o["weibo:webpage:title"]||o.title||o["twitter:title"],e.title||(e.title=this._getArticleTitle()),e.byline=o["dc:creator"]||o["dcterm:creator"]||o.author,e.excerpt=o["dc:description"]||o["dcterm:description"]||o["og:description"]||o["weibo:article:description"]||o["weibo:webpage:description"]||o.description||o["twitter:description"],e.siteName=o["og:site_name"],e},_removeScripts:function(e){this._removeNodes(e.getElementsByTagName("script"),function(e){return e.nodeValue="",e.removeAttribute("src"),!0}),this._removeNodes(e.getElementsByTagName("noscript"))},_hasSingleTagInsideElement:function(e,t){return 1==e.children.length&&e.children[0].tagName===t&&!this._someNode(e.childNodes,function(e){return e.nodeType===this.TEXT_NODE&&this.REGEXPS.hasContent.test(e.textContent)})},_isElementWithoutContent:function(e){return e.nodeType===this.ELEMENT_NODE&&0==e.textContent.trim().length&&(0==e.children.length||e.children.length==e.getElementsByTagName("br").length+e.getElementsByTagName("hr").length)},_hasChildBlockElement:function(e){return this._someNode(e.childNodes,function(e){return-1!==this.DIV_TO_P_ELEMS.indexOf(e.tagName)||this._hasChildBlockElement(e)})},_isPhrasingContent:function(e){return e.nodeType===this.TEXT_NODE||-1!==this.PHRASING_ELEMS.indexOf(e.tagName)||("A"===e.tagName||"DEL"===e.tagName||"INS"===e.tagName)&&this._everyNode(e.childNodes,this._isPhrasingContent)},_isWhitespace:function(e){return e.nodeType===this.TEXT_NODE&&0===e.textContent.trim().length||e.nodeType===this.ELEMENT_NODE&&"BR"===e.tagName},_getInnerText:function(e,t){t=void 0===t||t;var i=e.textContent.trim();return t?i.replace(this.REGEXPS.normalize," "):i},_getCharCount:function(e,t){return t=t||",",this._getInnerText(e).split(t).length-1},_cleanStyles:function(e){if(e&&"svg"!==e.tagName.toLowerCase()){for(var t=0;t<this.PRESENTATIONAL_ATTRIBUTES.length;t++)e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[t]);-1!==this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.indexOf(e.tagName)&&(e.removeAttribute("width"),e.removeAttribute("height"));for(var i=e.firstElementChild;null!==i;)this._cleanStyles(i),i=i.nextElementSibling}},_getLinkDensity:function(e){var t=this._getInnerText(e).length;if(0===t)return 0;var i=0;return this._forEachNode(e.getElementsByTagName("a"),function(e){i+=this._getInnerText(e).length}),i/t},_getClassWeight:function(e){if(!this._flagIsActive(this.FLAG_WEIGHT_CLASSES))return 0;var t=0;return"string"==typeof e.className&&""!==e.className&&(this.REGEXPS.negative.test(e.className)&&(t-=25),this.REGEXPS.positive.test(e.className)&&(t+=25)),"string"==typeof e.id&&""!==e.id&&(this.REGEXPS.negative.test(e.id)&&(t-=25),this.REGEXPS.positive.test(e.id)&&(t+=25)),t},_clean:function(e,t){var i=-1!==["object","embed","iframe"].indexOf(t);this._removeNodes(e.getElementsByTagName(t),function(e){if(i){for(var t=0;t<e.attributes.length;t++)if(this.REGEXPS.videos.test(e.attributes[t].value))return!1;if("object"===e.tagName&&this.REGEXPS.videos.test(e.innerHTML))return!1}return!0})},_hasAncestorTag:function(e,t,i,a){i=i||3,t=t.toUpperCase();for(var n=0;e.parentNode;){if(0<i&&i<n)return!1;if(e.parentNode.tagName===t&&(!a||a(e.parentNode)))return!0;e=e.parentNode,n++}return!1},_getRowAndColumnCount:function(e){for(var t=0,i=0,a=e.getElementsByTagName("tr"),n=0;n<a.length;n++){var r=a[n].getAttribute("rowspan")||0;r&&(r=parseInt(r,10)),t+=r||1;for(var s=0,o=a[n].getElementsByTagName("td"),l=0;l<o.length;l++){var h=o[l].getAttribute("colspan")||0;h&&(h=parseInt(h,10)),s+=h||1}i=Math.max(i,s)}return{rows:t,columns:i}},_markDataTables:function(e){for(var t=e.getElementsByTagName("table"),i=0;i<t.length;i++){var a=t[i];if("presentation"!=a.getAttribute("role"))if("0"!=a.getAttribute("datatable"))if(a.getAttribute("summary"))a._readabilityDataTable=!0;else{var n=a.getElementsByTagName("caption")[0];if(n&&0<n.childNodes.length)a._readabilityDataTable=!0;else{if(["col","colgroup","tfoot","thead","th"].some(function(e){return!!a.getElementsByTagName(e)[0]}))this.log("Data table because found data-y descendant"),a._readabilityDataTable=!0;else if(a.getElementsByTagName("table")[0])a._readabilityDataTable=!1;else{var r=this._getRowAndColumnCount(a);10<=r.rows||4<r.columns?a._readabilityDataTable=!0:a._readabilityDataTable=10<r.rows*r.columns}}}else a._readabilityDataTable=!1;else a._readabilityDataTable=!1}},_cleanConditionally:function(e,_){if(this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)){var m="ul"===_||"ol"===_;this._removeNodes(e.getElementsByTagName(_),function(e){var t=function(e){return e._readabilityDataTable};if("table"===_&&t(e))return!1;if(this._hasAncestorTag(e,"table",-1,t))return!1;var i=this._getClassWeight(e);if(this.log("Cleaning Conditionally",e),i+0<0)return!0;if(this._getCharCount(e,",")<10){for(var a=e.getElementsByTagName("p").length,n=e.getElementsByTagName("img").length,r=e.getElementsByTagName("li").length-100,s=e.getElementsByTagName("input").length,o=0,l=this._concatNodeLists(e.getElementsByTagName("object"),e.getElementsByTagName("embed"),e.getElementsByTagName("iframe")),h=0;h<l.length;h++){for(var c=0;c<l[h].attributes.length;c++)if(this.REGEXPS.videos.test(l[h].attributes[c].value))return!1;if("object"===l[h].tagName&&this.REGEXPS.videos.test(l[h].innerHTML))return!1;o++}var d=this._getLinkDensity(e),g=this._getInnerText(e).length;return 1<n&&a/n<.5&&!this._hasAncestorTag(e,"figure")||!m&&a<r||s>Math.floor(a/3)||!m&&g<25&&(0===n||2<n)&&!this._hasAncestorTag(e,"figure")||!m&&i<25&&.2<d||25<=i&&.5<d||1===o&&g<75||1<o}return!1})}},_cleanMatchedNodes:function(e,t){for(var i=this._getNextNode(e,!0),a=this._getNextNode(e);a&&a!=i;)a=t(a,a.className+" "+a.id)?this._removeAndGetNext(a):this._getNextNode(a)},_cleanHeaders:function(e){for(var t=1;t<3;t+=1)this._removeNodes(e.getElementsByTagName("h"+t),function(e){return this._getClassWeight(e)<0})},_flagIsActive:function(e){return 0<(this._flags&e)},_removeFlag:function(e){this._flags=this._flags&~e},_isProbablyVisible:function(e){return!(e.style&&"none"==e.style.display||e.hasAttribute("hidden"))},parse:function(){if(0<this._maxElemsToParse){var e=this._doc.getElementsByTagName("*").length;if(e>this._maxElemsToParse)throw new Error("Aborting parsing document; "+e+" elements found")}this._removeScripts(this._doc),this._prepDocument();var t=this._getArticleMetadata();this._articleTitle=t.title;var i=this._grabArticle();if(!i)return null;if(this.log("Grabbed: "+i.innerHTML),this._postProcessContent(i),!t.excerpt){var a=i.getElementsByTagName("p");0<a.length&&(t.excerpt=a[0].textContent.trim())}var n=i.textContent;return{title:this._articleTitle,byline:t.byline||this._articleByline,dir:this._articleDir,content:i.innerHTML,textContent:n,length:n.length,excerpt:t.excerpt,siteName:t.siteName||this._articleSiteName}}},"object"==typeof module&&(module.exports=Readability); + return JSON.stringify(new Readability(document).parse()); +} + +/************** Readability.js Licensce ****************/ +/***** From: https://github.com/mozilla/readability ****/ +/* + * Copyright (c) 2010 Arc90 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This code is heavily based on Arc90's readability.js (1.7.1) script + * available at: http://code.google.com/p/arc90labs-readability + */ From 002206d4fe5bef619c0b0d448b1df1146dde9fd6 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 11:39:51 -0400 Subject: [PATCH 019/365] remove distutils in favor of shutil --- archivebox/index.py | 9 ++------- archivebox/util.py | 7 +++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index 116de5a7..c1ea5dc5 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -6,12 +6,6 @@ from string import Template from typing import List, Tuple, Iterator, Optional from dataclasses import fields -try: - from distutils.dir_util import copy_tree -except ImportError: - print('[X] Missing "distutils" python package. To install it, run:') - print(' pip install distutils') - from schema import Link, ArchiveIndex, ArchiveResult from config import ( OUTPUT_DIR, @@ -29,6 +23,7 @@ from util import ( ExtendedEncoder, enforce_types, TimedProgress, + copy_and_overwrite, ) from parse import parse_links from links import validate_links @@ -149,7 +144,7 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False path = os.path.join(out_dir, 'index.html') - copy_tree(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static')) + copy_and_overwrite(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static')) with open(os.path.join(out_dir, 'robots.txt'), 'w+') as f: f.write('User-agent: *\nDisallow: /') diff --git a/archivebox/util.py b/archivebox/util.py index 970085ea..e6f93981 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -2,6 +2,7 @@ import os import re import sys import time +import shutil from json import JSONEncoder from typing import List, Optional, Any @@ -604,6 +605,12 @@ def chmod_file(path: str, cwd: str='.', permissions: str=OUTPUT_PERMISSIONS, tim raise Exception('Failed to chmod {}/{}'.format(cwd, path)) +@enforce_types +def copy_and_overwrite(from_path: str, to_path: str): + if os.path.exists(to_path): + shutil.rmtree(to_path) + shutil.copytree(from_path, to_path) + @enforce_types def chrome_args(**options) -> List[str]: """helper to build up a chrome shell command with arguments""" From dff830196f70360713ace0a876ca86fad0ffd1a3 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 11:40:00 -0400 Subject: [PATCH 020/365] add setup.py --- setup.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..64c9f104 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="archivebox", + version="0.3.0", + author="Nick Sweeting", + author_email="git@nicksweeting.com", + description="The self-hosted internet archive.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/pirate/ArchiveBox", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], +) From 79b319a48c3d4c075ce9e373b0118384d82e1e0c Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:13:47 -0400 Subject: [PATCH 021/365] make archive.py runnable as package and as module --- archivebox/__init__.py | 5 +++++ archivebox/archive.py | 1 + 2 files changed, 6 insertions(+) diff --git a/archivebox/__init__.py b/archivebox/__init__.py index e69de29b..0fb9e6f8 100644 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -0,0 +1,5 @@ + + +__name__ = 'archivebox' +__package__ = 'archivebox' + diff --git a/archivebox/archive.py b/archivebox/archive.py index ff4128c9..965f6a48 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -8,6 +8,7 @@ but you can also run it directly using `python3 archive.py` Usage & Documentation: https://github.com/pirate/ArchiveBox/Wiki """ +__package__ = 'archivebox' import os import sys From 2e8008009e9265742579f1ec3ff63e701daec870 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:14:22 -0400 Subject: [PATCH 022/365] make main compatible with setuptools entrypoint api --- archivebox/archive.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index 965f6a48..7cf56197 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -48,13 +48,19 @@ def print_help(): print("UI Usage:") print(" Open output/index.html to view your archive.\n") print("CLI Usage:") - print(" echo 'https://example.com' | ./archive\n") - print(" ./archive ~/Downloads/bookmarks_export.html\n") - print(" ./archive https://example.com/feed.rss\n") - print(" ./archive 15109948213.123\n") + print(" mkdir data; cd data/") + print(" archivebox init\n") + print(" echo 'https://example.com/some/page' | archivebox add") + print(" archivebox add https://example.com/some/other/page") + print(" archivebox add --depth=1 ~/Downloads/bookmarks_export.html") + print(" archivebox add --depth=1 https://example.com/feed.rss") + print(" archivebox update --resume=15109948213.123") -def main(*args) -> List[Link]: +def main(args=None) -> List[Link]: + if args is None: + args = sys.argv + if set(args).intersection(('-h', '--help', 'help')) or len(args) > 2: print_help() raise SystemExit(0) @@ -99,7 +105,7 @@ def main(*args) -> List[Link]: import_path = save_remote_source(import_path) ### Run the main archive update process - return update_archive_data(import_path=import_path, resume=resume) + update_archive_data(import_path=import_path, resume=resume) @enforce_types @@ -138,4 +144,4 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] return all_links if __name__ == '__main__': - main(*sys.argv) + main(sys.argv) From 0ad8d51f0eb0f7afd60c8a75721a0bb0b957b55e Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:14:47 -0400 Subject: [PATCH 023/365] remove base32 crockford in favor of dependency --- archivebox/base32_crockford.py | 172 --------------------------------- 1 file changed, 172 deletions(-) delete mode 100644 archivebox/base32_crockford.py diff --git a/archivebox/base32_crockford.py b/archivebox/base32_crockford.py deleted file mode 100644 index bafb69b4..00000000 --- a/archivebox/base32_crockford.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -base32-crockford -================ - -A Python module implementing the alternate base32 encoding as described -by Douglas Crockford at: http://www.crockford.com/wrmg/base32.html. - -He designed the encoding to: - - * Be human and machine readable - * Be compact - * Be error resistant - * Be pronounceable - -It uses a symbol set of 10 digits and 22 letters, excluding I, L O and -U. Decoding is not case sensitive, and 'i' and 'l' are converted to '1' -and 'o' is converted to '0'. Encoding uses only upper-case characters. - -Hyphens may be present in symbol strings to improve readability, and -are removed when decoding. - -A check symbol can be appended to a symbol string to detect errors -within the string. - -""" - -import re -import sys - -PY3 = sys.version_info[0] == 3 - -if not PY3: - import string as str - - -__all__ = ["encode", "decode", "normalize"] - - -if PY3: - string_types = str, -else: - string_types = basestring, - -# The encoded symbol space does not include I, L, O or U -symbols = '0123456789ABCDEFGHJKMNPQRSTVWXYZ' -# These five symbols are exclusively for checksum values -check_symbols = '*~$=U' - -encode_symbols = dict((i, ch) for (i, ch) in enumerate(symbols + check_symbols)) -decode_symbols = dict((ch, i) for (i, ch) in enumerate(symbols + check_symbols)) -normalize_symbols = str.maketrans('IiLlOo', '111100') -valid_symbols = re.compile('^[%s]+[%s]?$' % (symbols, - re.escape(check_symbols))) - -base = len(symbols) -check_base = len(symbols + check_symbols) - - -def encode(number, checksum=False, split=0): - """Encode an integer into a symbol string. - - A ValueError is raised on invalid input. - - If checksum is set to True, a check symbol will be - calculated and appended to the string. - - If split is specified, the string will be divided into - clusters of that size separated by hyphens. - - The encoded string is returned. - """ - number = int(number) - if number < 0: - raise ValueError("number '%d' is not a positive integer" % number) - - split = int(split) - if split < 0: - raise ValueError("split '%d' is not a positive integer" % split) - - check_symbol = '' - if checksum: - check_symbol = encode_symbols[number % check_base] - - if number == 0: - return '0' + check_symbol - - symbol_string = '' - while number > 0: - remainder = number % base - number //= base - symbol_string = encode_symbols[remainder] + symbol_string - symbol_string = symbol_string + check_symbol - - if split: - chunks = [] - for pos in range(0, len(symbol_string), split): - chunks.append(symbol_string[pos:pos + split]) - symbol_string = '-'.join(chunks) - - return symbol_string - - -def decode(symbol_string, checksum=False, strict=False): - """Decode an encoded symbol string. - - If checksum is set to True, the string is assumed to have a - trailing check symbol which will be validated. If the - checksum validation fails, a ValueError is raised. - - If strict is set to True, a ValueError is raised if the - normalization step requires changes to the string. - - The decoded string is returned. - """ - symbol_string = normalize(symbol_string, strict=strict) - if checksum: - symbol_string, check_symbol = symbol_string[:-1], symbol_string[-1] - - number = 0 - for symbol in symbol_string: - number = number * base + decode_symbols[symbol] - - if checksum: - check_value = decode_symbols[check_symbol] - modulo = number % check_base - if check_value != modulo: - raise ValueError("invalid check symbol '%s' for string '%s'" % - (check_symbol, symbol_string)) - - return number - - -def normalize(symbol_string, strict=False): - """Normalize an encoded symbol string. - - Normalization provides error correction and prepares the - string for decoding. These transformations are applied: - - 1. Hyphens are removed - 2. 'I', 'i', 'L' or 'l' are converted to '1' - 3. 'O' or 'o' are converted to '0' - 4. All characters are converted to uppercase - - A TypeError is raised if an invalid string type is provided. - - A ValueError is raised if the normalized string contains - invalid characters. - - If the strict parameter is set to True, a ValueError is raised - if any of the above transformations are applied. - - The normalized string is returned. - """ - if isinstance(symbol_string, string_types): - if not PY3: - try: - symbol_string = symbol_string.encode('ascii') - except UnicodeEncodeError: - raise ValueError("string should only contain ASCII characters") - else: - raise TypeError("string is of invalid type %s" % - symbol_string.__class__.__name__) - - norm_string = symbol_string.replace('-', '').translate(normalize_symbols).upper() - - if not valid_symbols.match(norm_string): - raise ValueError("string '%s' contains invalid characters" % norm_string) - - if strict and norm_string != symbol_string: - raise ValueError("string '%s' requires normalization" % symbol_string) - - return norm_string From b9839500b272a9794ab17a63fd49013c7137d13f Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:15:51 -0400 Subject: [PATCH 024/365] make archivebox use current directory as OUTPUT_DIR by default --- archivebox/config.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index 1a6b6d6d..e564942e 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -66,15 +66,30 @@ REPO_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__ if OUTPUT_DIR: OUTPUT_DIR = os.path.abspath(OUTPUT_DIR) else: - OUTPUT_DIR = os.path.join(REPO_DIR, 'output') + OUTPUT_DIR = os.path.abspath(os.curdir) + +if not os.path.exists(OUTPUT_DIR): + print('{green}[+] Created a new archive directory: {}{reset}'.format(OUTPUT_DIR, **ANSI)) + os.makedirs(OUTPUT_DIR) +else: + not_empty = len(set(os.listdir(OUTPUT_DIR)) - {'.DS_Store'}) + index_exists = os.path.exists(os.path.join(OUTPUT_DIR, 'index.json')) + if not_empty and not index_exists: + print( + ('{red}[X] Could not find index.json in the OUTPUT_DIR: {reset}{}\n' + ' You must run ArchiveBox in an existing archive directory, \n' + ' or an empty/new directory to start a new archive collection.' + ).format(OUTPUT_DIR, **ANSI) + ) + raise SystemExit(1) ARCHIVE_DIR_NAME = 'archive' SOURCES_DIR_NAME = 'sources' ARCHIVE_DIR = os.path.join(OUTPUT_DIR, ARCHIVE_DIR_NAME) SOURCES_DIR = os.path.join(OUTPUT_DIR, SOURCES_DIR_NAME) -PYTHON_PATH = os.path.join(REPO_DIR, 'archivebox') -TEMPLATES_DIR = os.path.join(PYTHON_PATH, 'templates') +PYTHON_DIR = os.path.join(REPO_DIR, 'archivebox') +TEMPLATES_DIR = os.path.join(PYTHON_DIR, 'templates') if COOKIES_FILE: COOKIES_FILE = os.path.abspath(COOKIES_FILE) From 88721512d40395a854b53cdddfed41f88c0cbea4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:16:53 -0400 Subject: [PATCH 025/365] more detailed parsing and indexing cli output --- archivebox/index.py | 5 +++-- archivebox/logs.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index c1ea5dc5..50cd000f 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -76,10 +76,11 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) - # merge existing links in out_dir and new links all_links = list(validate_links(existing_links + new_links)) - num_new_links = len(all_links) - len(existing_links) if import_path and parser_name: - log_parsing_finished(num_new_links, parser_name) + num_parsed = len(raw_links) + num_new_links = len(all_links) - len(existing_links) + log_parsing_finished(num_parsed, num_new_links, parser_name) return all_links, new_links diff --git a/archivebox/logs.py b/archivebox/logs.py index b2913c18..660e27cc 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -37,12 +37,13 @@ def log_parsing_started(source_file: str): **ANSI, )) -def log_parsing_finished(num_new_links: int, parser_name: str): +def log_parsing_finished(num_parsed: int, num_new_links: int, parser_name: str): end_ts = datetime.now() _LAST_RUN_STATS.parse_end_ts = end_ts - print(' > Adding {} new links to index (parsed import as {})'.format( + print(' > Parsed {} links as {}'.format(num_parsed, parser_name)) + print(' > Adding {} new links to collection: {}'.format( num_new_links, - parser_name, + OUTPUT_DIR, )) @@ -95,12 +96,10 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str): timestamp=timestamp, total=num_links, )) - print(' To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', ''))) - print(' Continue where you left off by running:') - print(' {} {}'.format( - pretty_path(sys.argv[0]), - timestamp, - )) + print(' To view your archive, open:') + print(' {}/index.html'.format(OUTPUT_DIR)) + print(' Continue archiving where you left off by running:') + print(' archivebox {}'.format(timestamp)) def log_archiving_finished(num_links: int): end_ts = datetime.now() @@ -121,7 +120,8 @@ def log_archiving_finished(num_links: int): print(' - {} links skipped'.format(_LAST_RUN_STATS.skipped)) print(' - {} links updated'.format(_LAST_RUN_STATS.succeeded)) print(' - {} links had errors'.format(_LAST_RUN_STATS.failed)) - print(' To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', ''))) + print(' To view your archive, open:') + print(' {}/index.html'.format(OUTPUT_DIR)) def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): From ea695b8bef54511d7b473a16aef3cbf42bafc269 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:32:39 -0400 Subject: [PATCH 026/365] remove dataclass for ArchiveIndex in favor of plain dict to simplify schema file --- archivebox/index.py | 22 ++++++++++------------ archivebox/logs.py | 45 +++++++++++++++++++++++++++----------------- archivebox/schema.py | 35 ---------------------------------- 3 files changed, 38 insertions(+), 64 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index 50cd000f..58b752b1 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -101,20 +101,18 @@ def write_json_links_index(out_dir: str, links: List[Link]) -> None: path = os.path.join(out_dir, 'index.json') - index_json = ArchiveIndex( - info='ArchiveBox Index', - source='https://github.com/pirate/ArchiveBox', - docs='https://github.com/pirate/ArchiveBox/wiki', - version=GIT_SHA, - num_links=len(links), - updated=datetime.now(), - links=links, - ) - - assert isinstance(index_json._asdict(), dict) + index_json = { + 'info': 'ArchiveBox Index', + 'source': 'https://github.com/pirate/ArchiveBox', + 'docs': 'https://github.com/pirate/ArchiveBox/wiki', + 'version': VERSION, + 'num_links': len(links), + 'updated': datetime.now(), + 'links': links, + } with open(path, 'w', encoding='utf-8') as f: - json.dump(index_json._asdict(), f, indent=4, cls=ExtendedEncoder) + json.dump(index_json, f, indent=4, cls=ExtendedEncoder) chmod_file(path) diff --git a/archivebox/logs.py b/archivebox/logs.py index 660e27cc..ccb9a10c 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -1,29 +1,40 @@ +import os import sys -from datetime import datetime +from datetime import datetime +from dataclasses import dataclass from typing import Optional -from schema import Link, ArchiveResult, RuntimeStats -from config import ANSI, REPO_DIR, OUTPUT_DIR + +from .schema import Link, ArchiveResult +from .config import ANSI, REPO_DIR, OUTPUT_DIR + + +@dataclass +class RuntimeStats: + """mutable stats counter for logging archiving timing info to CLI output""" + + skipped: int = 0 + succeeded: int = 0 + failed: int = 0 + + parse_start_ts: datetime = None + parse_end_ts: datetime = None + + index_start_ts: datetime = None + index_end_ts: datetime = None + + archiving_start_ts: datetime = None + archiving_end_ts: datetime = None # globals are bad, mmkay -_LAST_RUN_STATS = RuntimeStats( - skipped=0, - succeeded=0, - failed=0, +_LAST_RUN_STATS = RuntimeStats() - parse_start_ts=0, - parse_end_ts=0, - - index_start_ts=0, - index_end_ts=0, - - archiving_start_ts=0, - archiving_end_ts=0, -) def pretty_path(path: str) -> str: """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc""" - return path.replace(REPO_DIR + '/', '') + pwd = os.path.abspath('.') + # parent = os.path.abspath(os.path.join(pwd, os.path.pardir)) + return path.replace(pwd + '/', './') ### Parsing Stage diff --git a/archivebox/schema.py b/archivebox/schema.py index 5aa629d7..619ffd7c 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -268,38 +268,3 @@ class Link: 'dom_url': static_url, }) return canonical - - -@dataclass(frozen=True) -class ArchiveIndex: - info: str - version: str - source: str - docs: str - num_links: int - updated: str - links: List[Link] - schema: str = 'ArchiveIndex' - - def __post_init__(self): - assert self.schema == self.__class__.__name__ - - def _asdict(self): - return asdict(self) - -@dataclass -class RuntimeStats: - """mutable stats counter for logging archiving timing info to CLI output""" - - skipped: int - succeeded: int - failed: int - - parse_start_ts: datetime - parse_end_ts: datetime - - index_start_ts: datetime - index_end_ts: datetime - - archiving_start_ts: datetime - archiving_end_ts: datetime From b1b0c8d1c5f65a849f36eb3a0433e6f0cbc54e91 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:33:12 -0400 Subject: [PATCH 027/365] show prettier failure output during link archiving --- archivebox/logs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/archivebox/logs.py b/archivebox/logs.py index ccb9a10c..a4b83cfd 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -190,11 +190,11 @@ def log_archive_method_finished(result: ArchiveResult): # Collect and prefix output lines with indentation output_lines = [ - '{}Failed:{} {}{}'.format( - ANSI['red'], - result.output.__class__.__name__.replace('ArchiveError', ''), - result.output, - ANSI['reset'] + '{lightred}Failed:{reset}'.format(**ANSI), + ' {reset}{} {red}{}{reset}'.format( + result.output.__class__.__name__.replace('ArchiveError', ''), + result.output, + **ANSI, ), *hints, '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']), @@ -206,3 +206,4 @@ def log_archive_method_finished(result: ArchiveResult): for line in output_lines if line )) + print() From bc1bc9fe022097748abcafbd742de07a9af9b40a Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:33:59 -0400 Subject: [PATCH 028/365] htmldecode all urls and titles during parsing --- archivebox/parse.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/archivebox/parse.py b/archivebox/parse.py index 093d4a92..9430b305 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -96,9 +96,9 @@ def parse_pocket_html_export(html_file: IO[str]) -> Iterable[Link]: title = match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') yield Link( - url=url, + url=htmldecode(url), timestamp=str(time.timestamp()), - title=title or None, + title=htmldecode(title) or None, tags=tags or '', sources=[html_file.name], ) @@ -149,10 +149,10 @@ def parse_json_export(json_file: IO[str]) -> Iterable[Link]: title = link['name'].strip() yield Link( - url=url, + url=htmldecode(url), timestamp=ts_str, title=htmldecode(title) or None, - tags=link.get('tags') or '', + tags=htmldecode(link.get('tags')) or '', sources=[json_file.name], ) @@ -187,10 +187,10 @@ def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: title = str_between(get_row('title'), '<![CDATA[', ']]').strip() yield Link( - url=url, + url=htmldecode(url), timestamp=str(time.timestamp()), title=htmldecode(title) or None, - tags='', + tags=None, sources=[rss_file.name], ) @@ -225,10 +225,10 @@ def parse_shaarli_rss_export(rss_file: IO[str]) -> Iterable[Link]: time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") yield Link( - url=url, + url=htmldecode(url), timestamp=str(time.timestamp()), title=htmldecode(title) or None, - tags='', + tags=None, sources=[rss_file.name], ) @@ -250,10 +250,10 @@ def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: title = match.group(3).strip() yield Link( - url=url, + url=htmldecode(url), timestamp=str(time.timestamp()), title=htmldecode(title) or None, - tags='', + tags=None, sources=[html_file.name], ) @@ -282,10 +282,10 @@ def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: time = datetime.now() yield Link( - url=url, + url=htmldecode(url), timestamp=str(time.timestamp()), title=htmldecode(title) or None, - tags=tags or '', + tags=htmldecode(tags) or None, sources=[rss_file.name], ) @@ -304,10 +304,10 @@ def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z") yield Link( - url=url, + url=htmldecode(url), timestamp=str(time.timestamp()), title=htmldecode(title) or None, - tags='', + tags=None, sources=[rss_file.name], ) @@ -321,9 +321,9 @@ def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: urls = re.findall(URL_REGEX, line) if line.strip() else () for url in urls: yield Link( - url=url, + url=htmldecode(url), timestamp=str(datetime.now().timestamp()), title=None, - tags='', + tags=None, sources=[text_file.name], ) From 93216a3c3e59b7f2103faba53fb3f0de83e38cb5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:35:13 -0400 Subject: [PATCH 029/365] new version handling and absolute imports --- archivebox/archive.py | 21 ++++++++++++--------- archivebox/archive_methods.py | 15 ++++++++------- archivebox/config.py | 16 ++++------------ archivebox/index.py | 15 ++++++++------- archivebox/links.py | 4 ++-- archivebox/parse.py | 4 ++-- archivebox/schema.py | 32 ++++++++++++++++---------------- archivebox/templates/index.html | 2 +- archivebox/util.py | 10 +++++----- 9 files changed, 58 insertions(+), 61 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index 7cf56197..b381d141 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -13,34 +13,37 @@ __package__ = 'archivebox' import os import sys + from typing import List, Optional -from schema import Link -from links import links_after_timestamp -from index import write_links_index, load_links_index -from archive_methods import archive_link -from config import ( +from .schema import Link +from .links import links_after_timestamp +from .index import write_links_index, load_links_index +from .archive_methods import archive_link +from .config import ( ONLY_NEW, OUTPUT_DIR, - GIT_SHA, + PYTHON_DIR, + VERSION, ) -from util import ( +from .util import ( enforce_types, save_remote_source, save_stdin_source, ) -from logs import ( +from .logs import ( log_archiving_started, log_archiving_paused, log_archiving_finished, ) __AUTHOR__ = 'Nick Sweeting <git@nicksweeting.com>' -__VERSION__ = GIT_SHA[:9] +__VERSION__ = VERSION __DESCRIPTION__ = 'ArchiveBox: The self-hosted internet archive.' __DOCUMENTATION__ = 'https://github.com/pirate/ArchiveBox/wiki' + def print_help(): print('ArchiveBox: The self-hosted internet archive.\n') print("Documentation:") diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index fd726de2..2370c98b 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -4,13 +4,13 @@ from typing import Dict, List, Tuple from collections import defaultdict from datetime import datetime -from schema import Link, ArchiveResult, ArchiveError -from index import ( +from .schema import Link, ArchiveResult, ArchiveError +from .index import ( write_link_index, patch_links_index, load_json_link_index, ) -from config import ( +from .config import ( CURL_BINARY, GIT_BINARY, WGET_BINARY, @@ -31,7 +31,7 @@ from config import ( ANSI, OUTPUT_DIR, GIT_DOMAINS, - GIT_SHA, + VERSION, WGET_USER_AGENT, CHECK_SSL_VALIDITY, COOKIES_FILE, @@ -43,7 +43,7 @@ from config import ( ONLY_NEW, WGET_AUTO_COMPRESSION, ) -from util import ( +from .util import ( enforce_types, domain, extension, @@ -58,7 +58,7 @@ from util import ( run, PIPE, DEVNULL, Link, ) -from logs import ( +from .logs import ( log_link_archiving_started, log_link_archiving_finished, log_archive_method_started, @@ -123,6 +123,7 @@ def archive_link(link: Link, page=None) -> Link: if was_changed: patch_links_index(link) + log_link_archiving_finished(link.link_dir, link, is_new, stats) except KeyboardInterrupt: @@ -606,7 +607,7 @@ def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveR CURL_BINARY, '--location', '--head', - '--user-agent', 'ArchiveBox/{} (+https://github.com/pirate/ArchiveBox/)'.format(GIT_SHA), # be nice to the Archive.org people and show them where all this ArchiveBox traffic is coming from + '--user-agent', 'ArchiveBox/{} (+https://github.com/pirate/ArchiveBox/)'.format(VERSION), # be nice to the Archive.org people and show them where all this ArchiveBox traffic is coming from '--max-time', str(timeout), *(() if CHECK_SSL_VALIDITY else ('--insecure',)), submit_url, diff --git a/archivebox/config.py b/archivebox/config.py index e564942e..4573224f 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -40,7 +40,7 @@ SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True' CHECK_SSL_VALIDITY = os.getenv('CHECK_SSL_VALIDITY', 'True' ).lower() == 'true' RESOLUTION = os.getenv('RESOLUTION', '1440,2000' ) GIT_DOMAINS = os.getenv('GIT_DOMAINS', 'github.com,bitbucket.org,gitlab.com').split(',') -WGET_USER_AGENT = os.getenv('WGET_USER_AGENT', 'ArchiveBox/{GIT_SHA} (+https://github.com/pirate/ArchiveBox/) wget/{WGET_VERSION}') +WGET_USER_AGENT = os.getenv('WGET_USER_AGENT', 'ArchiveBox/{VERSION} (+https://github.com/pirate/ArchiveBox/) wget/{WGET_VERSION}') COOKIES_FILE = os.getenv('COOKIES_FILE', None) CHROME_USER_DATA_DIR = os.getenv('CHROME_USER_DATA_DIR', None) CHROME_HEADLESS = os.getenv('CHROME_HEADLESS', 'True' ).lower() == 'true' @@ -163,21 +163,13 @@ def find_chrome_data_dir() -> Optional[str]: return None -def get_git_version() -> str: - """get the git commit hash of the python code folder (aka code version)""" - try: - return run([GIT_BINARY, 'rev-list', '-1', 'HEAD', './'], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode() - except Exception: - print('[!] Warning: unable to determine git version, is git installed and in your $PATH?') - return 'unknown' - - # ****************************************************************************** # ************************ Environment & Dependencies ************************** # ****************************************************************************** try: - GIT_SHA = get_git_version() + VERSION = open(os.path.join(PYTHON_DIR, 'VERSION'), 'r').read().strip() + GIT_SHA = VERSION.split('+')[1] ### Terminal Configuration TERM_WIDTH = lambda: shutil.get_terminal_size((100, 10)).columns @@ -234,7 +226,7 @@ try: WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode WGET_USER_AGENT = WGET_USER_AGENT.format( - GIT_SHA=GIT_SHA[:9], + VERSION=VERSION, WGET_VERSION=WGET_VERSION or '', ) diff --git a/archivebox/index.py b/archivebox/index.py index 58b752b1..66b234a2 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -6,15 +6,16 @@ from string import Template from typing import List, Tuple, Iterator, Optional from dataclasses import fields -from schema import Link, ArchiveIndex, ArchiveResult -from config import ( +from .schema import Link, ArchiveResult +from .config import ( OUTPUT_DIR, TEMPLATES_DIR, + VERSION, GIT_SHA, FOOTER_INFO, TIMEOUT, ) -from util import ( +from .util import ( merge_links, chmod_file, urlencode, @@ -25,9 +26,9 @@ from util import ( TimedProgress, copy_and_overwrite, ) -from parse import parse_links -from links import validate_links -from logs import ( +from .parse import parse_links +from .links import validate_links +from .logs import ( log_indexing_process_started, log_indexing_started, log_indexing_finished, @@ -178,8 +179,8 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False 'date_updated': datetime.now().strftime('%Y-%m-%d'), 'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'), 'footer_info': FOOTER_INFO, + 'version': VERSION, 'git_sha': GIT_SHA, - 'short_git_sha': GIT_SHA[:8], 'rows': link_rows, 'status': 'finished' if finished else 'running', } diff --git a/archivebox/links.py b/archivebox/links.py index 4692943c..0d72472d 100644 --- a/archivebox/links.py +++ b/archivebox/links.py @@ -22,8 +22,8 @@ Link { from typing import Iterable from collections import OrderedDict -from schema import Link -from util import ( +from .schema import Link +from .util import ( scheme, fuzzy_url, merge_links, diff --git a/archivebox/parse.py b/archivebox/parse.py index 9430b305..6ecc0007 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -24,8 +24,8 @@ from typing import Tuple, List, IO, Iterable from datetime import datetime import xml.etree.ElementTree as etree -from config import TIMEOUT -from util import ( +from .config import TIMEOUT +from .util import ( htmldecode, str_between, URL_REGEX, diff --git a/archivebox/schema.py b/archivebox/schema.py index 619ffd7c..d1bb06ea 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -108,60 +108,60 @@ class Link: @property def link_dir(self) -> str: - from config import ARCHIVE_DIR + from .config import ARCHIVE_DIR return os.path.join(ARCHIVE_DIR, self.timestamp) @property def archive_path(self) -> str: - from config import ARCHIVE_DIR_NAME + from .config import ARCHIVE_DIR_NAME return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp) ### URL Helpers @property def urlhash(self): - from util import hashurl + from .util import hashurl return hashurl(self.url) @property def extension(self) -> str: - from util import extension + from .util import extension return extension(self.url) @property def domain(self) -> str: - from util import domain + from .util import domain return domain(self.url) @property def path(self) -> str: - from util import path + from .util import path return path(self.url) @property def basename(self) -> str: - from util import basename + from .util import basename return basename(self.url) @property def base_url(self) -> str: - from util import base_url + from .util import base_url return base_url(self.url) ### Pretty Printing Helpers @property def bookmarked_date(self) -> Optional[str]: - from util import ts_to_date + from .util import ts_to_date return ts_to_date(self.timestamp) if self.timestamp else None @property def updated_date(self) -> Optional[str]: - from util import ts_to_date + from .util import ts_to_date return ts_to_date(self.updated) if self.updated else None @property def oldest_archive_date(self) -> Optional[datetime]: - from util import ts_to_date + from .util import ts_to_date most_recent = min( (ts_to_date(result.start_ts) @@ -173,7 +173,7 @@ class Link: @property def newest_archive_date(self) -> Optional[datetime]: - from util import ts_to_date + from .util import ts_to_date most_recent = max( (ts_to_date(result.start_ts) @@ -197,13 +197,13 @@ class Link: @property def is_static(self) -> bool: - from util import is_static_file + from .util import is_static_file return is_static_file(self.url) @property def is_archived(self) -> bool: - from config import ARCHIVE_DIR - from util import domain + from .config import ARCHIVE_DIR + from .util import domain return os.path.exists(os.path.join( ARCHIVE_DIR, @@ -240,7 +240,7 @@ class Link: return latest def canonical_outputs(self) -> Dict[str, Optional[str]]: - from util import wget_output_path + from .util import wget_output_path canonical = { 'index_url': 'index.html', 'favicon_url': 'favicon.ico', diff --git a/archivebox/templates/index.html b/archivebox/templates/index.html index 8436a412..144f2ce7 100644 --- a/archivebox/templates/index.html +++ b/archivebox/templates/index.html @@ -209,7 +209,7 @@ <center> <small> Archive created using <a href="https://github.com/pirate/ArchiveBox" title="Github">ArchiveBox</a> - version <a href="https://github.com/pirate/ArchiveBox/commit/$git_sha" title="Git commit">$short_git_sha</a>   |   + version <a href="https://github.com/pirate/ArchiveBox/commit/$git_sha" title="Git commit">$version</a>   |   Download index as <a href="index.json" title="JSON summary of archived links.">JSON</a> <br/><br/> $footer_info diff --git a/archivebox/util.py b/archivebox/util.py index e6f93981..fe3c57cf 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -25,8 +25,8 @@ from subprocess import ( from base32_crockford import encode as base32_encode -from schema import Link -from config import ( +from .schema import Link +from .config import ( ANSI, TERM_WIDTH, SOURCES_DIR, @@ -37,9 +37,9 @@ from config import ( CHECK_SSL_VALIDITY, WGET_USER_AGENT, CHROME_OPTIONS, - PYTHON_PATH, + PYTHON_DIR, ) -from logs import pretty_path +from .logs import pretty_path ### Parsing Helpers @@ -334,7 +334,7 @@ def wget_output_path(link: Link) -> Optional[str]: @enforce_types def read_js_script(script_name: str) -> str: - script_path = os.path.join(PYTHON_PATH, 'scripts', script_name) + script_path = os.path.join(PYTHON_DIR, 'scripts', script_name) with open(script_path, 'r') as f: return f.read().split('// INFO BELOW HERE')[0].strip() From 64f92af60b950f5e32f85279079d14db6d9e1504 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:39:30 -0400 Subject: [PATCH 030/365] working setuptools package distribution --- .gitignore | 6 ++++++ MANIFEST.in | 3 +++ VERSION | 1 + archive | 1 - archivebox/requirements.txt | 0 archivebox/scripts/readability.js | 27 ------------------------ requirements.txt | 16 ++++++++++++++ setup.py | 35 ++++++++++++++++++++++++++++++- 8 files changed, 60 insertions(+), 29 deletions(-) create mode 100644 MANIFEST.in create mode 100644 VERSION delete mode 120000 archive delete mode 100644 archivebox/requirements.txt delete mode 100644 archivebox/scripts/readability.js create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 1dcc07e1..d44c22ec 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,9 @@ output/ data data/ archivebox/output +archivebox/data +archivebox/VERSION + +archivebox.egg-info/ +build/ +dist/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..e82bd579 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include archivebox/VERSION +graft archivebox/templates +graft archivebox/templates/static diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..0d91a54c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.0 diff --git a/archive b/archive deleted file mode 120000 index 041799a6..00000000 --- a/archive +++ /dev/null @@ -1 +0,0 @@ -bin/archivebox \ No newline at end of file diff --git a/archivebox/requirements.txt b/archivebox/requirements.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/scripts/readability.js b/archivebox/scripts/readability.js deleted file mode 100644 index ce937f54..00000000 --- a/archivebox/scripts/readability.js +++ /dev/null @@ -1,27 +0,0 @@ -() => { - function Readability(e,t){if(t&&t.documentElement)e=t,t=arguments[2];else if(!e||!e.documentElement)throw new Error("First argument to Readability constructor should be a document object.");var i;t=t||{},this._doc=e,this._articleTitle=null,this._articleByline=null,this._articleDir=null,this._articleSiteName=null,this._attempts=[],this._debug=!!t.debug,this._maxElemsToParse=t.maxElemsToParse||this.DEFAULT_MAX_ELEMS_TO_PARSE,this._nbTopCandidates=t.nbTopCandidates||this.DEFAULT_N_TOP_CANDIDATES,this._charThreshold=t.charThreshold||this.DEFAULT_CHAR_THRESHOLD,this._classesToPreserve=this.CLASSES_TO_PRESERVE.concat(t.classesToPreserve||[]),this._flags=this.FLAG_STRIP_UNLIKELYS|this.FLAG_WEIGHT_CLASSES|this.FLAG_CLEAN_CONDITIONALLY,this._debug?(i=function(e){var t=e.nodeName+" ";if(e.nodeType==e.TEXT_NODE)return t+'("'+e.textContent+'")';var i=e.className&&"."+e.className.replace(/ /g,"."),a="";return e.id?a="(#"+e.id+i+")":i&&(a="("+i+")"),t+a},this.log=function(){if("undefined"!=typeof dump){var e=Array.prototype.map.call(arguments,function(e){return e&&e.nodeName?i(e):e}).join(" ");dump("Reader: (Readability) "+e+"\n")}else if("undefined"!=typeof console){var t=["Reader: (Readability) "].concat(arguments);console.log.apply(console,t)}}):this.log=function(){}}Readability.prototype={FLAG_STRIP_UNLIKELYS:1,FLAG_WEIGHT_CLASSES:2,FLAG_CLEAN_CONDITIONALLY:4,ELEMENT_NODE:1,TEXT_NODE:3,DEFAULT_MAX_ELEMS_TO_PARSE:0,DEFAULT_N_TOP_CANDIDATES:5,DEFAULT_TAGS_TO_SCORE:"section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),DEFAULT_CHAR_THRESHOLD:500,REGEXPS:{unlikelyCandidates:/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,okMaybeItsACandidate:/and|article|body|column|main|shadow/i,positive:/article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,negative:/hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i,extraneous:/print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,byline:/byline|author|dateline|writtenby|p-author/i,replaceFonts:/<(\/?)font[^>]*>/gi,normalize:/\s{2,}/g,videos:/\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,nextLink:/(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,prevLink:/(prev|earl|old|new|<|«)/i,whitespace:/^\s*$/,hasContent:/\S$/},DIV_TO_P_ELEMS:["A","BLOCKQUOTE","DL","DIV","IMG","OL","P","PRE","TABLE","UL","SELECT"],ALTER_TO_DIV_EXCEPTIONS:["DIV","ARTICLE","SECTION","P"],PRESENTATIONAL_ATTRIBUTES:["align","background","bgcolor","border","cellpadding","cellspacing","frame","hspace","rules","style","valign","vspace"],DEPRECATED_SIZE_ATTRIBUTE_ELEMS:["TABLE","TH","TD","HR","PRE"],PHRASING_ELEMS:["ABBR","AUDIO","B","BDO","BR","BUTTON","CITE","CODE","DATA","DATALIST","DFN","EM","EMBED","I","IMG","INPUT","KBD","LABEL","MARK","MATH","METER","NOSCRIPT","OBJECT","OUTPUT","PROGRESS","Q","RUBY","SAMP","SCRIPT","SELECT","SMALL","SPAN","STRONG","SUB","SUP","TEXTAREA","TIME","VAR","WBR"],CLASSES_TO_PRESERVE:["page"],_postProcessContent:function(e){this._fixRelativeUris(e),this._cleanClasses(e)},_removeNodes:function(e,t){for(var i=e.length-1;0<=i;i--){var a=e[i],n=a.parentNode;n&&(t&&!t.call(this,a,i,e)||n.removeChild(a))}},_replaceNodeTags:function(e,t){for(var i=e.length-1;0<=i;i--){var a=e[i];this._setNodeTag(a,t)}},_forEachNode:function(e,t){Array.prototype.forEach.call(e,t,this)},_someNode:function(e,t){return Array.prototype.some.call(e,t,this)},_everyNode:function(e,t){return Array.prototype.every.call(e,t,this)},_concatNodeLists:function(){var t=Array.prototype.slice,e=t.call(arguments).map(function(e){return t.call(e)});return Array.prototype.concat.apply([],e)},_getAllNodesWithTag:function(i,e){return i.querySelectorAll?i.querySelectorAll(e.join(",")):[].concat.apply([],e.map(function(e){var t=i.getElementsByTagName(e);return Array.isArray(t)?t:Array.from(t)}))},_cleanClasses:function(e){var t=this._classesToPreserve,i=(e.getAttribute("class")||"").split(/\s+/).filter(function(e){return-1!=t.indexOf(e)}).join(" ");for(i?e.setAttribute("class",i):e.removeAttribute("class"),e=e.firstElementChild;e;e=e.nextElementSibling)this._cleanClasses(e)},_fixRelativeUris:function(e){var t=this._doc.baseURI,i=this._doc.documentURI;function a(e){if(t==i&&"#"==e.charAt(0))return e;try{return new URL(e,t).href}catch(e){}return e}var n=this._getAllNodesWithTag(e,["a"]);this._forEachNode(n,function(e){var t=e.getAttribute("href");if(t)if(0===t.indexOf("javascript:")){var i=this._doc.createTextNode(e.textContent);e.parentNode.replaceChild(i,e)}else e.setAttribute("href",a(t))});var r=this._getAllNodesWithTag(e,["img"]);this._forEachNode(r,function(e){var t=e.getAttribute("src");t&&e.setAttribute("src",a(t))})},_getArticleTitle:function(){var e=this._doc,t="",i="";try{"string"!=typeof(t=i=e.title.trim())&&(t=i=this._getInnerText(e.getElementsByTagName("title")[0]))}catch(e){}var a=!1;function n(e){return e.split(/\s+/).length}if(/ [\|\-\\\/>»] /.test(t))a=/ [\\\/>»] /.test(t),n(t=i.replace(/(.*)[\|\-\\\/>»] .*/gi,"$1"))<3&&(t=i.replace(/[^\|\-\\\/>»]*[\|\-\\\/>»](.*)/gi,"$1"));else if(-1!==t.indexOf(": ")){var r=this._concatNodeLists(e.getElementsByTagName("h1"),e.getElementsByTagName("h2")),s=t.trim();this._someNode(r,function(e){return e.textContent.trim()===s})||(n(t=i.substring(i.lastIndexOf(":")+1))<3?t=i.substring(i.indexOf(":")+1):5<n(i.substr(0,i.indexOf(":")))&&(t=i))}else if(150<t.length||t.length<15){var o=e.getElementsByTagName("h1");1===o.length&&(t=this._getInnerText(o[0]))}var l=n(t=t.trim().replace(this.REGEXPS.normalize," "));return l<=4&&(!a||l!=n(i.replace(/[\|\-\\\/>»]+/g,""))-1)&&(t=i),t},_prepDocument:function(){var e=this._doc;this._removeNodes(e.getElementsByTagName("style")),e.body&&this._replaceBrs(e.body),this._replaceNodeTags(e.getElementsByTagName("font"),"SPAN")},_nextElement:function(e){for(var t=e;t&&t.nodeType!=this.ELEMENT_NODE&&this.REGEXPS.whitespace.test(t.textContent);)t=t.nextSibling;return t},_replaceBrs:function(e){this._forEachNode(this._getAllNodesWithTag(e,["br"]),function(e){for(var t=e.nextSibling,i=!1;(t=this._nextElement(t))&&"BR"==t.tagName;){i=!0;var a=t.nextSibling;t.parentNode.removeChild(t),t=a}if(i){var n=this._doc.createElement("p");for(e.parentNode.replaceChild(n,e),t=n.nextSibling;t;){if("BR"==t.tagName){var r=this._nextElement(t.nextSibling);if(r&&"BR"==r.tagName)break}if(!this._isPhrasingContent(t))break;var s=t.nextSibling;n.appendChild(t),t=s}for(;n.lastChild&&this._isWhitespace(n.lastChild);)n.removeChild(n.lastChild);"P"===n.parentNode.tagName&&this._setNodeTag(n.parentNode,"DIV")}})},_setNodeTag:function(e,t){if(this.log("_setNodeTag",e,t),e.__JSDOMParser__)return e.localName=t.toLowerCase(),e.tagName=t.toUpperCase(),e;for(var i=e.ownerDocument.createElement(t);e.firstChild;)i.appendChild(e.firstChild);e.parentNode.replaceChild(i,e),e.readability&&(i.readability=e.readability);for(var a=0;a<e.attributes.length;a++)try{i.setAttribute(e.attributes[a].name,e.attributes[a].value)}catch(e){}return i},_prepArticle:function(e){this._cleanStyles(e),this._markDataTables(e),this._cleanConditionally(e,"form"),this._cleanConditionally(e,"fieldset"),this._clean(e,"object"),this._clean(e,"embed"),this._clean(e,"h1"),this._clean(e,"footer"),this._clean(e,"link"),this._clean(e,"aside");var i=this.DEFAULT_CHAR_THRESHOLD;this._forEachNode(e.children,function(e){this._cleanMatchedNodes(e,function(e,t){return/share/.test(t)&&e.textContent.length<i})});var t=e.getElementsByTagName("h2");if(1===t.length){var a=(t[0].textContent.length-this._articleTitle.length)/this._articleTitle.length;if(Math.abs(a)<.5){(0<a?t[0].textContent.includes(this._articleTitle):this._articleTitle.includes(t[0].textContent))&&this._clean(e,"h2")}}this._clean(e,"iframe"),this._clean(e,"input"),this._clean(e,"textarea"),this._clean(e,"select"),this._clean(e,"button"),this._cleanHeaders(e),this._cleanConditionally(e,"table"),this._cleanConditionally(e,"ul"),this._cleanConditionally(e,"div"),this._removeNodes(e.getElementsByTagName("p"),function(e){return 0===e.getElementsByTagName("img").length+e.getElementsByTagName("embed").length+e.getElementsByTagName("object").length+e.getElementsByTagName("iframe").length&&!this._getInnerText(e,!1)}),this._forEachNode(this._getAllNodesWithTag(e,["br"]),function(e){var t=this._nextElement(e.nextSibling);t&&"P"==t.tagName&&e.parentNode.removeChild(e)}),this._forEachNode(this._getAllNodesWithTag(e,["table"]),function(e){var t=this._hasSingleTagInsideElement(e,"TBODY")?e.firstElementChild:e;if(this._hasSingleTagInsideElement(t,"TR")){var i=t.firstElementChild;if(this._hasSingleTagInsideElement(i,"TD")){var a=i.firstElementChild;a=this._setNodeTag(a,this._everyNode(a.childNodes,this._isPhrasingContent)?"P":"DIV"),e.parentNode.replaceChild(a,e)}}})},_initializeNode:function(e){switch(e.readability={contentScore:0},e.tagName){case"DIV":e.readability.contentScore+=5;break;case"PRE":case"TD":case"BLOCKQUOTE":e.readability.contentScore+=3;break;case"ADDRESS":case"OL":case"UL":case"DL":case"DD":case"DT":case"LI":case"FORM":e.readability.contentScore-=3;break;case"H1":case"H2":case"H3":case"H4":case"H5":case"H6":case"TH":e.readability.contentScore-=5}e.readability.contentScore+=this._getClassWeight(e)},_removeAndGetNext:function(e){var t=this._getNextNode(e,!0);return e.parentNode.removeChild(e),t},_getNextNode:function(e,t){if(!t&&e.firstElementChild)return e.firstElementChild;if(e.nextElementSibling)return e.nextElementSibling;for(;(e=e.parentNode)&&!e.nextElementSibling;);return e&&e.nextElementSibling},_checkByline:function(e,t){if(this._articleByline)return!1;if(void 0!==e.getAttribute)var i=e.getAttribute("rel"),a=e.getAttribute("itemprop");return!(!("author"===i||a&&-1!==a.indexOf("author")||this.REGEXPS.byline.test(t))||!this._isValidByline(e.textContent))&&(this._articleByline=e.textContent.trim(),!0)},_getNodeAncestors:function(e,t){t=t||0;for(var i=0,a=[];e.parentNode&&(a.push(e.parentNode),!t||++i!==t);)e=e.parentNode;return a},_grabArticle:function(e){this.log("**** grabArticle ****");var t=this._doc,i=null!==e;if(!(e=e||this._doc.body))return this.log("No body found in document. Abort."),null;for(var a=e.innerHTML;;){for(var n=this._flagIsActive(this.FLAG_STRIP_UNLIKELYS),r=[],s=this._doc.documentElement;s;){var o=s.className+" "+s.id;if(this._isProbablyVisible(s))if(this._checkByline(s,o))s=this._removeAndGetNext(s);else if(!n||!this.REGEXPS.unlikelyCandidates.test(o)||this.REGEXPS.okMaybeItsACandidate.test(o)||this._hasAncestorTag(s,"table")||"BODY"===s.tagName||"A"===s.tagName)if("DIV"!==s.tagName&&"SECTION"!==s.tagName&&"HEADER"!==s.tagName&&"H1"!==s.tagName&&"H2"!==s.tagName&&"H3"!==s.tagName&&"H4"!==s.tagName&&"H5"!==s.tagName&&"H6"!==s.tagName||!this._isElementWithoutContent(s)){if(-1!==this.DEFAULT_TAGS_TO_SCORE.indexOf(s.tagName)&&r.push(s),"DIV"===s.tagName){for(var l=null,h=s.firstChild;h;){var c=h.nextSibling;if(this._isPhrasingContent(h))null!==l?l.appendChild(h):this._isWhitespace(h)||(l=t.createElement("p"),s.replaceChild(l,h),l.appendChild(h));else if(null!==l){for(;l.lastChild&&this._isWhitespace(l.lastChild);)l.removeChild(l.lastChild);l=null}h=c}if(this._hasSingleTagInsideElement(s,"P")&&this._getLinkDensity(s)<.25){var d=s.children[0];s.parentNode.replaceChild(d,s),s=d,r.push(s)}else this._hasChildBlockElement(s)||(s=this._setNodeTag(s,"P"),r.push(s))}s=this._getNextNode(s)}else s=this._removeAndGetNext(s);else this.log("Removing unlikely candidate - "+o),s=this._removeAndGetNext(s);else this.log("Removing hidden node - "+o),s=this._removeAndGetNext(s)}var g=[];this._forEachNode(r,function(e){if(e.parentNode&&void 0!==e.parentNode.tagName){var t=this._getInnerText(e);if(!(t.length<25)){var i=this._getNodeAncestors(e,3);if(0!==i.length){var a=0;a+=1,a+=t.split(",").length,a+=Math.min(Math.floor(t.length/100),3),this._forEachNode(i,function(e,t){if(e.tagName&&e.parentNode&&void 0!==e.parentNode.tagName){if(void 0===e.readability&&(this._initializeNode(e),g.push(e)),0===t)var i=1;else i=1===t?2:3*t;e.readability.contentScore+=a/i}})}}}});for(var _=[],m=0,u=g.length;m<u;m+=1){var f=g[m],N=f.readability.contentScore*(1-this._getLinkDensity(f));f.readability.contentScore=N,this.log("Candidate:",f,"with score "+N);for(var E=0;E<this._nbTopCandidates;E++){var p=_[E];if(!p||N>p.readability.contentScore){_.splice(E,0,f),_.length>this._nbTopCandidates&&_.pop();break}}}var T,b=_[0]||null,y=!1;if(null===b||"BODY"===b.tagName){b=t.createElement("DIV"),y=!0;for(var v=e.childNodes;v.length;)this.log("Moving child out:",v[0]),b.appendChild(v[0]);e.appendChild(b),this._initializeNode(b)}else if(b){for(var A=[],C=1;C<_.length;C++).75<=_[C].readability.contentScore/b.readability.contentScore&&A.push(this._getNodeAncestors(_[C]));if(3<=A.length)for(T=b.parentNode;"BODY"!==T.tagName;){for(var S=0,L=0;L<A.length&&S<3;L++)S+=Number(A[L].includes(T));if(3<=S){b=T;break}T=T.parentNode}b.readability||this._initializeNode(b),T=b.parentNode;for(var x=b.readability.contentScore,I=x/3;"BODY"!==T.tagName;)if(T.readability){var D=T.readability.contentScore;if(D<I)break;if(x<D){b=T;break}x=T.readability.contentScore,T=T.parentNode}else T=T.parentNode;for(T=b.parentNode;"BODY"!=T.tagName&&1==T.children.length;)T=(b=T).parentNode;b.readability||this._initializeNode(b)}var R=t.createElement("DIV");i&&(R.id="readability-content");for(var B=Math.max(10,.2*b.readability.contentScore),O=(T=b.parentNode).children,P=0,G=O.length;P<G;P++){var M=O[P],w=!1;if(this.log("Looking at sibling node:",M,M.readability?"with score "+M.readability.contentScore:""),this.log("Sibling has score",M.readability?M.readability.contentScore:"Unknown"),M===b)w=!0;else{var H=0;if(M.className===b.className&&""!==b.className&&(H+=.2*b.readability.contentScore),M.readability&&M.readability.contentScore+H>=B)w=!0;else if("P"===M.nodeName){var U=this._getLinkDensity(M),k=this._getInnerText(M),F=k.length;80<F&&U<.25?w=!0:F<80&&0<F&&0===U&&-1!==k.search(/\.( |$)/)&&(w=!0)}}w&&(this.log("Appending node:",M),-1===this.ALTER_TO_DIV_EXCEPTIONS.indexOf(M.nodeName)&&(this.log("Altering sibling:",M,"to div."),M=this._setNodeTag(M,"DIV")),R.appendChild(M),P-=1,G-=1)}if(this._debug&&this.log("Article content pre-prep: "+R.innerHTML),this._prepArticle(R),this._debug&&this.log("Article content post-prep: "+R.innerHTML),y)b.id="readability-page-1",b.className="page";else{var X=t.createElement("DIV");X.id="readability-page-1",X.className="page";for(var V=R.childNodes;V.length;)X.appendChild(V[0]);R.appendChild(X)}this._debug&&this.log("Article content after paging: "+R.innerHTML);var W=!0,Y=this._getInnerText(R,!0).length;if(Y<this._charThreshold)if(W=!1,e.innerHTML=a,this._flagIsActive(this.FLAG_STRIP_UNLIKELYS))this._removeFlag(this.FLAG_STRIP_UNLIKELYS),this._attempts.push({articleContent:R,textLength:Y});else if(this._flagIsActive(this.FLAG_WEIGHT_CLASSES))this._removeFlag(this.FLAG_WEIGHT_CLASSES),this._attempts.push({articleContent:R,textLength:Y});else if(this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY))this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY),this._attempts.push({articleContent:R,textLength:Y});else{if(this._attempts.push({articleContent:R,textLength:Y}),this._attempts.sort(function(e,t){return t.textLength-e.textLength}),!this._attempts[0].textLength)return null;R=this._attempts[0].articleContent,W=!0}if(W){var j=[T,b].concat(this._getNodeAncestors(T));return this._someNode(j,function(e){if(!e.tagName)return!1;var t=e.getAttribute("dir");return!!t&&(this._articleDir=t,!0)}),R}}},_isValidByline:function(e){return("string"==typeof e||e instanceof String)&&(0<(e=e.trim()).length&&e.length<100)},_getArticleMetadata:function(){var e={},o={},t=this._doc.getElementsByTagName("meta"),l=/\s*(dc|dcterm|og|twitter)\s*:\s*(author|creator|description|title|site_name)\s*/gi,h=/^\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\s*[\.:]\s*)?(author|creator|description|title|site_name)\s*$/i;return this._forEachNode(t,function(e){var t=e.getAttribute("name"),i=e.getAttribute("property"),a=e.getAttribute("content");if(a){var n=null,r=null;if(i&&(n=i.match(l)))for(var s=n.length-1;0<=s;s--)r=n[s].toLowerCase().replace(/\s/g,""),o[r]=a.trim();!n&&t&&h.test(t)&&(r=t,a&&(r=r.toLowerCase().replace(/\s/g,"").replace(/\./g,":"),o[r]=a.trim()))}}),e.title=o["dc:title"]||o["dcterm:title"]||o["og:title"]||o["weibo:article:title"]||o["weibo:webpage:title"]||o.title||o["twitter:title"],e.title||(e.title=this._getArticleTitle()),e.byline=o["dc:creator"]||o["dcterm:creator"]||o.author,e.excerpt=o["dc:description"]||o["dcterm:description"]||o["og:description"]||o["weibo:article:description"]||o["weibo:webpage:description"]||o.description||o["twitter:description"],e.siteName=o["og:site_name"],e},_removeScripts:function(e){this._removeNodes(e.getElementsByTagName("script"),function(e){return e.nodeValue="",e.removeAttribute("src"),!0}),this._removeNodes(e.getElementsByTagName("noscript"))},_hasSingleTagInsideElement:function(e,t){return 1==e.children.length&&e.children[0].tagName===t&&!this._someNode(e.childNodes,function(e){return e.nodeType===this.TEXT_NODE&&this.REGEXPS.hasContent.test(e.textContent)})},_isElementWithoutContent:function(e){return e.nodeType===this.ELEMENT_NODE&&0==e.textContent.trim().length&&(0==e.children.length||e.children.length==e.getElementsByTagName("br").length+e.getElementsByTagName("hr").length)},_hasChildBlockElement:function(e){return this._someNode(e.childNodes,function(e){return-1!==this.DIV_TO_P_ELEMS.indexOf(e.tagName)||this._hasChildBlockElement(e)})},_isPhrasingContent:function(e){return e.nodeType===this.TEXT_NODE||-1!==this.PHRASING_ELEMS.indexOf(e.tagName)||("A"===e.tagName||"DEL"===e.tagName||"INS"===e.tagName)&&this._everyNode(e.childNodes,this._isPhrasingContent)},_isWhitespace:function(e){return e.nodeType===this.TEXT_NODE&&0===e.textContent.trim().length||e.nodeType===this.ELEMENT_NODE&&"BR"===e.tagName},_getInnerText:function(e,t){t=void 0===t||t;var i=e.textContent.trim();return t?i.replace(this.REGEXPS.normalize," "):i},_getCharCount:function(e,t){return t=t||",",this._getInnerText(e).split(t).length-1},_cleanStyles:function(e){if(e&&"svg"!==e.tagName.toLowerCase()){for(var t=0;t<this.PRESENTATIONAL_ATTRIBUTES.length;t++)e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[t]);-1!==this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.indexOf(e.tagName)&&(e.removeAttribute("width"),e.removeAttribute("height"));for(var i=e.firstElementChild;null!==i;)this._cleanStyles(i),i=i.nextElementSibling}},_getLinkDensity:function(e){var t=this._getInnerText(e).length;if(0===t)return 0;var i=0;return this._forEachNode(e.getElementsByTagName("a"),function(e){i+=this._getInnerText(e).length}),i/t},_getClassWeight:function(e){if(!this._flagIsActive(this.FLAG_WEIGHT_CLASSES))return 0;var t=0;return"string"==typeof e.className&&""!==e.className&&(this.REGEXPS.negative.test(e.className)&&(t-=25),this.REGEXPS.positive.test(e.className)&&(t+=25)),"string"==typeof e.id&&""!==e.id&&(this.REGEXPS.negative.test(e.id)&&(t-=25),this.REGEXPS.positive.test(e.id)&&(t+=25)),t},_clean:function(e,t){var i=-1!==["object","embed","iframe"].indexOf(t);this._removeNodes(e.getElementsByTagName(t),function(e){if(i){for(var t=0;t<e.attributes.length;t++)if(this.REGEXPS.videos.test(e.attributes[t].value))return!1;if("object"===e.tagName&&this.REGEXPS.videos.test(e.innerHTML))return!1}return!0})},_hasAncestorTag:function(e,t,i,a){i=i||3,t=t.toUpperCase();for(var n=0;e.parentNode;){if(0<i&&i<n)return!1;if(e.parentNode.tagName===t&&(!a||a(e.parentNode)))return!0;e=e.parentNode,n++}return!1},_getRowAndColumnCount:function(e){for(var t=0,i=0,a=e.getElementsByTagName("tr"),n=0;n<a.length;n++){var r=a[n].getAttribute("rowspan")||0;r&&(r=parseInt(r,10)),t+=r||1;for(var s=0,o=a[n].getElementsByTagName("td"),l=0;l<o.length;l++){var h=o[l].getAttribute("colspan")||0;h&&(h=parseInt(h,10)),s+=h||1}i=Math.max(i,s)}return{rows:t,columns:i}},_markDataTables:function(e){for(var t=e.getElementsByTagName("table"),i=0;i<t.length;i++){var a=t[i];if("presentation"!=a.getAttribute("role"))if("0"!=a.getAttribute("datatable"))if(a.getAttribute("summary"))a._readabilityDataTable=!0;else{var n=a.getElementsByTagName("caption")[0];if(n&&0<n.childNodes.length)a._readabilityDataTable=!0;else{if(["col","colgroup","tfoot","thead","th"].some(function(e){return!!a.getElementsByTagName(e)[0]}))this.log("Data table because found data-y descendant"),a._readabilityDataTable=!0;else if(a.getElementsByTagName("table")[0])a._readabilityDataTable=!1;else{var r=this._getRowAndColumnCount(a);10<=r.rows||4<r.columns?a._readabilityDataTable=!0:a._readabilityDataTable=10<r.rows*r.columns}}}else a._readabilityDataTable=!1;else a._readabilityDataTable=!1}},_cleanConditionally:function(e,_){if(this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)){var m="ul"===_||"ol"===_;this._removeNodes(e.getElementsByTagName(_),function(e){var t=function(e){return e._readabilityDataTable};if("table"===_&&t(e))return!1;if(this._hasAncestorTag(e,"table",-1,t))return!1;var i=this._getClassWeight(e);if(this.log("Cleaning Conditionally",e),i+0<0)return!0;if(this._getCharCount(e,",")<10){for(var a=e.getElementsByTagName("p").length,n=e.getElementsByTagName("img").length,r=e.getElementsByTagName("li").length-100,s=e.getElementsByTagName("input").length,o=0,l=this._concatNodeLists(e.getElementsByTagName("object"),e.getElementsByTagName("embed"),e.getElementsByTagName("iframe")),h=0;h<l.length;h++){for(var c=0;c<l[h].attributes.length;c++)if(this.REGEXPS.videos.test(l[h].attributes[c].value))return!1;if("object"===l[h].tagName&&this.REGEXPS.videos.test(l[h].innerHTML))return!1;o++}var d=this._getLinkDensity(e),g=this._getInnerText(e).length;return 1<n&&a/n<.5&&!this._hasAncestorTag(e,"figure")||!m&&a<r||s>Math.floor(a/3)||!m&&g<25&&(0===n||2<n)&&!this._hasAncestorTag(e,"figure")||!m&&i<25&&.2<d||25<=i&&.5<d||1===o&&g<75||1<o}return!1})}},_cleanMatchedNodes:function(e,t){for(var i=this._getNextNode(e,!0),a=this._getNextNode(e);a&&a!=i;)a=t(a,a.className+" "+a.id)?this._removeAndGetNext(a):this._getNextNode(a)},_cleanHeaders:function(e){for(var t=1;t<3;t+=1)this._removeNodes(e.getElementsByTagName("h"+t),function(e){return this._getClassWeight(e)<0})},_flagIsActive:function(e){return 0<(this._flags&e)},_removeFlag:function(e){this._flags=this._flags&~e},_isProbablyVisible:function(e){return!(e.style&&"none"==e.style.display||e.hasAttribute("hidden"))},parse:function(){if(0<this._maxElemsToParse){var e=this._doc.getElementsByTagName("*").length;if(e>this._maxElemsToParse)throw new Error("Aborting parsing document; "+e+" elements found")}this._removeScripts(this._doc),this._prepDocument();var t=this._getArticleMetadata();this._articleTitle=t.title;var i=this._grabArticle();if(!i)return null;if(this.log("Grabbed: "+i.innerHTML),this._postProcessContent(i),!t.excerpt){var a=i.getElementsByTagName("p");0<a.length&&(t.excerpt=a[0].textContent.trim())}var n=i.textContent;return{title:this._articleTitle,byline:t.byline||this._articleByline,dir:this._articleDir,content:i.innerHTML,textContent:n,length:n.length,excerpt:t.excerpt,siteName:t.siteName||this._articleSiteName}}},"object"==typeof module&&(module.exports=Readability); - return JSON.stringify(new Readability(document).parse()); -} - -/************** Readability.js Licensce ****************/ -/***** From: https://github.com/mozilla/readability ****/ -/* - * Copyright (c) 2010 Arc90 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This code is heavily based on Arc90's readability.js (1.7.1) script - * available at: http://code.google.com/p/arc90labs-readability - */ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..15de6de4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +base32-crockford + +# pip +# setuptools +# ipdb +# mypy +# flake8 + +# wpull +# pywb +# pyppeteer +# GitPython +# youtube-dl +# archivenow +# requests +# diff --git a/setup.py b/setup.py index 64c9f104..d3ce3963 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,51 @@ +import os import setuptools with open("README.md", "r") as fh: long_description = fh.read() + +script_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) + +VERSION = open(os.path.join(script_dir, 'VERSION'), 'r').read().strip() +GIT_HEAD = open(os.path.join(script_dir, '.git', 'HEAD'), 'r').read().strip().split(': ')[1] +GIT_SHA = open(os.path.join(script_dir, '.git', GIT_HEAD), 'r').read().strip()[:9] +PYPI_VERSION = "{}+{}".format(VERSION, GIT_SHA) + +with open(os.path.join(script_dir, 'archivebox', 'VERSION'), 'w+') as f: + f.write(PYPI_VERSION) + setuptools.setup( name="archivebox", - version="0.3.0", + version=PYPI_VERSION, author="Nick Sweeting", author_email="git@nicksweeting.com", description="The self-hosted internet archive.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/pirate/ArchiveBox", + project_urls={ + 'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki', + 'Community': 'https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community', + 'Source': 'https://github.com/pirate/ArchiveBox', + 'Bug Tracker': 'https://github.com/pirate/ArchiveBox/issues', + 'Roadmap': 'https://github.com/pirate/ArchiveBox/wiki/Roadmap', + 'Changelog': 'https://github.com/pirate/ArchiveBox/wiki/Changelog', + 'Donations': 'https://github.com/pirate/ArchiveBox/wiki/Donations', + }, packages=setuptools.find_packages(), + python_requires='>=3.6', + install_requires=[ + "base32-crockford==0.3.0", + ], + entry_points={ + 'console_scripts': [ + 'archivebox = archivebox.archive:main', + ], + }, + package_data={ + 'archivebox': ['VERSION', 'templates/*', 'templates/static/*'], + }, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", From 3375522ff46e3fe0231b611a62cfbda16b1ceb58 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 15:49:41 -0400 Subject: [PATCH 031/365] remove uneeded bin shortcuts --- archivebox/logs.py | 6 +----- bin/bookmark-archiver | 1 - requirements.txt | 1 - setup | 1 - 4 files changed, 1 insertion(+), 8 deletions(-) delete mode 120000 bin/bookmark-archiver delete mode 120000 setup diff --git a/archivebox/logs.py b/archivebox/logs.py index a4b83cfd..4e21731b 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -51,11 +51,7 @@ def log_parsing_started(source_file: str): def log_parsing_finished(num_parsed: int, num_new_links: int, parser_name: str): end_ts = datetime.now() _LAST_RUN_STATS.parse_end_ts = end_ts - print(' > Parsed {} links as {}'.format(num_parsed, parser_name)) - print(' > Adding {} new links to collection: {}'.format( - num_new_links, - OUTPUT_DIR, - )) + print(' > Parsed {} links as {} ({} new links added)'.format(num_parsed, parser_name, num_new_links)) ### Indexing Stage diff --git a/bin/bookmark-archiver b/bin/bookmark-archiver deleted file mode 120000 index a1b02220..00000000 --- a/bin/bookmark-archiver +++ /dev/null @@ -1 +0,0 @@ -archivebox \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 15de6de4..835549a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,3 @@ base32-crockford # youtube-dl # archivenow # requests -# diff --git a/setup b/setup deleted file mode 120000 index 699bbdab..00000000 --- a/setup +++ /dev/null @@ -1 +0,0 @@ -bin/archivebox-setup \ No newline at end of file From a26c2fe4678d85201fad9b98708acd61a1ee7edc Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 16:44:00 -0400 Subject: [PATCH 032/365] show full version info using flag --- archivebox/archive.py | 64 +++++++++++++++++++-- archivebox/archive_methods.py | 12 ++-- archivebox/config.py | 102 +++++++++++++++------------------- archivebox/links.py | 30 +--------- 4 files changed, 114 insertions(+), 94 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index b381d141..f5b85103 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -12,7 +12,7 @@ __package__ = 'archivebox' import os import sys - +import shutil from typing import List, Optional @@ -23,8 +23,23 @@ from .archive_methods import archive_link from .config import ( ONLY_NEW, OUTPUT_DIR, - PYTHON_DIR, VERSION, + ANSI, + CURL_VERSION, + GIT_VERSION, + WGET_VERSION, + YOUTUBEDL_VERSION, + CHROME_VERSION, + USE_CURL, + USE_WGET, + USE_CHROME, + CURL_BINARY, + GIT_BINARY, + WGET_BINARY, + YOUTUBEDL_BINARY, + CHROME_BINARY, + FETCH_GIT, + FETCH_MEDIA, ) from .util import ( enforce_types, @@ -59,8 +74,37 @@ def print_help(): print(" archivebox add --depth=1 https://example.com/feed.rss") print(" archivebox update --resume=15109948213.123") +def print_version(): + print('ArchiveBox v{}'.format(__VERSION__)) + print() + print( + '[{}] CURL:'.format('√' if USE_CURL else 'X').ljust(14), + '{} --version\n'.format(shutil.which(CURL_BINARY)), + ' '*13, CURL_VERSION, '\n', + ) + print( + '[{}] GIT:'.format('√' if FETCH_GIT else 'X').ljust(14), + '{} --version\n'.format(shutil.which(GIT_BINARY)), + ' '*13, GIT_VERSION, '\n', + ) + print( + '[{}] WGET:'.format('√' if USE_WGET else 'X').ljust(14), + '{} --version\n'.format(shutil.which(WGET_BINARY)), + ' '*13, WGET_VERSION, '\n', + ) + print( + '[{}] YOUTUBEDL:'.format('√' if FETCH_MEDIA else 'X').ljust(14), + '{} --version\n'.format(shutil.which(YOUTUBEDL_BINARY)), + ' '*13, YOUTUBEDL_VERSION, '\n', + ) + print( + '[{}] CHROME:'.format('√' if USE_CHROME else 'X').ljust(14), + '{} --version\n'.format(shutil.which(CHROME_BINARY)), + ' '*13, CHROME_VERSION, '\n', + ) -def main(args=None) -> List[Link]: + +def main(args=None) -> None: if args is None: args = sys.argv @@ -69,7 +113,7 @@ def main(args=None) -> List[Link]: raise SystemExit(0) if set(args).intersection(('--version', 'version')): - print('ArchiveBox version {}'.format(__VERSION__)) + print_version() raise SystemExit(0) ### Handle CLI arguments @@ -86,7 +130,19 @@ def main(args=None) -> List[Link]: ### Set up output folder if not os.path.exists(OUTPUT_DIR): + print('{green}[+] Created a new archive directory: {}{reset}'.format(OUTPUT_DIR, **ANSI)) os.makedirs(OUTPUT_DIR) + else: + not_empty = len(set(os.listdir(OUTPUT_DIR)) - {'.DS_Store'}) + index_exists = os.path.exists(os.path.join(OUTPUT_DIR, 'index.json')) + if not_empty and not index_exists: + print( + ('{red}[X] Could not find index.json in the OUTPUT_DIR: {reset}{}\n' + ' You must run ArchiveBox in an existing archive directory, \n' + ' or an empty/new directory to start a new archive collection.' + ).format(OUTPUT_DIR, **ANSI) + ) + raise SystemExit(1) ### Handle ingesting urls piped in through stdin # (.e.g if user does cat example_urls.txt | ./archive) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 2370c98b..16ee3392 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -4,7 +4,7 @@ from typing import Dict, List, Tuple from collections import defaultdict from datetime import datetime -from .schema import Link, ArchiveResult, ArchiveError +from .schema import Link, ArchiveResult from .index import ( write_link_index, patch_links_index, @@ -28,8 +28,6 @@ from .config import ( SUBMIT_ARCHIVE_DOT_ORG, TIMEOUT, MEDIA_TIMEOUT, - ANSI, - OUTPUT_DIR, GIT_DOMAINS, VERSION, WGET_USER_AGENT, @@ -40,7 +38,6 @@ from .config import ( CHROME_VERSION, GIT_VERSION, YOUTUBEDL_VERSION, - ONLY_NEW, WGET_AUTO_COMPRESSION, ) from .util import ( @@ -56,7 +53,6 @@ from .util import ( wget_output_path, chrome_args, run, PIPE, DEVNULL, - Link, ) from .logs import ( log_link_archiving_started, @@ -66,6 +62,12 @@ from .logs import ( ) +class ArchiveError(Exception): + def __init__(self, message, hints=None): + super().__init__(message) + self.hints = hints + + @enforce_types def archive_link(link: Link, page=None) -> Link: """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" diff --git a/archivebox/config.py b/archivebox/config.py index 4573224f..cc032e83 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -59,8 +59,24 @@ CHROME_BINARY = os.getenv('CHROME_BINARY', None) CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' # ****************************************************************************** -# *************************** Directory Settings ******************************* -# ****************************************************************************** + +### Terminal Configuration +TERM_WIDTH = lambda: shutil.get_terminal_size((100, 10)).columns +ANSI = { + 'reset': '\033[00;00m', + 'lightblue': '\033[01;30m', + 'lightyellow': '\033[01;33m', + 'lightred': '\033[01;35m', + 'red': '\033[01;31m', + 'green': '\033[01;32m', + 'blue': '\033[01;34m', + 'white': '\033[01;37m', + 'black': '\033[01;30m', +} +if not USE_COLOR: + # dont show colors if USE_COLOR is False + ANSI = {k: '' for k in ANSI.keys()} + REPO_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) if OUTPUT_DIR: @@ -68,21 +84,6 @@ if OUTPUT_DIR: else: OUTPUT_DIR = os.path.abspath(os.curdir) -if not os.path.exists(OUTPUT_DIR): - print('{green}[+] Created a new archive directory: {}{reset}'.format(OUTPUT_DIR, **ANSI)) - os.makedirs(OUTPUT_DIR) -else: - not_empty = len(set(os.listdir(OUTPUT_DIR)) - {'.DS_Store'}) - index_exists = os.path.exists(os.path.join(OUTPUT_DIR, 'index.json')) - if not_empty and not index_exists: - print( - ('{red}[X] Could not find index.json in the OUTPUT_DIR: {reset}{}\n' - ' You must run ArchiveBox in an existing archive directory, \n' - ' or an empty/new directory to start a new archive collection.' - ).format(OUTPUT_DIR, **ANSI) - ) - raise SystemExit(1) - ARCHIVE_DIR_NAME = 'archive' SOURCES_DIR_NAME = 'sources' ARCHIVE_DIR = os.path.join(OUTPUT_DIR, ARCHIVE_DIR_NAME) @@ -94,13 +95,34 @@ TEMPLATES_DIR = os.path.join(PYTHON_DIR, 'templates') if COOKIES_FILE: COOKIES_FILE = os.path.abspath(COOKIES_FILE) + +VERSION = open(os.path.join(PYTHON_DIR, 'VERSION'), 'r').read().strip() +GIT_SHA = VERSION.split('+')[1] + +### Check Python environment +python_vers = float('{}.{}'.format(sys.version_info.major, sys.version_info.minor)) +if python_vers < 3.5: + print('{}[X] Python version is not new enough: {} (>3.5 is required){}'.format(ANSI['red'], python_vers, ANSI['reset'])) + print(' See https://github.com/pirate/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.') + raise SystemExit(1) + +if sys.stdout.encoding.upper() not in ('UTF-8', 'UTF8'): + print('[X] Your system is running python3 scripts with a bad locale setting: {} (it should be UTF-8).'.format(sys.stdout.encoding)) + print(' To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)') + print('') + print(' Confirm that it\'s fixed by opening a new shell and running:') + print(' python3 -c "import sys; print(sys.stdout.encoding)" # should output UTF-8') + print('') + print(' Alternatively, run this script with:') + print(' env PYTHONIOENCODING=UTF-8 ./archive.py export.html') + # ****************************************************************************** # ***************************** Helper Functions ******************************* # ****************************************************************************** def check_version(binary: str) -> str: """check the presence and return valid version line of a specified binary""" - if run(['which', binary], stdout=DEVNULL, stderr=DEVNULL).returncode: + if not shutil.which(binary): print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI)) print(' Install it, then confirm it works with: {} --version'.format(binary)) print(' See https://github.com/pirate/ArchiveBox/wiki/Install for help.') @@ -168,43 +190,6 @@ def find_chrome_data_dir() -> Optional[str]: # ****************************************************************************** try: - VERSION = open(os.path.join(PYTHON_DIR, 'VERSION'), 'r').read().strip() - GIT_SHA = VERSION.split('+')[1] - - ### Terminal Configuration - TERM_WIDTH = lambda: shutil.get_terminal_size((100, 10)).columns - ANSI = { - 'reset': '\033[00;00m', - 'lightblue': '\033[01;30m', - 'lightyellow': '\033[01;33m', - 'lightred': '\033[01;35m', - 'red': '\033[01;31m', - 'green': '\033[01;32m', - 'blue': '\033[01;34m', - 'white': '\033[01;37m', - 'black': '\033[01;30m', - } - if not USE_COLOR: - # dont show colors if USE_COLOR is False - ANSI = {k: '' for k in ANSI.keys()} - - ### Check Python environment - python_vers = float('{}.{}'.format(sys.version_info.major, sys.version_info.minor)) - if python_vers < 3.5: - print('{}[X] Python version is not new enough: {} (>3.5 is required){}'.format(ANSI['red'], python_vers, ANSI['reset'])) - print(' See https://github.com/pirate/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.') - raise SystemExit(1) - - if sys.stdout.encoding.upper() not in ('UTF-8', 'UTF8'): - print('[X] Your system is running python3 scripts with a bad locale setting: {} (it should be UTF-8).'.format(sys.stdout.encoding)) - print(' To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)') - print('') - print(' Confirm that it\'s fixed by opening a new shell and running:') - print(' python3 -c "import sys; print(sys.stdout.encoding)" # should output UTF-8') - print('') - print(' Alternatively, run this script with:') - print(' env PYTHONIOENCODING=UTF-8 ./archive.py export.html') - ### Make sure curl is installed if USE_CURL: USE_CURL = FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG @@ -238,17 +223,18 @@ try: ### Make sure youtube-dl is installed YOUTUBEDL_VERSION = None if FETCH_MEDIA: - check_version(YOUTUBEDL_BINARY) + YOUTUBEDL_VERSION = check_version(YOUTUBEDL_BINARY) ### Make sure chrome is installed and calculate version if USE_CHROME: USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM else: FETCH_PDF = FETCH_SCREENSHOT = FETCH_DOM = False + + if CHROME_BINARY is None: + CHROME_BINARY = find_chrome_binary() or 'chromium-browser' CHROME_VERSION = None if USE_CHROME: - if CHROME_BINARY is None: - CHROME_BINARY = find_chrome_binary() if CHROME_BINARY: CHROME_VERSION = check_version(CHROME_BINARY) # print('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY)) diff --git a/archivebox/links.py b/archivebox/links.py index 0d72472d..ffb4d415 100644 --- a/archivebox/links.py +++ b/archivebox/links.py @@ -1,24 +1,3 @@ -""" -In ArchiveBox, a Link represents a single entry that we track in the -json index. All links pass through all archiver functions and the latest, -most up-to-date canonical output for each is stored in "latest". - -Link { - timestamp: str, (how we uniquely id links) - url: str, - title: str, - tags: str, - sources: [str], - history: { - pdf: [ - {start_ts, end_ts, cmd, pwd, cmd_version, status, output}, - ... - ], - ... - }, -} -""" - from typing import Iterable from collections import OrderedDict @@ -27,8 +6,6 @@ from .util import ( scheme, fuzzy_url, merge_links, - htmldecode, - hashurl, ) @@ -68,10 +45,9 @@ def uniquefied_links(sorted_links: Iterable[Link]) -> Iterable[Link]: unique_timestamps: OrderedDict[str, Link] = OrderedDict() for link in unique_urls.values(): - new_link = Link(**{ - **link._asdict(), - 'timestamp': lowest_uniq_timestamp(unique_timestamps, link.timestamp), - }) + new_link = link.overwrite( + timestamp=lowest_uniq_timestamp(unique_timestamps, link.timestamp), + ) unique_timestamps[new_link.timestamp] = new_link return unique_timestamps.values() From 03047e428e4b23cfe367402737b627c5b99e8730 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 16:44:13 -0400 Subject: [PATCH 033/365] add initial dev requirements --- requirements.txt | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 835549a6..6c12aee4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,14 @@ base32-crockford -# pip -# setuptools -# ipdb -# mypy -# flake8 +setuptools +ipdb +mypy +flake8 -# wpull -# pywb -# pyppeteer -# GitPython -# youtube-dl -# archivenow -# requests +#wpull +#pywb +#pyppeteer +#GitPython +#youtube-dl +#archivenow +#requests From 9fc1e3c3e10759316921cb4fdcd4e3a7ca4a990c Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 18:21:53 -0400 Subject: [PATCH 034/365] provide migration hint when initializing --- archivebox/archive.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index f5b85103..8779f5cf 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -137,9 +137,13 @@ def main(args=None) -> None: index_exists = os.path.exists(os.path.join(OUTPUT_DIR, 'index.json')) if not_empty and not index_exists: print( - ('{red}[X] Could not find index.json in the OUTPUT_DIR: {reset}{}\n' - ' You must run ArchiveBox in an existing archive directory, \n' - ' or an empty/new directory to start a new archive collection.' + ("{red}[X] Could not find index.json in the OUTPUT_DIR: {reset}{}\n\n" + " If you're trying to update an existing archive, you must set OUTPUT_DIR to or run archivebox from inside the archive folder you're trying to update.\n" + " If you're trying to create a new archive, you must run archivebox inside a completely empty directory." + "\n\n" + " {lightred}Hint:{reset} To import a data folder created by an older version of ArchiveBox, \n" + " just cd into the folder and run the archivebox comamnd to pick up where you left off.\n\n" + " (Always make sure your data folder is backed up first before updating ArchiveBox)" ).format(OUTPUT_DIR, **ANSI) ) raise SystemExit(1) From a214bd7c021216b71df8d49956d793d86be93d4e Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 18:24:30 -0400 Subject: [PATCH 035/365] make everything take link_dir as an optional arg since its derivable from link url --- archivebox/archive.py | 6 ++-- archivebox/archive_methods.py | 55 ++++++++++++++++++----------------- archivebox/index.py | 42 ++++++++++++++------------ archivebox/logs.py | 18 ++++++------ archivebox/util.py | 3 +- 5 files changed, 65 insertions(+), 59 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index 8779f5cf..18d31023 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -180,7 +180,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] all_links, new_links = load_links_index(out_dir=OUTPUT_DIR, import_path=import_path) # Step 2: Write updated index with deduped old and new links back to disk - write_links_index(out_dir=OUTPUT_DIR, links=list(all_links)) + write_links_index(links=list(all_links), out_dir=OUTPUT_DIR) # Step 3: Run the archive methods for each link links = new_links if ONLY_NEW else all_links @@ -189,7 +189,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] link: Optional[Link] = None try: for idx, link in enumerate(links_after_timestamp(links, resume)): - archive_link(link) + archive_link(link, link_dir=link.link_dir) except KeyboardInterrupt: log_archiving_paused(len(links), idx, link.timestamp if link else '0') @@ -203,7 +203,7 @@ def update_archive_data(import_path: Optional[str]=None, resume: Optional[float] # Step 4: Re-write links index with updated titles, icons, and resources all_links, _ = load_links_index(out_dir=OUTPUT_DIR) - write_links_index(out_dir=OUTPUT_DIR, links=list(all_links), finished=True) + write_links_index(links=list(all_links), out_dir=OUTPUT_DIR, finished=True) return all_links if __name__ == '__main__': diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 16ee3392..7a76df13 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -1,6 +1,6 @@ import os -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Optional from collections import defaultdict from datetime import datetime @@ -69,7 +69,7 @@ class ArchiveError(Exception): @enforce_types -def archive_link(link: Link, page=None) -> Link: +def archive_link(link: Link, link_dir: Optional[str]=None) -> Link: """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" ARCHIVE_METHODS = ( @@ -84,13 +84,14 @@ def archive_link(link: Link, page=None) -> Link: ('archive_org', should_fetch_archive_dot_org, archive_dot_org), ) + link_dir = link_dir or link.link_dir try: - is_new = not os.path.exists(link.link_dir) + is_new = not os.path.exists(link_dir) if is_new: - os.makedirs(link.link_dir) + os.makedirs(link_dir) - link = load_json_link_index(link.link_dir, link) - log_link_archiving_started(link.link_dir, link, is_new) + link = load_json_link_index(link, link_dir) + log_link_archiving_started(link, link_dir, is_new) link = link.overwrite(updated=datetime.now()) stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} @@ -99,10 +100,10 @@ def archive_link(link: Link, page=None) -> Link: if method_name not in link.history: link.history[method_name] = [] - if should_run(link.link_dir, link): + if should_run(link, link_dir): log_archive_method_started(method_name) - result = method_function(link.link_dir, link) + result = method_function(link, link_dir) link.history[method_name].append(result) @@ -126,7 +127,7 @@ def archive_link(link: Link, page=None) -> Link: patch_links_index(link) - log_link_archiving_finished(link.link_dir, link, is_new, stats) + log_link_archiving_finished(link, link.link_dir, is_new, stats) except KeyboardInterrupt: raise @@ -141,7 +142,7 @@ def archive_link(link: Link, page=None) -> Link: ### Archive Method Functions @enforce_types -def should_fetch_title(link_dir: str, link: Link) -> bool: +def should_fetch_title(link: Link, link_dir: Optional[str]=None) -> bool: # if link already has valid title, skip it if link.title and not link.title.lower().startswith('http'): return False @@ -152,7 +153,7 @@ def should_fetch_title(link_dir: str, link: Link) -> bool: return FETCH_TITLE @enforce_types -def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_title(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """try to guess the page's title from its content""" output = None @@ -186,14 +187,14 @@ def fetch_title(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResul @enforce_types -def should_fetch_favicon(link_dir: str, link: Link) -> bool: +def should_fetch_favicon(link: Link, link_dir: Optional[str]=None) -> bool: if os.path.exists(os.path.join(link_dir, 'favicon.ico')): return False return FETCH_FAVICON @enforce_types -def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_favicon(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download site favicon from google's favicon api""" output = 'favicon.ico' @@ -226,7 +227,7 @@ def fetch_favicon(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveRes ) @enforce_types -def should_fetch_wget(link_dir: str, link: Link) -> bool: +def should_fetch_wget(link: Link, link_dir: Optional[str]=None) -> bool: output_path = wget_output_path(link) if output_path and os.path.exists(os.path.join(link_dir, output_path)): return False @@ -235,7 +236,7 @@ def should_fetch_wget(link_dir: str, link: Link) -> bool: @enforce_types -def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using wget""" if FETCH_WARC: @@ -315,7 +316,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult ) @enforce_types -def should_fetch_pdf(link_dir: str, link: Link) -> bool: +def should_fetch_pdf(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -326,7 +327,7 @@ def should_fetch_pdf(link_dir: str, link: Link) -> bool: @enforce_types -def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_pdf(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """print PDF of site to file using chrome --headless""" output = 'output.pdf' @@ -361,7 +362,7 @@ def fetch_pdf(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) @enforce_types -def should_fetch_screenshot(link_dir: str, link: Link) -> bool: +def should_fetch_screenshot(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -371,7 +372,7 @@ def should_fetch_screenshot(link_dir: str, link: Link) -> bool: return FETCH_SCREENSHOT @enforce_types -def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_screenshot(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """take screenshot of site using chrome --headless""" output = 'screenshot.png' @@ -406,7 +407,7 @@ def fetch_screenshot(link_dir: str, link: Link, timeout: int=TIMEOUT) -> Archive ) @enforce_types -def should_fetch_dom(link_dir: str, link: Link) -> bool: +def should_fetch_dom(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -416,7 +417,7 @@ def should_fetch_dom(link_dir: str, link: Link) -> bool: return FETCH_DOM @enforce_types -def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_dom(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """print HTML of site to file using chrome --dump-html""" output = 'output.html' @@ -453,7 +454,7 @@ def fetch_dom(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: ) @enforce_types -def should_fetch_git(link_dir: str, link: Link) -> bool: +def should_fetch_git(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -471,7 +472,7 @@ def should_fetch_git(link_dir: str, link: Link) -> bool: @enforce_types -def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def fetch_git(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using git""" output = 'git' @@ -514,7 +515,7 @@ def fetch_git(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: @enforce_types -def should_fetch_media(link_dir: str, link: Link) -> bool: +def should_fetch_media(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -524,7 +525,7 @@ def should_fetch_media(link_dir: str, link: Link) -> bool: return FETCH_MEDIA @enforce_types -def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: +def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: """Download playlists or individual video, audio, and subtitles using youtube-dl""" output = 'media' @@ -588,7 +589,7 @@ def fetch_media(link_dir: str, link: Link, timeout: int=MEDIA_TIMEOUT) -> Archiv @enforce_types -def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: +def should_fetch_archive_dot_org(link: Link, link_dir: Optional[str]=None) -> bool: if is_static_file(link.url): return False @@ -599,7 +600,7 @@ def should_fetch_archive_dot_org(link_dir: str, link: Link) -> bool: return SUBMIT_ARCHIVE_DOT_ORG @enforce_types -def archive_dot_org(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult: +def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """submit site to archive.org for archiving via their service, save returned archive url""" output = 'archive.org.txt' diff --git a/archivebox/index.py b/archivebox/index.py index 66b234a2..d3fe7965 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -39,23 +39,25 @@ from .logs import ( TITLE_LOADING_MSG = 'Not yet archived...' + + ### Homepage index for all the links @enforce_types -def write_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: +def write_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None: """create index.html file for a given list of links""" log_indexing_process_started() log_indexing_started(out_dir, 'index.json') timer = TimedProgress(TIMEOUT * 2, prefix=' ') - write_json_links_index(out_dir, links) + write_json_links_index(links, out_dir=out_dir) timer.end() log_indexing_finished(out_dir, 'index.json') log_indexing_started(out_dir, 'index.html') timer = TimedProgress(TIMEOUT * 2, prefix=' ') - write_html_links_index(out_dir, links, finished=finished) + write_html_links_index(links, out_dir=out_dir, finished=finished) timer.end() log_indexing_finished(out_dir, 'index.html') @@ -87,7 +89,7 @@ def load_links_index(out_dir: str=OUTPUT_DIR, import_path: Optional[str]=None) - @enforce_types -def write_json_links_index(out_dir: str, links: List[Link]) -> None: +def write_json_links_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None: """write the json link index to a given path""" assert isinstance(links, List), 'Links must be a list, not a generator.' @@ -199,7 +201,6 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: successful = link.num_outputs # Patch JSON index - changed = False json_file_links = parse_json_links_index(out_dir) patched_links = [] for saved_link in json_file_links: @@ -212,7 +213,7 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: else: patched_links.append(saved_link) - write_json_links_index(out_dir, patched_links) + write_json_links_index(patched_links, out_dir=out_dir) # Patch HTML index html_path = os.path.join(out_dir, 'index.html') @@ -231,27 +232,27 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: ### Individual link index @enforce_types -def write_link_index(out_dir: str, link: Link) -> None: - write_json_link_index(out_dir, link) - write_html_link_index(out_dir, link) +def write_link_index(link: Link, link_dir: Optional[str]=None) -> None: + link_dir = link_dir or link.link_dir + + write_json_link_index(link, link_dir) + write_html_link_index(link, link_dir) @enforce_types -def write_json_link_index(out_dir: str, link: Link) -> None: +def write_json_link_index(link: Link, link_dir: Optional[str]=None) -> None: """write a json file with some info about the link""" - path = os.path.join(out_dir, 'index.json') - - with open(path, 'w', encoding='utf-8') as f: - json.dump(link._asdict(), f, indent=4, cls=ExtendedEncoder) + link_dir = link_dir or link.link_dir + path = os.path.join(link_dir, 'index.json') chmod_file(path) @enforce_types -def parse_json_link_index(out_dir: str) -> Optional[Link]: +def parse_json_link_index(link_dir: str) -> Optional[Link]: """load the json link index from a given directory""" - existing_index = os.path.join(out_dir, 'index.json') + existing_index = os.path.join(link_dir, 'index.json') if os.path.exists(existing_index): with open(existing_index, 'r', encoding='utf-8') as f: link_json = json.load(f) @@ -260,18 +261,21 @@ def parse_json_link_index(out_dir: str) -> Optional[Link]: @enforce_types -def load_json_link_index(out_dir: str, link: Link) -> Link: +def load_json_link_index(link: Link, link_dir: Optional[str]=None) -> Link: """check for an existing link archive in the given directory, and load+merge it into the given link dict """ - existing_link = parse_json_link_index(out_dir) + link_dir = link_dir or link.link_dir + existing_link = parse_json_link_index(link_dir) if existing_link: return merge_links(existing_link, link) return link @enforce_types -def write_html_link_index(out_dir: str, link: Link) -> None: +def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None: + link_dir = link_dir or link.link_dir + with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: link_html = f.read() diff --git a/archivebox/logs.py b/archivebox/logs.py index 4e21731b..155f81e6 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Optional from .schema import Link, ArchiveResult -from .config import ANSI, REPO_DIR, OUTPUT_DIR +from .config import ANSI, OUTPUT_DIR @dataclass @@ -17,14 +17,14 @@ class RuntimeStats: succeeded: int = 0 failed: int = 0 - parse_start_ts: datetime = None - parse_end_ts: datetime = None + parse_start_ts: Optional[datetime] = None + parse_end_ts: Optional[datetime] = None - index_start_ts: datetime = None - index_end_ts: datetime = None + index_start_ts: Optional[datetime] = None + index_end_ts: Optional[datetime] = None - archiving_start_ts: datetime = None - archiving_end_ts: datetime = None + archiving_start_ts: Optional[datetime] = None + archiving_end_ts: Optional[datetime] = None # globals are bad, mmkay _LAST_RUN_STATS = RuntimeStats() @@ -131,7 +131,7 @@ def log_archiving_finished(num_links: int): print(' {}/index.html'.format(OUTPUT_DIR)) -def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): +def log_link_archiving_started(link: Link, link_dir: str, is_new: bool): # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford" # http://www.benstopford.com/2015/02/14/log-structured-merge-trees/ # > output/archive/1478739709 @@ -149,7 +149,7 @@ def log_link_archiving_started(link_dir: str, link: Link, is_new: bool): pretty_path(link_dir), )) -def log_link_archiving_finished(link_dir: str, link: Link, is_new: bool, stats: dict): +def log_link_archiving_finished(link: Link, link_dir: str, is_new: bool, stats: dict): total = sum(stats.values()) if stats['failed'] > 0 : diff --git a/archivebox/util.py b/archivebox/util.py index fe3c57cf..dd71eecd 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -1,11 +1,12 @@ import os import re import sys +import json import time import shutil from json import JSONEncoder -from typing import List, Optional, Any +from typing import List, Optional, Any, Union from inspect import signature, _empty from functools import wraps from hashlib import sha256 From d2a34f260287af8348ca2ffc7f5b0116bb2ecaf1 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 18:24:57 -0400 Subject: [PATCH 036/365] switch to atomic disk writes for all index operations --- archivebox/index.py | 61 +++++++++++++++++++++------------------------ archivebox/util.py | 19 ++++++++++++++ 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index d3fe7965..749f282c 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -25,6 +25,7 @@ from .util import ( enforce_types, TimedProgress, copy_and_overwrite, + atomic_write, ) from .parse import parse_links from .links import validate_links @@ -113,11 +114,7 @@ def write_json_links_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None: 'updated': datetime.now(), 'links': links, } - - with open(path, 'w', encoding='utf-8') as f: - json.dump(index_json, f, indent=4, cls=ExtendedEncoder) - - chmod_file(path) + atomic_write(index_json, path) @enforce_types @@ -141,15 +138,17 @@ def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: @enforce_types -def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False) -> None: +def write_html_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None: """write the html link index to a given path""" path = os.path.join(out_dir, 'index.html') - copy_and_overwrite(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static')) + copy_and_overwrite( + os.path.join(TEMPLATES_DIR, 'static'), + os.path.join(out_dir, 'static'), + ) - with open(os.path.join(out_dir, 'robots.txt'), 'w+') as f: - f.write('User-agent: *\nDisallow: /') + atomic_write('User-agent: *\nDisallow: /', os.path.join(out_dir, 'robots.txt')) with open(os.path.join(TEMPLATES_DIR, 'index.html'), 'r', encoding='utf-8') as f: index_html = f.read() @@ -187,10 +186,8 @@ def write_html_links_index(out_dir: str, links: List[Link], finished: bool=False 'status': 'finished' if finished else 'running', } - with open(path, 'w', encoding='utf-8') as f: - f.write(Template(index_html).substitute(**template_vars)) + atomic_write(Template(index_html).substitute(**template_vars), path) - chmod_file(path) @enforce_types @@ -225,8 +222,7 @@ def patch_links_index(link: Link, out_dir: str=OUTPUT_DIR) -> None: html[idx] = '<span>{}</span>'.format(successful) break - with open(html_path, 'w') as f: - f.write('\n'.join(html)) + atomic_write('\n'.join(html), html_path) ### Individual link index @@ -246,7 +242,7 @@ def write_json_link_index(link: Link, link_dir: Optional[str]=None) -> None: link_dir = link_dir or link.link_dir path = os.path.join(link_dir, 'index.json') - chmod_file(path) + atomic_write(link._asdict(), path) @enforce_types @@ -279,23 +275,22 @@ def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None: with open(os.path.join(TEMPLATES_DIR, 'link_index.html'), 'r', encoding='utf-8') as f: link_html = f.read() - path = os.path.join(out_dir, 'index.html') + path = os.path.join(link_dir, 'index.html') - with open(path, 'w', encoding='utf-8') as f: - f.write(Template(link_html).substitute({ - **derived_link_info(link), - 'title': ( - link.title - or (link.base_url if link.is_archived else TITLE_LOADING_MSG) - ), - 'archive_url': urlencode( - wget_output_path(link) - or (link.domain if link.is_archived else 'about:blank') - ), - 'extension': link.extension or 'html', - 'tags': link.tags or 'untagged', - 'status': 'archived' if link.is_archived else 'not yet archived', - 'status_color': 'success' if link.is_archived else 'danger', - })) + html_index = Template(link_html).substitute({ + **derived_link_info(link), + 'title': ( + link.title + or (link.base_url if link.is_archived else TITLE_LOADING_MSG) + ), + 'archive_url': urlencode( + wget_output_path(link) + or (link.domain if link.is_archived else 'about:blank') + ), + 'extension': link.extension or 'html', + 'tags': link.tags or 'untagged', + 'status': 'archived' if link.is_archived else 'not yet archived', + 'status_color': 'success' if link.is_archived else 'danger', + }) - chmod_file(path) + atomic_write(html_index, path) diff --git a/archivebox/util.py b/archivebox/util.py index dd71eecd..31f64a60 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -670,3 +670,22 @@ class ExtendedEncoder(JSONEncoder): return tuple(obj) return JSONEncoder.default(self, obj) + + +def atomic_write(contents: Union[dict, str], path: str): + """Safe atomic file write and swap using a tmp file""" + try: + tmp_file = '{}.tmp'.format(path) + with open(tmp_file, 'w+', encoding='utf-8') as f: + if isinstance(contents, dict): + json.dump(contents, f, indent=4, cls=ExtendedEncoder) + else: + f.write(contents) + + os.fsync(f.fileno()) + + os.rename(tmp_file, path) + chmod_file(path) + finally: + if os.path.exists(tmp_file): + os.remove(tmp_file) From 8b50fee0f54c65abe489edbbfdc2767dd6fa3800 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 18:25:17 -0400 Subject: [PATCH 037/365] better type checking of latest output methods --- archivebox/schema.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/archivebox/schema.py b/archivebox/schema.py index d1bb06ea..472bdc58 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -14,12 +14,14 @@ class ArchiveError(Exception): LinkDict = Dict[str, Any] +ArchiveOutput = Union[str, Exception, None] + @dataclass(frozen=True) class ArchiveResult: cmd: List[str] pwd: Optional[str] cmd_version: Optional[str] - output: Union[str, Exception, None] + output: ArchiveOutput status: str start_ts: datetime end_ts: datetime @@ -211,31 +213,26 @@ class Link: domain(self.url), )) - def latest_outputs(self, status: str=None) -> Dict[str, Optional[str]]: + def latest_outputs(self, status: str=None) -> Dict[str, ArchiveOutput]: """get the latest output that each archive method produced for link""" - latest = { - 'title': None, - 'favicon': None, - 'wget': None, - 'warc': None, - 'pdf': None, - 'screenshot': None, - 'dom': None, - 'git': None, - 'media': None, - 'archive_org': None, - } - for archive_method in latest.keys(): + ARCHIVE_METHODS = ( + 'title', 'favicon', 'wget', 'warc', 'pdf', + 'screenshot', 'dom', 'git', 'media', 'archive_org', + ) + latest: Dict[str, ArchiveOutput] = {} + for archive_method in ARCHIVE_METHODS: # get most recent succesful result in history for each archive method history = self.history.get(archive_method) or [] - history = filter(lambda result: result.output, reversed(history)) + history = list(filter(lambda result: result.output, reversed(history))) if status is not None: - history = filter(lambda result: result.status == status, history) + history = list(filter(lambda result: result.status == status, history)) history = list(history) if history: latest[archive_method] = history[0].output + else: + latest[archive_method] = None return latest From cc3d1e9cc9335af5e1b757487d2557226ecdc8bb Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 18:26:22 -0400 Subject: [PATCH 038/365] limit length of stringified arg_vals in exceptions --- archivebox/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/util.py b/archivebox/util.py index 31f64a60..f3427155 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -138,7 +138,7 @@ def enforce_types(func): annotation.__name__, type(arg_val).__name__, arg_key, - arg_val, + str(arg_val)[:64], ) ) From 4c8e45b8d70b08a817323b7aefff4859432116e9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 20:48:41 -0400 Subject: [PATCH 039/365] save all imports to sources dir --- archivebox/archive.py | 12 ++++---- archivebox/archive_methods.py | 4 +-- archivebox/util.py | 53 ++++++++++++++++++----------------- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/archivebox/archive.py b/archivebox/archive.py index 18d31023..b0a28428 100755 --- a/archivebox/archive.py +++ b/archivebox/archive.py @@ -43,8 +43,8 @@ from .config import ( ) from .util import ( enforce_types, - save_remote_source, - save_stdin_source, + handle_stdin_import, + handle_file_import, ) from .logs import ( log_archiving_started, @@ -160,12 +160,12 @@ def main(args=None) -> None: print_help() raise SystemExit(1) - import_path = save_stdin_source(stdin_raw_text) + import_path = handle_stdin_import(stdin_raw_text) - ### Handle ingesting urls from a remote file/feed + ### Handle ingesting url from a remote file/feed # (e.g. if an RSS feed URL is used as the import path) - if import_path and any(import_path.startswith(s) for s in ('http://', 'https://', 'ftp://')): - import_path = save_remote_source(import_path) + if import_path: + import_path = handle_file_import(import_path) ### Run the main archive update process update_archive_data(import_path=import_path, resume=resume) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 7a76df13..fdd941da 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -90,7 +90,7 @@ def archive_link(link: Link, link_dir: Optional[str]=None) -> Link: if is_new: os.makedirs(link_dir) - link = load_json_link_index(link, link_dir) + link = load_json_link_index(link, link_dir=link_dir) log_link_archiving_started(link, link_dir, is_new) link = link.overwrite(updated=datetime.now()) stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} @@ -103,7 +103,7 @@ def archive_link(link: Link, link_dir: Optional[str]=None) -> Link: if should_run(link, link_dir): log_archive_method_started(method_name) - result = method_function(link, link_dir) + result = method_function(link=link, link_dir=link_dir) link.history[method_name].append(result) diff --git a/archivebox/util.py b/archivebox/util.py index f3427155..2d8a546a 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -187,7 +187,7 @@ def check_url_parsing_invariants() -> None: ### Random Helpers @enforce_types -def save_stdin_source(raw_text: str) -> str: +def handle_stdin_import(raw_text: str) -> str: if not os.path.exists(SOURCES_DIR): os.makedirs(SOURCES_DIR) @@ -195,14 +195,12 @@ def save_stdin_source(raw_text: str) -> str: source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format('stdin', ts)) - with open(source_path, 'w', encoding='utf-8') as f: - f.write(raw_text) - + atomic_write(raw_text, source_path) return source_path @enforce_types -def save_remote_source(url: str, timeout: int=TIMEOUT) -> str: +def handle_file_import(path: str, timeout: int=TIMEOUT) -> str: """download a given url's content into output/sources/domain-<timestamp>.txt""" if not os.path.exists(SOURCES_DIR): @@ -210,30 +208,35 @@ def save_remote_source(url: str, timeout: int=TIMEOUT) -> str: ts = str(datetime.now().timestamp()).split('.', 1)[0] - source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format(domain(url), ts)) + source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format(basename(path), ts)) - print('{}[*] [{}] Downloading {}{}'.format( - ANSI['green'], - datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - url, - ANSI['reset'], - )) - timer = TimedProgress(timeout, prefix=' ') - try: - downloaded_xml = download_url(url, timeout=timeout) - timer.end() - except Exception as e: - timer.end() - print('{}[!] Failed to download {}{}\n'.format( - ANSI['red'], - url, + if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')): + source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format(domain(path), ts)) + print('{}[*] [{}] Downloading {}{}'.format( + ANSI['green'], + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + path, ANSI['reset'], )) - print(' ', e) - raise SystemExit(1) + timer = TimedProgress(timeout, prefix=' ') + try: + raw_source_text = download_url(path, timeout=timeout) + timer.end() + except Exception as e: + timer.end() + print('{}[!] Failed to download {}{}\n'.format( + ANSI['red'], + path, + ANSI['reset'], + )) + print(' ', e) + raise SystemExit(1) - with open(source_path, 'w', encoding='utf-8') as f: - f.write(downloaded_xml) + else: + with open(path, 'r') as f: + raw_source_text = f.read() + + atomic_write(raw_source_text, source_path) print(' > {}'.format(pretty_path(source_path))) From 9cdceecda864095400972df717f12b1ea273824b Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Wed, 27 Mar 2019 20:49:09 -0400 Subject: [PATCH 040/365] final link rewrite on ctrl+c --- archivebox/archive_methods.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index fdd941da..888d6c87 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -120,16 +120,19 @@ def archive_link(link: Link, link_dir: Optional[str]=None) -> Link: # print(' ', stats) # If any changes were made, update the link index json and html - write_link_index(link.link_dir, link) + write_link_index(link, link_dir=link.link_dir) was_changed = stats['succeeded'] or stats['failed'] if was_changed: patch_links_index(link) - log_link_archiving_finished(link, link.link_dir, is_new, stats) except KeyboardInterrupt: + try: + write_link_index(link, link_dir=link.link_dir) + except: + pass raise except Exception as err: From 35c05c321ff8030952d725723f8dd6cf327f8c9a Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:03:31 -0400 Subject: [PATCH 041/365] minor bin version checking changes --- archivebox/config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index cc032e83..df634102 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -120,7 +120,7 @@ if sys.stdout.encoding.upper() not in ('UTF-8', 'UTF8'): # ***************************** Helper Functions ******************************* # ****************************************************************************** -def check_version(binary: str) -> str: +def bin_version(binary: str) -> str: """check the presence and return valid version line of a specified binary""" if not shutil.which(binary): print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI)) @@ -139,6 +139,7 @@ def check_version(binary: str) -> str: def find_chrome_binary() -> Optional[str]: """find any installed chrome binaries in the default locations""" # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev + # make sure data dir finding precedence order always matches binary finding order default_executable_paths = ( 'chromium-browser', 'chromium', @@ -164,6 +165,7 @@ def find_chrome_binary() -> Optional[str]: def find_chrome_data_dir() -> Optional[str]: """find any installed chrome user data directories in the default locations""" # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev + # make sure data dir finding precedence order always matches binary finding order default_profile_paths = ( '~/.config/chromium', '~/Library/Application Support/Chromium', @@ -197,7 +199,7 @@ try: FETCH_FAVICON = SUBMIT_ARCHIVE_DOT_ORG = False CURL_VERSION = None if USE_CURL: - CURL_VERSION = check_version(CURL_BINARY) + CURL_VERSION = bin_version(CURL_BINARY) ### Make sure wget is installed and calculate version if USE_WGET: @@ -207,7 +209,7 @@ try: WGET_VERSION = None WGET_AUTO_COMPRESSION = False if USE_WGET: - WGET_VERSION = check_version(WGET_BINARY) + WGET_VERSION = bin_version(WGET_BINARY) WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode WGET_USER_AGENT = WGET_USER_AGENT.format( @@ -218,12 +220,12 @@ try: ### Make sure git is installed GIT_VERSION = None if FETCH_GIT: - GIT_VERSION = check_version(GIT_BINARY) + GIT_VERSION = bin_version(GIT_BINARY) ### Make sure youtube-dl is installed YOUTUBEDL_VERSION = None if FETCH_MEDIA: - YOUTUBEDL_VERSION = check_version(YOUTUBEDL_BINARY) + YOUTUBEDL_VERSION = bin_version(YOUTUBEDL_BINARY) ### Make sure chrome is installed and calculate version if USE_CHROME: @@ -236,7 +238,7 @@ try: CHROME_VERSION = None if USE_CHROME: if CHROME_BINARY: - CHROME_VERSION = check_version(CHROME_BINARY) + CHROME_VERSION = bin_version(CHROME_BINARY) # print('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY)) if CHROME_USER_DATA_DIR is None: From 73f46b0b293e400f1feca59218d60d5f809f2cfd Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:03:46 -0400 Subject: [PATCH 042/365] add proper typechecked json parsing and dumping --- archivebox/index.py | 8 +---- archivebox/schema.py | 85 +++++++++++++++++++++++++++++++++++++------- archivebox/util.py | 4 +-- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index 749f282c..de3c2f58 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -121,18 +121,12 @@ def write_json_links_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None: def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: """parse a archive index json file and return the list of links""" - allowed_fields = {f.name for f in fields(Link)} - index_path = os.path.join(out_dir, 'index.json') if os.path.exists(index_path): with open(index_path, 'r', encoding='utf-8') as f: links = json.load(f)['links'] for link_json in links: - yield Link(**{ - key: val - for key, val in link_json.items() - if key in allowed_fields - }) + yield Link.from_json(link_json) return () diff --git a/archivebox/schema.py b/archivebox/schema.py index 472bdc58..a4d3a836 100644 --- a/archivebox/schema.py +++ b/archivebox/schema.py @@ -4,7 +4,7 @@ from datetime import datetime from typing import List, Dict, Any, Optional, Union -from dataclasses import dataclass, asdict, field +from dataclasses import dataclass, asdict, field, fields class ArchiveError(Exception): @@ -28,11 +28,38 @@ class ArchiveResult: schema: str = 'ArchiveResult' def __post_init__(self): - assert self.schema == self.__class__.__name__ + self.typecheck() def _asdict(self): return asdict(self) + def typecheck(self) -> None: + assert self.schema == self.__class__.__name__ + assert isinstance(self.status, str) and self.status + assert isinstance(self.start_ts, datetime) + assert isinstance(self.end_ts, datetime) + assert isinstance(self.cmd, list) + assert all(isinstance(arg, str) and arg for arg in self.cmd) + assert self.pwd is None or isinstance(self.pwd, str) and self.pwd + assert self.cmd_version is None or isinstance(self.cmd_version, str) and self.cmd_version + assert self.output is None or isinstance(self.output, (str, Exception)) + if isinstance(self.output, str): + assert self.output + + @classmethod + def from_json(cls, json_info): + from .util import parse_date + + allowed_fields = {f.name for f in fields(cls)} + info = { + key: val + for key, val in json_info.items() + if key in allowed_fields + } + info['start_ts'] = parse_date(info['start_ts']) + info['end_ts'] = parse_date(info['end_ts']) + return cls(**info) + @property def duration(self) -> int: return (self.end_ts - self.start_ts).seconds @@ -49,17 +76,7 @@ class Link: schema: str = 'Link' def __post_init__(self): - """fix any history result items to be type-checked ArchiveResults""" - assert self.schema == self.__class__.__name__ - cast_history = {} - for method, method_history in self.history.items(): - cast_history[method] = [] - for result in method_history: - if isinstance(result, dict): - result = ArchiveResult(**result) - cast_history[method].append(result) - - object.__setattr__(self, 'history', cast_history) + self.typecheck() def overwrite(self, **kwargs): """pure functional version of dict.update that returns a new instance""" @@ -76,6 +93,22 @@ class Link: if not self.timestamp or not other.timestamp: return return float(self.timestamp) > float(other.timestamp) + + def typecheck(self) -> None: + assert self.schema == self.__class__.__name__ + assert isinstance(self.timestamp, str) and self.timestamp + assert self.timestamp.replace('.', '').isdigit() + assert isinstance(self.url, str) and '://' in self.url + assert self.updated is None or isinstance(self.updated, datetime) + assert self.title is None or isinstance(self.title, str) and self.title + assert self.tags is None or isinstance(self.tags, str) and self.tags + assert isinstance(self.sources, list) + assert all(isinstance(source, str) and source for source in self.sources) + assert isinstance(self.history, dict) + for method, results in self.history.items(): + assert isinstance(method, str) and method + assert isinstance(results, list) + assert all(isinstance(result, ArchiveResult) for result in results) def _asdict(self, extended=False): info = { @@ -108,6 +141,32 @@ class Link: }) return info + @classmethod + def from_json(cls, json_info): + from .util import parse_date + + allowed_fields = {f.name for f in fields(cls)} + info = { + key: val + for key, val in json_info.items() + if key in allowed_fields + } + info['updated'] = parse_date(info['updated']) + + json_history = info['history'] + cast_history = {} + + for method, method_history in json_history.items(): + cast_history[method] = [] + for json_result in method_history: + assert isinstance(json_result, dict), 'Items in Link["history"][method] must be dicts' + cast_result = ArchiveResult.from_json(json_result) + cast_history[method].append(cast_result) + + info['history'] = cast_history + return cls(**info) + + @property def link_dir(self) -> str: from .config import ARCHIVE_DIR diff --git a/archivebox/util.py b/archivebox/util.py index 2d8a546a..1e785d43 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -675,8 +675,8 @@ class ExtendedEncoder(JSONEncoder): return JSONEncoder.default(self, obj) -def atomic_write(contents: Union[dict, str], path: str): - """Safe atomic file write and swap using a tmp file""" +def atomic_write(contents: Union[dict, str], path: str) -> None: + """Safe atomic write to filesystem by writing to temp file + atomic rename""" try: tmp_file = '{}.tmp'.format(path) with open(tmp_file, 'w+', encoding='utf-8') as f: From b58afcf97496a428e81dc8f6817589a8b628a405 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:37:36 -0400 Subject: [PATCH 043/365] fix broken import --- archivebox/links.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/archivebox/links.py b/archivebox/links.py index 8844ef9b..6fb5af38 100644 --- a/archivebox/links.py +++ b/archivebox/links.py @@ -8,9 +8,8 @@ from .util import ( merge_links, ) -from config import ( - URL_BLACKLIST, -) +from .config import URL_BLACKLIST + def validate_links(links: Iterable[Link]) -> Iterable[Link]: links = archivable_links(links) # remove chrome://, about:, mailto: etc. From 03a388300f60cec28b91a818d47f84b9f9344b0f Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:38:28 -0400 Subject: [PATCH 044/365] fix link parsing --- archivebox/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/index.py b/archivebox/index.py index de3c2f58..1b9cdb66 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -246,7 +246,7 @@ def parse_json_link_index(link_dir: str) -> Optional[Link]: if os.path.exists(existing_index): with open(existing_index, 'r', encoding='utf-8') as f: link_json = json.load(f) - return Link(**link_json) + return Link.from_json(link_json) return None From 1e45b026845bc8cadf3c821bf9ea20f6bb058efe Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:41:26 -0400 Subject: [PATCH 045/365] add flake8 config --- archivebox/.flake8 | 4 ++++ archivebox/index.py | 3 --- archivebox/parse.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 archivebox/.flake8 diff --git a/archivebox/.flake8 b/archivebox/.flake8 new file mode 100644 index 00000000..46da144b --- /dev/null +++ b/archivebox/.flake8 @@ -0,0 +1,4 @@ +[flake8] +ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E127,E131,E241,E252,E266,E272,E701,E731,W293,W503 +select = F,E9 +exclude = migrations,util_scripts,node_modules,venv diff --git a/archivebox/index.py b/archivebox/index.py index 1b9cdb66..3621b35e 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -4,7 +4,6 @@ import json from datetime import datetime from string import Template from typing import List, Tuple, Iterator, Optional -from dataclasses import fields from .schema import Link, ArchiveResult from .config import ( @@ -17,11 +16,9 @@ from .config import ( ) from .util import ( merge_links, - chmod_file, urlencode, derived_link_info, wget_output_path, - ExtendedEncoder, enforce_types, TimedProgress, copy_and_overwrite, diff --git a/archivebox/parse.py b/archivebox/parse.py index 6ecc0007..5c5a6438 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -66,7 +66,7 @@ def parse_links(source_file: str) -> Tuple[List[Link], str]: if links: timer.end() return links, parser_name - except Exception as err: + except Exception as err: # noqa # Parsers are tried one by one down the list, and the first one # that succeeds is used. To see why a certain parser was not used # due to error or format incompatibility, uncomment this line: From f6a25d9c1289163157c3bf6a1ea3fd190981a8fb Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:45:48 -0400 Subject: [PATCH 046/365] better Archivebox.conf.default examples --- etc/ArchiveBox.conf.default | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/etc/ArchiveBox.conf.default b/etc/ArchiveBox.conf.default index dcb8aeac..8d646bfa 100644 --- a/etc/ArchiveBox.conf.default +++ b/etc/ArchiveBox.conf.default @@ -15,7 +15,7 @@ #MEDIA_TIMEOUT=3600 #TEMPLATES_DIR="archivebox/templates" #FOOTER_INFO="Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests." - +#URL_BLACKLIST="(://(.*\.)?youtube\.com)|(://(.*\.)?amazon\.com)|(.*\.exe$)" ################################################################################ ## Archive Method Toggles @@ -45,6 +45,8 @@ #GIT_DOMAINS="github.com,bitbucket.org,gitlab.com" #COOKIES_FILE="path/to/cookies.txt" #CHROME_USER_DATA_DIR="~/.config/google-chrome/Default" +#CROME_HEADLESS=True +#CROME_SANDBOX=True ################################################################################ @@ -59,6 +61,10 @@ ## Dependency Options ################################################################################ +#USE_CURL=True +#USE_WGET=True +#USE_CHROME=True + #CURL_BINARY="curl" #GIT_BINARY="git" #WGET_BINARY="wget" From fffeb21ad461f285db405b285c267e9028ec5f6b Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 15:50:32 -0400 Subject: [PATCH 047/365] remove progress dots in title fetching --- archivebox/util.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/archivebox/util.py b/archivebox/util.py index 1e785d43..487cfeab 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -251,10 +251,6 @@ def fetch_page_title(url: str, timeout: int=10, progress: bool=SHOW_PROGRESS) -> return None try: - if progress: - sys.stdout.write('.') - sys.stdout.flush() - html = download_url(url, timeout=timeout) match = re.search(HTML_TITLE_REGEX, html) From 307193c3eaa978da37742b50b4319cbe69392f13 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 17:35:25 -0400 Subject: [PATCH 048/365] add zlib support to wget in Docker --- Dockerfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5683cad..aa0f90e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # This Dockerfile for ArchiveBox installs the following in a container: -# - curl, wget, python3, youtube-dl, google-chrome-unstable +# - curl, wget, python3, youtube-dl, chromium-browser # - ArchiveBox # Usage: # docker build github.com/pirate/ArchiveBox -t archivebox @@ -13,15 +13,14 @@ LABEL maintainer="Nick Sweeting <archivebox-git@sweeting.me>" RUN apt-get update \ && apt-get install -yq --no-install-recommends \ - git wget curl youtube-dl gnupg2 libgconf-2-4 python3 python3-pip \ + git zlib1g-dev wget curl youtube-dl gnupg2 libgconf-2-4 python3 python3-pip \ && rm -rf /var/lib/apt/lists/* # Install latest chrome package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) -RUN apt-get update && apt-get install -y wget --no-install-recommends \ - && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ && apt-get update \ - && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ + && apt-get install -y chromium-browser fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /src/*.deb @@ -32,7 +31,7 @@ RUN chmod +x /usr/local/bin/dumb-init # Uncomment to skip the chromium download when installing puppeteer. If you do, # you'll need to launch puppeteer with: -# browser.launch({executablePath: 'google-chrome-unstable'}) +# browser.launch({executablePath: 'chromium-browser'}) ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true # Install puppeteer so it's available in the container. @@ -61,7 +60,7 @@ ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ PYTHONIOENCODING=UTF-8 \ CHROME_SANDBOX=False \ - CHROME_BINARY=google-chrome-unstable \ + CHROME_BINARY=chromium-browser \ OUTPUT_DIR=/data # Run everything from here on out as non-privileged user From 68d1eb03899e951aacbb0a90584efca9742f4a21 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 17:40:55 -0400 Subject: [PATCH 049/365] change chromium to google-chrome --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa0f90e9..216e60b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # This Dockerfile for ArchiveBox installs the following in a container: -# - curl, wget, python3, youtube-dl, chromium-browser +# - curl, wget, python3, youtube-dl, google-chrome-beta # - ArchiveBox # Usage: # docker build github.com/pirate/ArchiveBox -t archivebox @@ -20,7 +20,7 @@ RUN apt-get update \ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ && apt-get update \ - && apt-get install -y chromium-browser fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ + && apt-get install -y google-chrome-beta fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /src/*.deb @@ -31,7 +31,7 @@ RUN chmod +x /usr/local/bin/dumb-init # Uncomment to skip the chromium download when installing puppeteer. If you do, # you'll need to launch puppeteer with: -# browser.launch({executablePath: 'chromium-browser'}) +# browser.launch({executablePath: 'google-chrome-beta'}) ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true # Install puppeteer so it's available in the container. @@ -60,7 +60,7 @@ ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ PYTHONIOENCODING=UTF-8 \ CHROME_SANDBOX=False \ - CHROME_BINARY=chromium-browser \ + CHROME_BINARY=google-chrome-beta \ OUTPUT_DIR=/data # Run everything from here on out as non-privileged user From 8283b353f4fb5476c4785e8646c844aa31781516 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 17:43:44 -0400 Subject: [PATCH 050/365] fix wget_auto_compression check prining stderr to console --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index 84e74710..ef6b09fd 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -221,7 +221,7 @@ try: WGET_AUTO_COMPRESSION = False if USE_WGET: WGET_VERSION = bin_version(WGET_BINARY) - WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode + WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL, stderr=DEVNULL).returncode WGET_USER_AGENT = WGET_USER_AGENT.format( VERSION=VERSION, From 6d92784dbb9dc1cfe4e9124d54d04ed56b3e44f9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 17:51:23 -0400 Subject: [PATCH 051/365] add docstring about timestamp parsing --- archivebox/util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/archivebox/util.py b/archivebox/util.py index 487cfeab..c47741d5 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -368,6 +368,11 @@ def parse_date(date: Any) -> Optional[datetime]: if isinstance(date, str): if date.replace('.', '').isdigit(): + # this is a brittle attempt at unix timestamp parsing (which is + # notoriously hard to do). It may lead to dates being off by + # anything from hours to decades, depending on which app, OS, + # and sytem time configuration was used for the original timestamp + # more info: https://github.com/pirate/ArchiveBox/issues/119 timestamp = float(date) EARLIEST_POSSIBLE = 473403600.0 # 1985 From 880b425df64f40944d78e307d52b845e01318b77 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 17:57:39 -0400 Subject: [PATCH 052/365] add note about saving timestamp strings independently --- archivebox/util.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/archivebox/util.py b/archivebox/util.py index c47741d5..617c7e2e 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -373,6 +373,11 @@ def parse_date(date: Any) -> Optional[datetime]: # anything from hours to decades, depending on which app, OS, # and sytem time configuration was used for the original timestamp # more info: https://github.com/pirate/ArchiveBox/issues/119 + + # Note: always always always store the original timestamp string + # somewhere indepentendly of the parsed datetime, so that later + # bugs dont repeatedly misparse and rewrite increasingly worse dates. + # the correct date can always be re-derived from the timestamp str timestamp = float(date) EARLIEST_POSSIBLE = 473403600.0 # 1985 @@ -389,6 +394,12 @@ def parse_date(date: Any) -> Optional[datetime]: # number is microseconds return datetime.fromtimestamp(timestamp / (1000*1000)) + else: + # continue to the end and raise a parsing failed error. + # we dont want to even attempt parsing timestamp strings that + # arent within these ranges + pass + if '-' in date: try: return datetime.fromisoformat(date) From ac9fed06fdc8fb5443295b3b4ea670bc0a4edaa6 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 17:59:56 -0400 Subject: [PATCH 053/365] whitespace --- archivebox/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archivebox/util.py b/archivebox/util.py index 617c7e2e..9c62526d 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -386,6 +386,7 @@ def parse_date(date: Any) -> Optional[datetime]: if EARLIEST_POSSIBLE < timestamp < LATEST_POSSIBLE: # number is seconds return datetime.fromtimestamp(timestamp) + elif EARLIEST_POSSIBLE * 1000 < timestamp < LATEST_POSSIBLE * 1000: # number is milliseconds return datetime.fromtimestamp(timestamp / 1000) From f4e018ba0c903ed9ed64387ead1492497c462dcf Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 20:49:45 -0400 Subject: [PATCH 054/365] fix a bunch of mypy errors --- archivebox/archive_methods.py | 75 +++++++++++++++++++++-------------- archivebox/config.py | 17 +++----- archivebox/links.py | 4 +- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index 888d6c87..acf332a4 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -4,7 +4,7 @@ from typing import Dict, List, Tuple, Optional from collections import defaultdict from datetime import datetime -from .schema import Link, ArchiveResult +from .schema import Link, ArchiveResult, ArchiveOutput from .index import ( write_link_index, patch_links_index, @@ -159,13 +159,13 @@ def should_fetch_title(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_title(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """try to guess the page's title from its content""" - output = None + output: ArchiveOutput = None cmd = [ CURL_BINARY, link.url, '|', 'grep', - '<title>', + '<title', ] status = 'succeeded' timer = TimedProgress(timeout, prefix=' ') @@ -191,6 +191,7 @@ def fetch_title(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) @enforce_types def should_fetch_favicon(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir if os.path.exists(os.path.join(link_dir, 'favicon.ico')): return False @@ -200,13 +201,14 @@ def should_fetch_favicon(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_favicon(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download site favicon from google's favicon api""" - output = 'favicon.ico' + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'favicon.ico' cmd = [ CURL_BINARY, '--max-time', str(timeout), '--location', - '--output', output, - *(() if CHECK_SSL_VALIDITY else ('--insecure',)), + '--output', str(output), + *([] if CHECK_SSL_VALIDITY else ['--insecure']), 'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)), ] status = 'succeeded' @@ -232,6 +234,7 @@ def fetch_favicon(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT @enforce_types def should_fetch_wget(link: Link, link_dir: Optional[str]=None) -> bool: output_path = wget_output_path(link) + link_dir = link_dir or link.link_dir if output_path and os.path.exists(os.path.join(link_dir, output_path)): return False @@ -242,13 +245,14 @@ def should_fetch_wget(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using wget""" + link_dir = link_dir or link.link_dir if FETCH_WARC: warc_dir = os.path.join(link_dir, 'warc') os.makedirs(warc_dir, exist_ok=True) warc_path = os.path.join('warc', str(int(datetime.now().timestamp()))) # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html - output = None + output: ArchiveOutput = None cmd = [ WGET_BINARY, # '--server-response', # print headers for better error parsing @@ -262,13 +266,13 @@ def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) - '-e', 'robots=off', '--restrict-file-names=unix', '--timeout={}'.format(timeout), - *(() if FETCH_WARC else ('--timestamping',)), - *(('--warc-file={}'.format(warc_path),) if FETCH_WARC else ()), - *(('--page-requisites',) if FETCH_WGET_REQUISITES else ()), - *(('--user-agent={}'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()), - *(('--load-cookies', COOKIES_FILE) if COOKIES_FILE else ()), - *(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()), - *((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', '--no-hsts'))), + *([] if FETCH_WARC else ['--timestamping']), + *(['--warc-file={}'.format(warc_path)] if FETCH_WARC else []), + *(['--page-requisites'] if FETCH_WGET_REQUISITES else []), + *(['--user-agent={}'.format(WGET_USER_AGENT)] if WGET_USER_AGENT else []), + *(['--load-cookies', COOKIES_FILE] if COOKIES_FILE else []), + *(['--compression=auto'] if WGET_AUTO_COMPRESSION else []), + *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']), link.url, ] status = 'succeeded' @@ -320,6 +324,7 @@ def fetch_wget(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) - @enforce_types def should_fetch_pdf(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir if is_static_file(link.url): return False @@ -333,7 +338,8 @@ def should_fetch_pdf(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_pdf(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """print PDF of site to file using chrome --headless""" - output = 'output.pdf' + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'output.pdf' cmd = [ *chrome_args(TIMEOUT=timeout), '--print-to-pdf', @@ -366,6 +372,7 @@ def fetch_pdf(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> @enforce_types def should_fetch_screenshot(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir if is_static_file(link.url): return False @@ -377,8 +384,9 @@ def should_fetch_screenshot(link: Link, link_dir: Optional[str]=None) -> bool: @enforce_types def fetch_screenshot(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """take screenshot of site using chrome --headless""" - - output = 'screenshot.png' + + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'screenshot.png' cmd = [ *chrome_args(TIMEOUT=timeout), '--screenshot', @@ -411,6 +419,7 @@ def fetch_screenshot(link: Link, link_dir: Optional[str]=None, timeout: int=TIME @enforce_types def should_fetch_dom(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir if is_static_file(link.url): return False @@ -423,8 +432,9 @@ def should_fetch_dom(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_dom(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """print HTML of site to file using chrome --dump-html""" - output = 'output.html' - output_path = os.path.join(link_dir, output) + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'output.html' + output_path = os.path.join(link_dir, str(output)) cmd = [ *chrome_args(TIMEOUT=timeout), '--dump-dom', @@ -458,6 +468,7 @@ def fetch_dom(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> @enforce_types def should_fetch_git(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir if is_static_file(link.url): return False @@ -478,15 +489,16 @@ def should_fetch_git(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_git(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """download full site using git""" - output = 'git' - output_path = os.path.join(link_dir, 'git') + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'git' + output_path = os.path.join(link_dir, str(output)) os.makedirs(output_path, exist_ok=True) cmd = [ GIT_BINARY, 'clone', '--mirror', '--recursive', - *(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')), + *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']), without_query(without_fragment(link.url)), ] status = 'succeeded' @@ -519,6 +531,8 @@ def fetch_git(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> @enforce_types def should_fetch_media(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir + if is_static_file(link.url): return False @@ -531,8 +545,9 @@ def should_fetch_media(link: Link, link_dir: Optional[str]=None) -> bool: def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult: """Download playlists or individual video, audio, and subtitles using youtube-dl""" - output = 'media' - output_path = os.path.join(link_dir, 'media') + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'media' + output_path = os.path.join(link_dir, str(output)) os.makedirs(output_path, exist_ok=True) cmd = [ YOUTUBEDL_BINARY, @@ -553,7 +568,7 @@ def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIM '--audio-quality', '320K', '--embed-thumbnail', '--add-metadata', - *(() if CHECK_SSL_VALIDITY else ('--no-check-certificate',)), + *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']), link.url, ] status = 'succeeded' @@ -593,6 +608,7 @@ def fetch_media(link: Link, link_dir: Optional[str]=None, timeout: int=MEDIA_TIM @enforce_types def should_fetch_archive_dot_org(link: Link, link_dir: Optional[str]=None) -> bool: + link_dir = link_dir or link.link_dir if is_static_file(link.url): return False @@ -606,7 +622,8 @@ def should_fetch_archive_dot_org(link: Link, link_dir: Optional[str]=None) -> bo def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: """submit site to archive.org for archiving via their service, save returned archive url""" - output = 'archive.org.txt' + link_dir = link_dir or link.link_dir + output: ArchiveOutput = 'archive.org.txt' archive_org_url = None submit_url = 'https://web.archive.org/save/{}'.format(link.url) cmd = [ @@ -615,7 +632,7 @@ def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEO '--head', '--user-agent', 'ArchiveBox/{} (+https://github.com/pirate/ArchiveBox/)'.format(VERSION), # be nice to the Archive.org people and show them where all this ArchiveBox traffic is coming from '--max-time', str(timeout), - *(() if CHECK_SSL_VALIDITY else ('--insecure',)), + *([] if CHECK_SSL_VALIDITY else ['--insecure']), submit_url, ] status = 'succeeded' @@ -638,13 +655,13 @@ def archive_dot_org(link: Link, link_dir: Optional[str]=None, timeout: int=TIMEO finally: timer.end() - if not isinstance(output, Exception): + if output and not isinstance(output, Exception): # instead of writing None when archive.org rejects the url write the # url to resubmit it to archive.org. This is so when the user visits # the URL in person, it will attempt to re-archive it, and it'll show the # nicer error message explaining why the url was rejected if it fails. archive_org_url = archive_org_url or submit_url - with open(os.path.join(link_dir, output), 'w', encoding='utf-8') as f: + with open(os.path.join(link_dir, str(output)), 'w', encoding='utf-8') as f: f.write(archive_org_url) chmod_file('archive.org.txt', cwd=link_dir) output = archive_org_url diff --git a/archivebox/config.py b/archivebox/config.py index ef6b09fd..f9f5ea57 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -3,13 +3,10 @@ import re import sys import shutil -from typing import Optional, Pattern +from typing import Optional from subprocess import run, PIPE, DEVNULL -OUTPUT_DIR: str -URL_BLACKLIST: Optional[Pattern[str]] - # ****************************************************************************** # Documentation: https://github.com/pirate/ArchiveBox/wiki/Configuration # Use the 'env' command to pass config options to ArchiveBox. e.g.: @@ -48,6 +45,7 @@ COOKIES_FILE = os.getenv('COOKIES_FILE', None) CHROME_USER_DATA_DIR = os.getenv('CHROME_USER_DATA_DIR', None) CHROME_HEADLESS = os.getenv('CHROME_HEADLESS', 'True' ).lower() == 'true' CHROME_USER_AGENT = os.getenv('CHROME_USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36') +CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True' ).lower() == 'true' USE_CURL = os.getenv('USE_CURL', 'True' ).lower() == 'true' USE_WGET = os.getenv('USE_WGET', 'True' ).lower() == 'true' @@ -59,12 +57,7 @@ WGET_BINARY = os.getenv('WGET_BINARY', 'wget') YOUTUBEDL_BINARY = os.getenv('YOUTUBEDL_BINARY', 'youtube-dl') CHROME_BINARY = os.getenv('CHROME_BINARY', None) -CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' -try: - OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR')) -except Exception: - OUTPUT_DIR = None # ****************************************************************************** @@ -103,7 +96,7 @@ TEMPLATES_DIR = os.path.join(PYTHON_DIR, 'templates') if COOKIES_FILE: COOKIES_FILE = os.path.abspath(COOKIES_FILE) -URL_BLACKLIST = URL_BLACKLIST and re.compile(URL_BLACKLIST, re.IGNORECASE) +URL_BLACKLIST_PTN = re.compile(URL_BLACKLIST, re.IGNORECASE) if URL_BLACKLIST else None ########################### Environment & Dependencies ######################### @@ -147,7 +140,7 @@ def bin_version(binary: str) -> str: raise SystemExit(1) -def find_chrome_binary() -> Optional[str]: +def find_chrome_binary() -> str: """find any installed chrome binaries in the default locations""" # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev # make sure data dir finding precedence order always matches binary finding order @@ -244,7 +237,7 @@ try: else: FETCH_PDF = FETCH_SCREENSHOT = FETCH_DOM = False - if CHROME_BINARY is None: + if not CHROME_BINARY: CHROME_BINARY = find_chrome_binary() or 'chromium-browser' CHROME_VERSION = None if USE_CHROME: diff --git a/archivebox/links.py b/archivebox/links.py index 6fb5af38..914c3575 100644 --- a/archivebox/links.py +++ b/archivebox/links.py @@ -8,7 +8,7 @@ from .util import ( merge_links, ) -from .config import URL_BLACKLIST +from .config import URL_BLACKLIST_PTN def validate_links(links: Iterable[Link]) -> Iterable[Link]: @@ -26,7 +26,7 @@ def archivable_links(links: Iterable[Link]) -> Iterable[Link]: """remove chrome://, about:// or other schemed links that cant be archived""" for link in links: scheme_is_valid = scheme(link.url) in ('http', 'https', 'ftp') - not_blacklisted = (not URL_BLACKLIST.match(link.url)) if URL_BLACKLIST else True + not_blacklisted = (not URL_BLACKLIST_PTN.match(link.url)) if URL_BLACKLIST_PTN else True if scheme_is_valid and not_blacklisted: yield link From 6a8f6f52afe6e822db7a1034b8b5d710204fa314 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 21:29:16 -0400 Subject: [PATCH 055/365] 0 mypy errors --- archivebox/index.py | 23 +++++++++++------------ archivebox/logs.py | 3 ++- archivebox/parse.py | 22 ++++++++++++---------- archivebox/purge.py | 17 ++++++++--------- archivebox/util.py | 10 +++++----- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/archivebox/index.py b/archivebox/index.py index 3621b35e..d7e230a3 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -3,7 +3,7 @@ import json from datetime import datetime from string import Template -from typing import List, Tuple, Iterator, Optional +from typing import List, Tuple, Iterator, Optional, Mapping from .schema import Link, ArchiveResult from .config import ( @@ -132,8 +132,6 @@ def parse_json_links_index(out_dir: str=OUTPUT_DIR) -> Iterator[Link]: def write_html_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None: """write the html link index to a given path""" - path = os.path.join(out_dir, 'index.html') - copy_and_overwrite( os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static'), @@ -147,8 +145,9 @@ def write_html_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: with open(os.path.join(TEMPLATES_DIR, 'index_row.html'), 'r', encoding='utf-8') as f: link_row_html = f.read() - link_rows = '\n'.join( - Template(link_row_html).substitute(**{ + link_rows = [] + for link in links: + template_row_vars: Mapping[str, str] = { **derived_link_info(link), 'title': ( link.title @@ -162,22 +161,22 @@ def write_html_links_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: 'archive_url': urlencode( wget_output_path(link) or 'index.html' ), - }) - for link in links - ) + } + link_rows.append(Template(link_row_html).substitute(**template_row_vars)) - template_vars = { - 'num_links': len(links), + template_vars: Mapping[str, str] = { + 'num_links': str(len(links)), 'date_updated': datetime.now().strftime('%Y-%m-%d'), 'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'), 'footer_info': FOOTER_INFO, 'version': VERSION, 'git_sha': GIT_SHA, - 'rows': link_rows, + 'rows': '\n'.join(link_rows), 'status': 'finished' if finished else 'running', } + template_html = Template(index_html).substitute(**template_vars) - atomic_write(Template(index_html).substitute(**template_vars), path) + atomic_write(template_html, os.path.join(out_dir, 'index.html')) diff --git a/archivebox/logs.py b/archivebox/logs.py index 155f81e6..d9b92422 100644 --- a/archivebox/logs.py +++ b/archivebox/logs.py @@ -111,6 +111,7 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str): def log_archiving_finished(num_links: int): end_ts = datetime.now() _LAST_RUN_STATS.archiving_end_ts = end_ts + assert _LAST_RUN_STATS.archiving_start_ts is not None seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp() if seconds > 60: duration = '{0:.2f} min'.format(seconds / 60, 2) @@ -194,7 +195,7 @@ def log_archive_method_finished(result: ArchiveResult): ), *hints, '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']), - *((' cd {};'.format(result.pwd),) if result.pwd else ()), + *([' cd {};'.format(result.pwd)] if result.pwd else []), ' {}'.format(quoted_cmd), ] print('\n'.join( diff --git a/archivebox/parse.py b/archivebox/parse.py index 5c5a6438..49ffa7fd 100644 --- a/archivebox/parse.py +++ b/archivebox/parse.py @@ -266,10 +266,12 @@ def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: root = etree.parse(rss_file).getroot() items = root.findall("{http://purl.org/rss/1.0/}item") for item in items: - url = item.find("{http://purl.org/rss/1.0/}link").text - tags = item.find("{http://purl.org/dc/elements/1.1/}subject").text if item.find("{http://purl.org/dc/elements/1.1/}subject") else None - title = item.find("{http://purl.org/rss/1.0/}title").text.strip() if item.find("{http://purl.org/rss/1.0/}title").text.strip() else None - ts_str = item.find("{http://purl.org/dc/elements/1.1/}date").text if item.find("{http://purl.org/dc/elements/1.1/}date").text else None + find = lambda p: item.find(p).text.strip() if item.find(p) else None # type: ignore + + url = find("{http://purl.org/rss/1.0/}link") + tags = find("{http://purl.org/dc/elements/1.1/}subject") + title = find("{http://purl.org/rss/1.0/}title") + ts_str = find("{http://purl.org/dc/elements/1.1/}date") # Pinboard includes a colon in its date stamp timezone offsets, which # Python can't parse. Remove it: @@ -296,12 +298,12 @@ def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: rss_file.seek(0) root = etree.parse(rss_file).getroot() - items = root.find("channel").findall("item") + items = root.find("channel").findall("item") # type: ignore for item in items: - url = item.find("link").text - title = item.find("title").text.strip() - ts_str = item.find("pubDate").text - time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z") + url = item.find("link").text # type: ignore + title = item.find("title").text.strip() # type: ignore + ts_str = item.find("pubDate").text # type: ignore + time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z") # type: ignore yield Link( url=htmldecode(url), @@ -319,7 +321,7 @@ def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: text_file.seek(0) for line in text_file.readlines(): urls = re.findall(URL_REGEX, line) if line.strip() else () - for url in urls: + for url in urls: # type: ignore yield Link( url=htmldecode(url), timestamp=str(datetime.now().timestamp()), diff --git a/archivebox/purge.py b/archivebox/purge.py index 26b18817..ddc64b6b 100755 --- a/archivebox/purge.py +++ b/archivebox/purge.py @@ -6,9 +6,8 @@ from os.path import exists, join from shutil import rmtree from typing import List -from archive import parse_json_link_index -from config import ARCHIVE_DIR, OUTPUT_DIR -from index import write_html_links_index, write_json_links_index +from .config import ARCHIVE_DIR, OUTPUT_DIR +from .index import parse_json_links_index, write_html_links_index, write_json_links_index def cleanup_index(regexes: List[str], proceed: bool, delete: bool) -> None: @@ -16,18 +15,18 @@ def cleanup_index(regexes: List[str], proceed: bool, delete: bool) -> None: exit('index.json is missing; nothing to do') compiled = [re.compile(r) for r in regexes] - links = parse_json_link_index(OUTPUT_DIR)['links'] + links = parse_json_links_index(OUTPUT_DIR) filtered = [] remaining = [] - for l in links: - url = l['url'] + for link in links: + url = link.url for r in compiled: if r.search(url): - filtered.append((l, r)) + filtered.append((link, r)) break else: - remaining.append(l) + remaining.append(link) if not filtered: exit('Search did not match any entries.') @@ -35,7 +34,7 @@ def cleanup_index(regexes: List[str], proceed: bool, delete: bool) -> None: print('Filtered out {}/{} urls:'.format(len(filtered), len(links))) for link, regex in filtered: - url = link['url'] + url = link.url print(' {url} via {regex}'.format(url=url, regex=regex.pattern)) if not proceed: diff --git a/archivebox/util.py b/archivebox/util.py index 9c62526d..bc3fd1a0 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -7,7 +7,7 @@ import shutil from json import JSONEncoder from typing import List, Optional, Any, Union -from inspect import signature, _empty +from inspect import signature from functools import wraps from hashlib import sha256 from urllib.request import Request, urlopen @@ -24,7 +24,7 @@ from subprocess import ( CalledProcessError, ) -from base32_crockford import encode as base32_encode +from base32_crockford import encode as base32_encode # type: ignore from .schema import Link from .config import ( @@ -127,9 +127,9 @@ def enforce_types(func): try: annotation = sig.parameters[arg_key].annotation except KeyError: - annotation = _empty + annotation = None - if annotation is not _empty and annotation.__class__ is type: + if annotation is not None and annotation.__class__ is type: if not isinstance(arg_val, annotation): raise TypeError( '{}(..., {}: {}) got unexpected {} argument {}={}'.format( @@ -605,7 +605,7 @@ def download_url(url: str, timeout: int=TIMEOUT) -> str: insecure = ssl._create_unverified_context() resp = urlopen(req, timeout=timeout, context=insecure) - encoding = resp.headers.get_content_charset() or 'utf-8' + encoding = resp.headers.get_content_charset() or 'utf-8' # type: ignore return resp.read().decode(encoding) From 97249a1861ecef2764967709e2b5b3689e7b6637 Mon Sep 17 00:00:00 2001 From: Nick Sweeting <git@nicksweeting.com> Date: Sat, 30 Mar 2019 22:25:10 -0400 Subject: [PATCH 056/365] handle urls with special characters properly --- archivebox/archive_methods.py | 2 +- archivebox/index.py | 9 +++++++-- archivebox/templates/index.html | 9 +++++---- archivebox/templates/link_index.html | 22 ++++++++-------------- archivebox/util.py | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index acf332a4..105c39b7 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -119,9 +119,9 @@ def archive_link(link: Link, link_dir: Optional[str]=None) -> Link: # print(' ', stats) - # If any changes were made, update the link index json and html write_link_index(link, link_dir=link.link_dir) + # If any changes were made, update the main links index json and html was_changed = stats['succeeded'] or stats['failed'] if was_changed: patch_links_index(link) diff --git a/archivebox/index.py b/archivebox/index.py index d7e230a3..b3cd350e 100644 --- a/archivebox/index.py +++ b/archivebox/index.py @@ -17,6 +17,8 @@ from .config import ( from .util import ( merge_links, urlencode, + htmlencode, + urldecode, derived_link_info, wget_output_path, enforce_types, @@ -267,12 +269,13 @@ def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None: path = os.path.join(link_dir, 'index.html') - html_index = Template(link_html).substitute({ + template_vars: Mapping[str, str] = { **derived_link_info(link), 'title': ( link.title or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), + 'url_str': htmlencode(urldecode(link.base_url)), 'archive_url': urlencode( wget_output_path(link) or (link.domain if link.is_archived else 'about:blank') @@ -281,6 +284,8 @@ def write_html_link_index(link: Link, link_dir: Optional[str]=None) -> None: 'tags': link.tags or 'untagged', 'status': 'archived' if link.is_archived else 'not yet archived', 'status_color': 'success' if link.is_archived else 'danger', - }) + } + + html_index = Template(link_html).substitute(**template_vars) atomic_write(html_index, path) diff --git a/archivebox/templates/index.html b/archivebox/templates/index.html index 144f2ce7..6b40000a 100644 --- a/archivebox/templates/index.html +++ b/archivebox/templates/index.html @@ -1,7 +1,8 @@ +<!DOCTYPE html> <html lang="en"> <head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> <title>Archived Sites + + + + + + + + +
+
+
+
+ + Logo + ArchiveBox: Add + +
+
+ Main Index   |   + Admin   |   + Docs +
+
+
+
+
+

+
{% csrf_token %} + Add new links...
+
+ +
+
+ + + diff --git a/archivebox/themes/default/main_index.html b/archivebox/themes/default/main_index.html new file mode 100644 index 00000000..f8ab9edc --- /dev/null +++ b/archivebox/themes/default/main_index.html @@ -0,0 +1,243 @@ +{% load static %} + + + + + Archived Sites + + + + + + + + + +
+
+
+
+ + Logo + ArchiveBox: Index + +
+
+ Add Links   |   + Admin   |   + Docs +
+
+
+
+ + + + + + + + + + + {% for link in links %} + + + + + + + {% endfor %} + +
BookmarkedSaved Link ({{num_links}})FilesOriginal URL
{{link.bookmarked_date}} + + + {{link.title}} + {{link.tags|default:''}} + + + 📄 + {{link.num_outputs}} + + {{link.url}}
+
+
+
+ + Archive created using ArchiveBox + version v$version   |   + Download index as JSON +

+ $footer_info + +
+
+
+ + diff --git a/archivebox/themes/static/archive.png b/archivebox/themes/static/archive.png new file mode 100644 index 0000000000000000000000000000000000000000..307b45013382851ef1026e111921bd94ba55af1d GIT binary patch literal 17730 zcmeHvWmuH$7bo(9A|W89fPkQM2?I#T5CTdoNOyNDHGqSH(kLa3QUcQ52ojPbNXJk^ zNaxV(GvT}OWv|`q+WoNm{~#C7^W1Urcg}szjUno)3M7QIgg7`jBu}3_evX5K3j+Qz z5nKh{eC*we#KE~@W~-^=p`)TKX723BYii+aX36X2=mLDl!I1!axtN;UTY4~=Sz6mV zJ!0OfZ)Rq)wRps=Eu_M);v#ElWBbI%%~HchRny$Z-dxmz87xUC;Uxwv;ArV#%H-wf z;N&jm^@th0t{Ct;>NOuT6MBe;{Uc@_6?GmcAjQf>YLw0>Vk<|Xn^IU3$XtBpg(1_3Z>3INE zfQ=3_9ZuIBQ!N{2@f&kt1YG3{wNDA6?s6w2QAY7MXsInQ|HG`TzwrL0meE4=%bD%$ zj>#cf(=4{MUNct~9Hy&xy?qG9aaUz~AL}Gj(6&n-5w|X_Kha`ubtfq3YUSvI z?WZ!1vS75uLG={%BJsrf2bQLD9=3PuYE?KH3j*Sa$z^G>#gF)36L?)Ad5_D%bS)No z6^0+&Ldg&1alv(Jp~HL4^O}VDLr5(Xl||qqrfZAPH?sun_*Ao$U-7NKd&3CDT3)#P zGj-)3lUov37fc8n@b*l|xn;@TzF{Y$x*3#p7yk*#V9>2-Q46A4sWf#QN@6)FdiLAh zZx3a>)$eK&zL0qdQ-j@K;q)M)eaj$20ps4kdP0zCBB1={zNwrJkyT54iQqbkb1=&+ z-orcn!K5wsU8o>eouI|7p0lQ2^#0fRGH^oQ-jw;70a7Q973wCTdQYg}`b_*;?N>8R zs(kX{$Ukz^vJsg}>WnZQ5S5C8eTK@f8#as}y42S*A^i#oU+jPJtT609+Q;)@IKM6X z{fWtMe%4zG;b2zWn9!f!Db4dA^nF_UWZFwGl+Skd@DQsF_ z@@DMmI_`~k6zzF)y)I4{+9ma6IF72+vlcBTO(y->*cKt^Q_fIMh02_=Jm0+B zywN-j5VuOm(vz?p#IOFr6WhH&+`yB-ra;?3%fJNu&LDM}P?<=XQW<=iYT?khtF z)f84I&VK3vDlp&06Wn<9xb4`k=!f@zv6V7$+_&Ys&F8JWo3xk`8%?Rfsh9sGU&Br- ziPMqy&V9psesU-HhS5$>qp}lVzAJkIHwA78{1k{2I2Y(CQ!JA#(=AgeQ(JuRMqKMs z%UtVJ>+Nmpt$nO~eDzp?5D!0CCad*XkWtWmx%+b9cE8ZVP(ejz#Yn|fMG{5xZc0lc z1RjDDamuMf6|5NE?HYIGh0d^)(DO>=gCg2*R$r;JUl};6-^q0P*8i;dIpZ*sy^dX$ z4ai2>hS0Xo#>%$Ju4(Y&DB*9htl@$K-AO5Nt<#D_^E+4Wu%&XRzDgCT%dc~o7MMQu zp?oCXXz1hUQ+zPI-?p!|0_M-4t)X2PToFF9u(9Y8asZ_WeH0-S<`ki|gx%T+(B z+eGnlaqWUR^n$UveJNF%rQuZ)=EvQNlcDy*4wV*;W{quiJy$Kwon!2WoViTa%=PU~ zf2)r0wibPNnwe>lY-{SdvM{^u(q;Pl+2Dr_o#lsH7K;^ad7X*t>j;BCXT5U6L2HAX zg!N*z46p7?el7V>UOG%tBU*RomG6AfV%qFQ`L&U@>CE1R+T2L%9P475VfzB@#?dd; zah_15IiqE|MY@x*5u6Dw2EPxcYnEwt4dA)FbK!~m1NRH=1gep#&J(zpJ$y8oRh8{X7p@&P5W=X&|McNU{(+ZET5K{Q$rO_xZMxJd`E*5(>lZ^P z_hQRpk$FvdG=oP&4{iM>UsP|wB~zUeF3hiry-EF*$`Mn`=5F>55!syz>yJ*Iu3vm* zxs^2G*j)dL)K#rF&A6;}bc@!_%dJK<^Hdr60ups0`RDZF z+~+iE<1&RV$yZ!Wy!fH>{q)p;(`%`WivBu}$ow;!TS{hsNUuxxfBzs${mzc)m}rQI zjkKP@iQ@|mg1S+>?4YaA?0h~n^GD`Sg-#^{#fiwDjN)Dn*`P9yn%?|xc?Mq~0mp+o zumxGRPi(YY_qcXBCDX#ijxz?E%CZ$p^N%PGHM{ecbN$qi7hOlCMS9=Ficd61a;+Lh z%BynwzZMl4eQCM5bRNnsNSm}HGo$05q~t< zt$FC)y*F>Wx>|R8Ix$kWejG}ZY^2;69#AWhuqS@Gg(!|SHavTCtN?GgJo|0Gxi&0v z>?{W#X&4F^ZJce|_ITjr>dk>{TsfT}4JU16JeO4E%PpGKQk2Yt*j=ulc{>O>y%b7o zxAZH;WxqGO?SeD&8%OFo&TFyPNB7fS@*UMze3M+Jrw6bG>&5O$^BJXBf?xYIfkc=qTbh& zu7UA_G$^0Ej&-?N5%lJVkXeSRy7Wh$7hfEo2-~*Dvxm&5*7r2_?VsN89%)M)pL63Y zckf5i*?Mg*LoVT$>-_-(jxyWKQm=8Y;NlUGG5z2BWswV}W52=(+5X;dKU#GDV!f)! zptkImv6Aa}wpKyqlnp38sn62;C(s7)u$b))@Dg(W;8h4?ejb!+%>xq z;qoVb(vGpZq$K*Puj-mmKvJ~<>p`w#wv$igKzeYGZR)~M#i6#S9|QuiG~CcqOq@ku+V6W0zMwCzi#T#0d2*`$r^r%Y zLqnsWziL8k^JU`jNQ`iUd1qwYMkid!Z*uB3(r&NQuYOBxVQal<@+pyLKSjXRO`RM+ zpXG{`=Cboc+l`*A+giz5yl}}!ea67gCi>B)rbE{qPlKdbej%e>E~;;kjppV_F&brM7RbWrvTl+I`zv0|N|u8J$0z5A znz9H)&4&-_ng^ERJ*&1g>wNVa3tDSYVTBIW22Pd+T;J}o@}$qK4v(kDD;4+eAZiXg zPI-px^#J8;I5r3^&e&N`K$@=qnwY_P_6TM|y zxRv##Ue&MKa8ukDWKROTPuJ=R6KP%O7M2l8t}y{e!^QpwUCtLv&i?dKgHvMorHbOu zrD4c}KLy2+VC0m0nDObv=wbV9@R(5J+V9@=-hlV#NTgwj3?5o7gaA@k6__gTo-p9*p-( zsp2N^8i`RYIGzuI&xgK`8VC4!HjkZBgRBSAcz9zjC!5bFk7eK8ctoqc|I?pKFair4 zCAzrf<>j68b@KhG;vyFSsT*6>Gfns^4RXE+hT5J4zRP-lE`fiC?CV2l_AIfjJxGo5%zG5L+zf zL3wUI3F>{`5NuTkKSAwuO4*$&;=O+jIhEZH1oCg!?@IY20N&`t5 zq)Bm51Pc~79ZFk@7CNIu7RZwhXh1v3(mgDoYgIL+T54E;UAbQ(?yoP+#b9x~I+}|7 zRIo*!wcLhP&q4GkDvh+!Z=Bh4o}SsWB=+jw51=^H13e2U#;zT@JNOtozfu9 zjZ>@YkDl`1NZ}A6<*I()Um9W7JdEVAnfDZp%S}0vOf80ARsL5hQW;;RfiH!Z1S$=V z_1rdtB8EQX*cGmFlJWU+6fI1{$dmrLc7xsKYLYM=cBU9!&fs=%(PzYp%l;ymfU^7&mi3&^jT%ubzr z{03&}=u|1!+=JjqSi@ewc=m)|)MD)38g^+sVcCp_GJ?XwRTs;!ipo}wTw&rytWsO> zg#9NEOU*mGHB=*E#N5upk1>k!Aaq!X)tKz21UcUFxsHy((C@%IQIK_i3Wt6{2FHC5 z>}zQvQ2asxT%L;a+~co!gJqazsv0>4K0iAmmJ<^SL@RHR17-sfMiBJbq(Q#0Vnh}! zZBoul$Plbk0mUe|OCzK7bKug^&H}=5(t9@~3G@0EHB?0uu}GU_`ZV_yh8$vi;WgSM z;?=jx7?O$KwB%{z%5XKERJMOe@E6O$YE$-ZiEgzcsT@D8G-lu;$84-dnm3IQT6SOK zq#)|sCyd`b57)=#3qg3Ym{FR~nNR{JN>Y^n_6uv@g?RIxN;z9@dI8dV`$KA@gpc{L zUxe_5J)UjXZ#AvOY6$HW4WnOYXyMX{AwWVx*%^7mn@qRR7Ot)WC7DKmL`Cy5@~mbm zG5q~jG?5*JDJv_p+t^i#69l7~*3>YORU>Tj+W8WgljJ>WhnY?X^@@3JZ)mj!4kvCH2dqeS%@~ zx0Z>6!?&&Pab+f}*U4yj0e?+#x`$1>OnUHDgGxSYs5;`S!1HZ-{wc+NVA6+b%xR9n zIC6DJ;uLHsLF1e9xNK;JPV$v}4zzmpN=v%@T=X|Cn&jwuDSKJ`2o_N77I+>rtnF^e z=di#@LRwEz5(UXhjD1*ekTKpLLhdFsd5ZFIbH@&y7Tr_QmBw=M4@Hgt`N>JT?9i*X zR<6pb?-&{Mn0TYGN$AcvU(khrNk{5y!{J2aKO>e~fO( zbJ|or)c*_H`;?zl%%d2=O&9hy6Hb$9Tea6&FdlgYSD(vep*4soVsCi>Ij!g&p*)Z{ zZyp@B8Ua&>?NbVT#Q4;h8PwyM%ctaI{Y#Eq0PwxL^J*;Z!sr|hFh-376~~*GjvGH} zkBtBlrlRHy+xlR6Y4>Zze+@(mSHx+iL6^Zq_hpQqnrQo$3VSXM&EmH@YKp&-(P!b` z+ioLu{sv5DuXz|yDYQ|c^;dJwP%9fFHI2cb?~ThfLRAvDL(XYr+Dil~#&;>O=)!Ze zhJla3Nm)guusI`ybcBm2jZXjflY+m{xKHsZgml;*aGh@``g)#94@FJUi5(3tqyq4OEtk$1#TM!3(}BEw_fdD4HrtvTWF4l}@p=F`(r*pFQNsLH7I z^=5r(4gYe)a6dlDU&UT!;->&OYPVhhIMUVAQ#4jgj2r|I`gQm?*3@A4)g`7xX1;=E zgt8qU1L0sNbbg%181kU7B_?nZwvR=2z$I?0i3sTzj#xDF9P-#C^({?}jg4K@1O^?w zZBjbdvcN_jDH9^RCFI6r0xxuY+<6+vY&sbOAWOB&X}I_l0#jZZWK>wy*($K9RNef-Q zy}fih6Nw(~&)2r++S1pn`+;mtZ~I3G#a=Vgx7l+e%=m2nF@3{rk?nB>2EWC`)z#HY z+IKyZb>_u1f35NycgJV?@?ch$^rA<{KIOfA?RF3!WG>Ik!wOXRz(`>a;cw5&m@?{| z=CV0e>$k(+Hw>he;A!`{)}VpT$=XW^Bkh>b#EP?`BS|v&syb|La_bF?=^}KjSw#3x z)7bPKxNXzw(F{3UV<(j`%j4RMx?%|`t<_Xozfoqc{spXaJ2Xpqo0y9V-#_4MN18$S zet7!q_c%b{;amInN44obyNbR7I|0d^@Dc>l3_onTc{=1n8?b#XV7r*nXXW#y+u7Vg z^PYYBWuVu_k@}GlI*|%)@oa2t=GJ+)bNM>M!V(tB9g}v*JNk}(H-Dka z6# zzquCX)Nq*V3Pk9RBEx#xQl{G>YqulBe~bYUSN!;J(s=-K++*D2T8+*e=Cu70+O2{E z5)_e=UZ5W@XQ+M?qan?t950?ap>=)A!R!2Bv~Bg$Uo1tmyvo=8x%3`ez%EldfV?dyQ<+Vu8~ff*C@-&eMpr8w{GB6@oAZoE%F z=t)1FS|0oruVs+4B^~tT8FG7|?uO{uAm90Ou`G|B!&3%f5&ClfRVCv;zH8d!-8#fU z3sHAS2hzNRB@V~T=bBCj#|8P&P|AM`ssqUa&&zV(XoYVa*ZWSaY+YV#U7kPUQQt`{ zZ!khE^?5m#V?9Kj4$2LjZS?*W(a;RDw&C7IN6p&=7P_$B{(do;nD6~Ok5TJ(DKPZ} zvZn8cVv_I0yU~`?<<rL~R{G%GL zjN9g3ItCuUCP>BO_g$Qy8p>)EpX-27dWo(D-l?3Oyy+d{g`gQy=R=X`ldX>aF7+Dy zU6nOAW&ISb$P9U0U22d}ce*-ZK&=+B%SMR_LrjG1HtSc>%f6OE;8TL))IQ^2mj#GmQHZPw?r6dsmhA2UBGAY@ zyB>t$<~=SUZT%Gt@ThxV!B;_O0R2n!T2j*Hj93ad`G8q_6ZXX~Bwhbf<Uqz7a*vQj{}eTN0Mj8I&{Bu*=pWNc^@$D;&!y4=EUk`IU= z&upY35kuMkY$_m?G@m~}hF)tf!jDOt@_A6qZN%C>!fXiR$jDfwe1J`_HC3ggHELsR z`TsXQ0v55fzd97B6nhmL1q+A(yZP2{N{E%693VTY@|-@58d~n4)T;(QuVBN6m(pQb z9H#Cn%gB4xU6F=iafMPH<`H@en`m*e0sI?ugFgK~_z~b+)O**WfT9&EeRV^WfBeZ& z>cQHA5nvidoE_LBPuxww12_$7l`wX+4?ONae-Me0%-7i0fjQ4t2X!!ZnvnoJCWN|T z9UvPpsyM}j4s43i2s}1whGWmZoc944Yu16{u`&i8SF_FNu{z`cbeKJuU5Z_a4|wFG zTEKde;&pZ#&N$u(Y~JPrJo@C_!j=Uz6#)y$`DLTY`Sf3!X#l2I)yz1tiwgm(4GZL8 z53W)uqQia=U|FyOxX-s#_QeW=7Z^(uLXM?l7+~Q=&!2cFe918DQDb2iS2mBgX3O0icWa zn>n>uWmE%WCx2j0!crM#0P&&smEEy=a{|UzhdjUzZ3lEgrSxeDtGE9!igLO-{Ej+> z+p-x=xP@9=S=BVqIzk}3X09mHAAQ&^m!Sd3p2MuB3O6s>Ryeu6zDk;q+ZU?Y6&^k% zc2xBBr@vUR#aG zT>_)nal))t#P@5A!~@_<$kD#F6NeLORQNyhn|I>}_Lg#5tDb81GZ%qtahhJ$|8K6oOpX1MQDIr zd92i2VaNYy?p@nw*>`0^T9!9ph{KeneW*wAfT@#4SX=dbE9OB(o74uggjAjY#{jk0&uR;-q3;? zE2eWfpVi&$l9C#!w>Z!v<)0pJJD$NZqn z&C1H!y0WsO#&)BG@6oRVpt3P{FltO{eEvtft#ZqVZg#{m6foFfAVzBnI_ov!QhlgSE&i|F|!V%Oi3JpE3I?qO- zAJ=AqdQuGp;a1}^Co@*-ZVeQ7;nbnEwY4rliR=qco{AAFV7Pcfr-+j3i4s}ppoxiz zE}QGMB>|TgC7xR|{Gy_@Jf-j&R~85m>gU$S%jY_JBOGntL;wkB#@`c0weswr;exro z>KS6eazRt%=CrAqnORTOuyZ?tJgFQkL04p4KjW;0?$FLwy$yc>9ht{3CQuioUuK6pOlI>|UWhh7 z4sYacMuzUSB)R;%(N6WMp4=Og&CZj&+l9fgT2?G%Yl-qK@j^Qrav4szMW6HZ<=e}d zqa*w2B|RR}OG0WMJ}(!LCWVIM<>cgqwqC6K%Dj8vOa8S$YE!``>7~lI8ksTCMnQV< zSKRczM>;^tC@;gL1oepB{o&3Qiu6NHFRv)a`${uNFB`!YKOmGdryf+Y(2Kg=qYqg3 z@>(zLY<)v2xx)8b85u<1raJIU5x+OHlaXWqjD#HZT1!$K0wrsfpihG2K(zk2Lf@v` z3?rQiV1mqtsJ*4}-wE12+Z&*1o^`JvbN9!sf3-mv1>W)UIvlrd8@N0hc$b^z!wKk- z4NO38%`!`y17*ZIe;xf*pvtdHz-#vOvS3gQ?;-`n%48}0jE zo+IC}N$x!MT1vDI-Ta4CqQsFj?AxO{oOd!((t?z>u=j;GpEOS)*QYhkUMBJ0Pz@+F zs`ux~;`4Lh?fk-eu`HLti@RF2zW;XZSLv^LBCVkV!PN>4PNzXHPm$sKL-Q+`qq9LF z-e7_LRH^7$^*y#-k<(!C*Ux>`Ttg~>@y(m# z_(d5%c@f7~Q~c+0t%tIP)K_Wzbh~1wA(oGgu0hK(o|eYSDUTP*Ld=+4M6TfXO}|wx zj*vPD0&bZkG@y%szm-~?4}B0CnDQD^kM%c%5zHdd1H6XvlIS2QEx79kU+I{31|D?N ztV@lldhH)7(DB9sGp62B31m>ed9!gb&VRRIe^5zPbetUR?>JC9q7(`5vWY8G=4*F8 z*DCOrV^6DXY|*~b3%t2=lWdjm(U3A#4&YDUqc^&kQiZ1UEKn!{Oi|`b#Rv2NGA4Dx zhcX(9E9i3|)Q4c|WNF&xN7)iPCCENUfCZiFeT{4h<7*nrSG5)?sOa5&#zrS{+#a{fb^T&6UfOeGMb*$-98q%I_>y zp=+*q1P*0@_s2*VVk$Zg!vynZd6NV6a22{(0F5BB@D`N=B18OB_|ZTpT2?sFxtm>$8MliZ zY)G)%0F{=$x-o9-r2*m?fUHktpLzb>!Hv(xK(@SSxJh$W)}7f^#PLpnBvtWPc-7-Giq^Gzxsed2TVQMp$cVwR=#@J z{YyvyF*r(9KSl3V5`?Rp<_GZnQ(#s9NY(<=);H0|0|=_ZpO(ieu#XBD0yL7{k!SPw8*;B(Oz>zU=j>HS9*@@UNK)29>y;BH~iQ%<)A0^&@7> zOo97=*)6w~?Hv#d_+@7fJ&hT)oqj*o-;h0N?*-zh*K(?7tMS>f@mrA#INj&kDQ;CT ziP>qD^lK@~_t(tkpXSc9tB5_{TZ~Z)h95=13DUin1WkbhB_XZ&PZ-)OblY!C8;$+S z00Qv#+&jkLEkC6193sK^Msd@LT_^q3>tweoT7Yf}) z^?VZBE&sGbit<)}S}qC$epPj4npK=~9^HQG(oY=YXSnEl91OS4Qd@jJ%sqOQv zfM#}!!2z4P93DgVZyDnkQ~Ng3@ig>4H&mh?egC>a&hGq6RSxLHTZbYm?MJI}A5H37h{s)n8W&T`6b4+pU|A-~JKgvv zZ662=SIAJdwA-2&>1SedRr`jaZ*Mzn@v$s{2O(v(d#8p9qU}&DmyUrD$_<`$10z6` zzG?DP93{X#x9y(+J3qI7s`(47&orL)hXfOljZs5^_UAUxU4Lgz%18vnKezMVDhty- zx)-o-iTRR(R*Czjs>Ua4TR`BR+e-&3F$V*eKEJ@d8^gl5wFFZ@4|k5ne~zY=sQF07 zwWo?G(ETUm<-A9Co>*;nPWI~4tO1EVW!JQeZUom{3-q6M{P(L|WP_y=9F83@ZHr0A<)w8B=C`LD{t%bpS|(bMxy2=Cp|_KAf!n_Rx!eT5QW3 zgl--K9W%fy&j|qZMN(ioPk&eeIcQWvLNV72ERH*U(D;WeS4Cl6B{hIJD$*KjetIq_ zsEz*oI#7O4kdjOQg{!|&<>>10zjJ|3(T9Cve9XnEm#9F_lWG5+ttYGadpkF3Hmbb< zo;08|n7*Th=`cv4tNC$BQ9S~yLIZi*s4eQH;?o_0&}NHh8Vy6ksU51wVyL*qpdSCU zSv*V>h2I&iXwdh6o6~w)kb^0>qyaKX<33aSZef|q5^8h%9GI>dvq4)KW_@@IG z3U^S^{|`@z!Pd5qLV~+=%wwPvfh00VNj0MmwI+v6etnyrC^~Rp@~P*`t`RVo$yccq z+`<6ciyS4f=3@Yb0DOyXu9yMA6sV5XUX-Q*CWSDqHSqu14n$O_GFEXN9J$H?6G1?{ zhtd-LE8*fD(o%G3jR~*X(Yl6)-9|vUE*c<1-5o_U{pY-6BOVDa6@Mr5o?OQ37!ydMFb!R|F)p#4)_69exAlIgLF$Bq3i< z7BY|@oe|ivs6(nM^$O)3ne}9uYSZ1^-dd4}hv=f!@fHNp27Hxi0Zsr#0pS{&uqwQsBBG_iU#=Mq zL$x^%s6Q2WdNpPO@e8lVW3R?a_E+lbhI5DbwK zqB4T?z+Y6NNEwp=LiUobmX%5*Oq2BmEt+O~z_kIKLEWSLwv*fD0E}Z&zYQF9EuU9< zlkhZcneAy=tDxRWnM`5~js>J$?N>d7X&9H(0#4C)KtEAg8IXEX`Pnf$7*qZ{xNdPw zdA1%2tI+e5ux9Vss@H)Jyvs%{l)((l3B6}0n4Fg}uB@_#X490cW`PfBY6D}Mh@K`- zN-*-tJCD_fSJR;X5ot+>BA~<(nC6(tI2){%CNbV5H`gWI- z{nbMjkglZ4j*~pvYo)S*L@iDgPD>X6l<^M%l+&Lad18t3GZdrH^`g(?jlR`^IRf>$ z!Ke(ZMGS8OygX9sB}DVHC?hc`$xh89{v9H}w!jX3^YYuXjDzt3|5tVqdm~m=>x4-K z+QpGomTJ;N!JpC90K2;-5~8Bs8Pp%9WXZ;;p;+NT zL6>MKO|j82aWmp1J8pQ z;H;l=J~v)sjZE7HniX%oq+LfPFwH+`;Y}Sq1Hy9m@;M;%&rMVc`JK-iI4BkTva#^& z=BN*}GBq_7*zig}`JH>C@nA&XFA9dP8>|^$`k>kuG7X;|hDTU;R=xmdVhFM)vk}ra z_4XDusCB!4cGL+rYP&hNzL9vN*m6m%FJ_v}qd`5tX4lukVo~G4hhn7CQg5&FL*J?? zk&`}K%tfwSgwU;Qtsg8PB#E5;)v@=0%z-wKuhXd{ZN1Vz6;&K)Wt3#2?&x1^kgSUX zmkYJhcXaWSKQ`3V=%i^Y(5EZCX76kabV6 zX20EU%{#7{ZL`Xu`Jh;M!>GfvD${-3zwJDyvbfr>sjPWD=cwnRf75PAYcS8LH+3k?a^5z$b!|vHgLBJJVBdk?8MF2;o2@9 zsU|-29AW}@OKFlAq1QKJFpPZDhw!Q?7Oet}7~ImeiQtJ8qUoeg&dhWranCFa3JKCb zcYSE|V_0!v;jG>F?8IyBc@5>Di^Mg(^%@Ea3OVF_dzdG2CmUn&`q<7%M%;yA{j17W zPuVl@1%2{}c(e&CEO zYh0R0a-FQ*C=BV+1M&wLYcxX+4Lq-=do#{aI!569;&e-`6KUMRcP7Rd_5o>5?)KnS zaVIiD=SgqJyY1=lpdLsAi``vSedR439q#s3CD$o82{m0xo;+~V^JmXJ%s>1Bx~X>A zo(2j#Gv(H5T>Wq{mszQ`v*W$HFE!}=cVp70o<07dVaCN+2c*YbCqoGzv$Y#w_Ljii zq(*AVoe;He#596a7t8MHrv_XMFYOpB2N@YFJURvV8utggD-P3uAU^ZJ__Sj9B=f~- zmah?Tm^{a`V|M8zmGf2JK|VR1Z8=#OILmj`Dk5o>{k4r;q0NZw4q(PlM?9z)8UxA}V18bSgwybHCG# zN|Pg-qT0vjjk|jmTqhoihGpcode{padding:0;color:inherit;background-color:inherit}kbd{padding:.2rem .4rem;font-size:90%;color:#fff;background-color:#292b2c;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;margin-top:0;margin-bottom:1rem;font-size:90%;color:#292b2c}pre code{padding:0;font-size:inherit;color:inherit;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{position:relative;margin-left:auto;margin-right:auto;padding-right:15px;padding-left:15px}@media (min-width:576px){.container{padding-right:15px;padding-left:15px}}@media (min-width:768px){.container{padding-right:15px;padding-left:15px}}@media (min-width:992px){.container{padding-right:15px;padding-left:15px}}@media (min-width:1200px){.container{padding-right:15px;padding-left:15px}}@media (min-width:576px){.container{width:540px;max-width:100%}}@media (min-width:768px){.container{width:720px;max-width:100%}}@media (min-width:992px){.container{width:960px;max-width:100%}}@media (min-width:1200px){.container{width:1140px;max-width:100%}}.container-fluid{position:relative;margin-left:auto;margin-right:auto;padding-right:15px;padding-left:15px}@media (min-width:576px){.container-fluid{padding-right:15px;padding-left:15px}}@media (min-width:768px){.container-fluid{padding-right:15px;padding-left:15px}}@media (min-width:992px){.container-fluid{padding-right:15px;padding-left:15px}}@media (min-width:1200px){.container-fluid{padding-right:15px;padding-left:15px}}.row{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:wrap;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}@media (min-width:576px){.row{margin-right:-15px;margin-left:-15px}}@media (min-width:768px){.row{margin-right:-15px;margin-left:-15px}}@media (min-width:992px){.row{margin-right:-15px;margin-left:-15px}}@media (min-width:1200px){.row{margin-right:-15px;margin-left:-15px}}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{position:relative;width:100%;min-height:1px;padding-right:15px;padding-left:15px}@media (min-width:576px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}@media (min-width:768px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}@media (min-width:992px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}@media (min-width:1200px){.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9{padding-right:15px;padding-left:15px}}.col{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-0{right:auto}.pull-1{right:8.333333%}.pull-2{right:16.666667%}.pull-3{right:25%}.pull-4{right:33.333333%}.pull-5{right:41.666667%}.pull-6{right:50%}.pull-7{right:58.333333%}.pull-8{right:66.666667%}.pull-9{right:75%}.pull-10{right:83.333333%}.pull-11{right:91.666667%}.pull-12{right:100%}.push-0{left:auto}.push-1{left:8.333333%}.push-2{left:16.666667%}.push-3{left:25%}.push-4{left:33.333333%}.push-5{left:41.666667%}.push-6{left:50%}.push-7{left:58.333333%}.push-8{left:66.666667%}.push-9{left:75%}.push-10{left:83.333333%}.push-11{left:91.666667%}.push-12{left:100%}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-sm-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-sm-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-sm-0{right:auto}.pull-sm-1{right:8.333333%}.pull-sm-2{right:16.666667%}.pull-sm-3{right:25%}.pull-sm-4{right:33.333333%}.pull-sm-5{right:41.666667%}.pull-sm-6{right:50%}.pull-sm-7{right:58.333333%}.pull-sm-8{right:66.666667%}.pull-sm-9{right:75%}.pull-sm-10{right:83.333333%}.pull-sm-11{right:91.666667%}.pull-sm-12{right:100%}.push-sm-0{left:auto}.push-sm-1{left:8.333333%}.push-sm-2{left:16.666667%}.push-sm-3{left:25%}.push-sm-4{left:33.333333%}.push-sm-5{left:41.666667%}.push-sm-6{left:50%}.push-sm-7{left:58.333333%}.push-sm-8{left:66.666667%}.push-sm-9{left:75%}.push-sm-10{left:83.333333%}.push-sm-11{left:91.666667%}.push-sm-12{left:100%}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-md-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-md-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-md-0{right:auto}.pull-md-1{right:8.333333%}.pull-md-2{right:16.666667%}.pull-md-3{right:25%}.pull-md-4{right:33.333333%}.pull-md-5{right:41.666667%}.pull-md-6{right:50%}.pull-md-7{right:58.333333%}.pull-md-8{right:66.666667%}.pull-md-9{right:75%}.pull-md-10{right:83.333333%}.pull-md-11{right:91.666667%}.pull-md-12{right:100%}.push-md-0{left:auto}.push-md-1{left:8.333333%}.push-md-2{left:16.666667%}.push-md-3{left:25%}.push-md-4{left:33.333333%}.push-md-5{left:41.666667%}.push-md-6{left:50%}.push-md-7{left:58.333333%}.push-md-8{left:66.666667%}.push-md-9{left:75%}.push-md-10{left:83.333333%}.push-md-11{left:91.666667%}.push-md-12{left:100%}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-lg-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-lg-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-lg-0{right:auto}.pull-lg-1{right:8.333333%}.pull-lg-2{right:16.666667%}.pull-lg-3{right:25%}.pull-lg-4{right:33.333333%}.pull-lg-5{right:41.666667%}.pull-lg-6{right:50%}.pull-lg-7{right:58.333333%}.pull-lg-8{right:66.666667%}.pull-lg-9{right:75%}.pull-lg-10{right:83.333333%}.pull-lg-11{right:91.666667%}.pull-lg-12{right:100%}.push-lg-0{left:auto}.push-lg-1{left:8.333333%}.push-lg-2{left:16.666667%}.push-lg-3{left:25%}.push-lg-4{left:33.333333%}.push-lg-5{left:41.666667%}.push-lg-6{left:50%}.push-lg-7{left:58.333333%}.push-lg-8{left:66.666667%}.push-lg-9{left:75%}.push-lg-10{left:83.333333%}.push-lg-11{left:91.666667%}.push-lg-12{left:100%}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-webkit-flex-basis:0;-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-xl-auto{-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;width:auto}.col-xl-1{-webkit-box-flex:0;-webkit-flex:0 0 8.333333%;-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-webkit-box-flex:0;-webkit-flex:0 0 16.666667%;-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-webkit-box-flex:0;-webkit-flex:0 0 25%;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-webkit-box-flex:0;-webkit-flex:0 0 33.333333%;-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-webkit-box-flex:0;-webkit-flex:0 0 41.666667%;-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-webkit-box-flex:0;-webkit-flex:0 0 50%;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-webkit-box-flex:0;-webkit-flex:0 0 58.333333%;-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-webkit-box-flex:0;-webkit-flex:0 0 66.666667%;-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-webkit-box-flex:0;-webkit-flex:0 0 75%;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-webkit-box-flex:0;-webkit-flex:0 0 83.333333%;-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-webkit-box-flex:0;-webkit-flex:0 0 91.666667%;-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-webkit-box-flex:0;-webkit-flex:0 0 100%;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.pull-xl-0{right:auto}.pull-xl-1{right:8.333333%}.pull-xl-2{right:16.666667%}.pull-xl-3{right:25%}.pull-xl-4{right:33.333333%}.pull-xl-5{right:41.666667%}.pull-xl-6{right:50%}.pull-xl-7{right:58.333333%}.pull-xl-8{right:66.666667%}.pull-xl-9{right:75%}.pull-xl-10{right:83.333333%}.pull-xl-11{right:91.666667%}.pull-xl-12{right:100%}.push-xl-0{left:auto}.push-xl-1{left:8.333333%}.push-xl-2{left:16.666667%}.push-xl-3{left:25%}.push-xl-4{left:33.333333%}.push-xl-5{left:41.666667%}.push-xl-6{left:50%}.push-xl-7{left:58.333333%}.push-xl-8{left:66.666667%}.push-xl-9{left:75%}.push-xl-10{left:83.333333%}.push-xl-11{left:91.666667%}.push-xl-12{left:100%}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;max-width:100%;margin-bottom:1rem}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #eceeef}.table thead th{vertical-align:bottom;border-bottom:2px solid #eceeef}.table tbody+tbody{border-top:2px solid #eceeef}.table .table{background-color:#fff}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #eceeef}.table-bordered td,.table-bordered th{border:1px solid #eceeef}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{background-color:rgba(0,0,0,.075)}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table-success,.table-success>td,.table-success>th{background-color:#dff0d8}.table-hover .table-success:hover{background-color:#d0e9c6}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#d0e9c6}.table-info,.table-info>td,.table-info>th{background-color:#d9edf7}.table-hover .table-info:hover{background-color:#c4e3f3}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#c4e3f3}.table-warning,.table-warning>td,.table-warning>th{background-color:#fcf8e3}.table-hover .table-warning:hover{background-color:#faf2cc}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#faf2cc}.table-danger,.table-danger>td,.table-danger>th{background-color:#f2dede}.table-hover .table-danger:hover{background-color:#ebcccc}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#ebcccc}.thead-inverse th{color:#fff;background-color:#292b2c}.thead-default th{color:#464a4c;background-color:#eceeef}.table-inverse{color:#fff;background-color:#292b2c}.table-inverse td,.table-inverse th,.table-inverse thead th{border-color:#fff}.table-inverse.table-bordered{border:0}.table-responsive{display:block;width:100%;overflow-x:auto;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive.table-bordered{border:0}.form-control{display:block;width:100%;padding:.5rem .75rem;font-size:1rem;line-height:1.25;color:#464a4c;background-color:#fff;background-image:none;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem;-webkit-transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,-webkit-box-shadow ease-in-out .15s;-o-transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s,-webkit-box-shadow ease-in-out .15s}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:focus{color:#464a4c;background-color:#fff;border-color:#5cb3fd;outline:0}.form-control::-webkit-input-placeholder{color:#636c72;opacity:1}.form-control::-moz-placeholder{color:#636c72;opacity:1}.form-control:-ms-input-placeholder{color:#636c72;opacity:1}.form-control::placeholder{color:#636c72;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#eceeef;opacity:1}.form-control:disabled{cursor:not-allowed}select.form-control:not([size]):not([multiple]){height:calc(2.25rem + 2px)}select.form-control:focus::-ms-value{color:#464a4c;background-color:#fff}.form-control-file,.form-control-range{display:block}.col-form-label{padding-top:calc(.5rem - 1px * 2);padding-bottom:calc(.5rem - 1px * 2);margin-bottom:0}.col-form-label-lg{padding-top:calc(.75rem - 1px * 2);padding-bottom:calc(.75rem - 1px * 2);font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem - 1px * 2);padding-bottom:calc(.25rem - 1px * 2);font-size:.875rem}.col-form-legend{padding-top:.5rem;padding-bottom:.5rem;margin-bottom:0;font-size:1rem}.form-control-static{padding-top:.5rem;padding-bottom:.5rem;margin-bottom:0;line-height:1.25;border:solid transparent;border-width:1px 0}.form-control-static.form-control-lg,.form-control-static.form-control-sm,.input-group-lg>.form-control-static.form-control,.input-group-lg>.form-control-static.input-group-addon,.input-group-lg>.input-group-btn>.form-control-static.btn,.input-group-sm>.form-control-static.form-control,.input-group-sm>.form-control-static.input-group-addon,.input-group-sm>.input-group-btn>.form-control-static.btn{padding-right:0;padding-left:0}.form-control-sm,.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-sm>.input-group-btn>select.btn:not([size]):not([multiple]),.input-group-sm>select.form-control:not([size]):not([multiple]),.input-group-sm>select.input-group-addon:not([size]):not([multiple]),select.form-control-sm:not([size]):not([multiple]){height:1.8125rem}.form-control-lg,.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{padding:.75rem 1.5rem;font-size:1.25rem;border-radius:.3rem}.input-group-lg>.input-group-btn>select.btn:not([size]):not([multiple]),.input-group-lg>select.form-control:not([size]):not([multiple]),.input-group-lg>select.input-group-addon:not([size]):not([multiple]),select.form-control-lg:not([size]):not([multiple]){height:3.166667rem}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-check{position:relative;display:block;margin-bottom:.5rem}.form-check.disabled .form-check-label{color:#636c72;cursor:not-allowed}.form-check-label{padding-left:1.25rem;margin-bottom:0;cursor:pointer}.form-check-input{position:absolute;margin-top:.25rem;margin-left:-1.25rem}.form-check-input:only-child{position:static}.form-check-inline{display:inline-block}.form-check-inline .form-check-label{vertical-align:middle}.form-check-inline+.form-check-inline{margin-left:.75rem}.form-control-feedback{margin-top:.25rem}.form-control-danger,.form-control-success,.form-control-warning{padding-right:2.25rem;background-repeat:no-repeat;background-position:center right .5625rem;-webkit-background-size:1.125rem 1.125rem;background-size:1.125rem 1.125rem}.has-success .col-form-label,.has-success .custom-control,.has-success .form-check-label,.has-success .form-control-feedback,.has-success .form-control-label{color:#5cb85c}.has-success .form-control{border-color:#5cb85c}.has-success .input-group-addon{color:#5cb85c;border-color:#5cb85c;background-color:#eaf6ea}.has-success .form-control-success{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%235cb85c' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3E%3C/svg%3E")}.has-warning .col-form-label,.has-warning .custom-control,.has-warning .form-check-label,.has-warning .form-control-feedback,.has-warning .form-control-label{color:#f0ad4e}.has-warning .form-control{border-color:#f0ad4e}.has-warning .input-group-addon{color:#f0ad4e;border-color:#f0ad4e;background-color:#fff}.has-warning .form-control-warning{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%23f0ad4e' d='M4.4 5.324h-.8v-2.46h.8zm0 1.42h-.8V5.89h.8zM3.76.63L.04 7.075c-.115.2.016.425.26.426h7.397c.242 0 .372-.226.258-.426C6.726 4.924 5.47 2.79 4.253.63c-.113-.174-.39-.174-.494 0z'/%3E%3C/svg%3E")}.has-danger .col-form-label,.has-danger .custom-control,.has-danger .form-check-label,.has-danger .form-control-feedback,.has-danger .form-control-label{color:#d9534f}.has-danger .form-control{border-color:#d9534f}.has-danger .input-group-addon{color:#d9534f;border-color:#d9534f;background-color:#fdf7f7}.has-danger .form-control-danger{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23d9534f' viewBox='-2 -2 7 7'%3E%3Cpath stroke='%23d9534f' d='M0 0l3 3m0-3L0 3'/%3E%3Ccircle r='.5'/%3E%3Ccircle cx='3' r='.5'/%3E%3Ccircle cy='3' r='.5'/%3E%3Ccircle cx='3' cy='3' r='.5'/%3E%3C/svg%3E")}.form-inline{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:0 0 auto;-ms-flex:0 0 auto;flex:0 0 auto;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-static{display:inline-block}.form-inline .input-group{width:auto}.form-inline .form-control-label{margin-bottom:0;vertical-align:middle}.form-inline .form-check{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;width:auto;margin-top:0;margin-bottom:0}.form-inline .form-check-label{padding-left:0}.form-inline .form-check-input{position:relative;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;padding-left:0}.form-inline .custom-control-indicator{position:static;display:inline-block;margin-right:.25rem;vertical-align:text-bottom}.form-inline .has-feedback .form-control-feedback{top:0}}.btn{display:inline-block;font-weight:400;line-height:1.25;text-align:center;white-space:nowrap;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;border:1px solid transparent;padding:.5rem 1rem;font-size:1rem;border-radius:.25rem;-webkit-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}.btn:focus,.btn:hover{text-decoration:none}.btn.focus,.btn:focus{outline:0;-webkit-box-shadow:0 0 0 2px rgba(2,117,216,.25);box-shadow:0 0 0 2px rgba(2,117,216,.25)}.btn.disabled,.btn:disabled{cursor:not-allowed;opacity:.65}.btn.active,.btn:active{background-image:none}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#0275d8;border-color:#0275d8}.btn-primary:hover{color:#fff;background-color:#025aa5;border-color:#01549b}.btn-primary.focus,.btn-primary:focus{-webkit-box-shadow:0 0 0 2px rgba(2,117,216,.5);box-shadow:0 0 0 2px rgba(2,117,216,.5)}.btn-primary.disabled,.btn-primary:disabled{background-color:#0275d8;border-color:#0275d8}.btn-primary.active,.btn-primary:active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#025aa5;background-image:none;border-color:#01549b}.btn-secondary{color:#292b2c;background-color:#fff;border-color:#ccc}.btn-secondary:hover{color:#292b2c;background-color:#e6e6e6;border-color:#adadad}.btn-secondary.focus,.btn-secondary:focus{-webkit-box-shadow:0 0 0 2px rgba(204,204,204,.5);box-shadow:0 0 0 2px rgba(204,204,204,.5)}.btn-secondary.disabled,.btn-secondary:disabled{background-color:#fff;border-color:#ccc}.btn-secondary.active,.btn-secondary:active,.show>.btn-secondary.dropdown-toggle{color:#292b2c;background-color:#e6e6e6;background-image:none;border-color:#adadad}.btn-info{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#2aabd2}.btn-info.focus,.btn-info:focus{-webkit-box-shadow:0 0 0 2px rgba(91,192,222,.5);box-shadow:0 0 0 2px rgba(91,192,222,.5)}.btn-info.disabled,.btn-info:disabled{background-color:#5bc0de;border-color:#5bc0de}.btn-info.active,.btn-info:active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#31b0d5;background-image:none;border-color:#2aabd2}.btn-success{color:#fff;background-color:#5cb85c;border-color:#5cb85c}.btn-success:hover{color:#fff;background-color:#449d44;border-color:#419641}.btn-success.focus,.btn-success:focus{-webkit-box-shadow:0 0 0 2px rgba(92,184,92,.5);box-shadow:0 0 0 2px rgba(92,184,92,.5)}.btn-success.disabled,.btn-success:disabled{background-color:#5cb85c;border-color:#5cb85c}.btn-success.active,.btn-success:active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#449d44;background-image:none;border-color:#419641}.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#f0ad4e}.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#eb9316}.btn-warning.focus,.btn-warning:focus{-webkit-box-shadow:0 0 0 2px rgba(240,173,78,.5);box-shadow:0 0 0 2px rgba(240,173,78,.5)}.btn-warning.disabled,.btn-warning:disabled{background-color:#f0ad4e;border-color:#f0ad4e}.btn-warning.active,.btn-warning:active,.show>.btn-warning.dropdown-toggle{color:#fff;background-color:#ec971f;background-image:none;border-color:#eb9316}.btn-danger{color:#fff;background-color:#d9534f;border-color:#d9534f}.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#c12e2a}.btn-danger.focus,.btn-danger:focus{-webkit-box-shadow:0 0 0 2px rgba(217,83,79,.5);box-shadow:0 0 0 2px rgba(217,83,79,.5)}.btn-danger.disabled,.btn-danger:disabled{background-color:#d9534f;border-color:#d9534f}.btn-danger.active,.btn-danger:active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#c9302c;background-image:none;border-color:#c12e2a}.btn-outline-primary{color:#0275d8;background-image:none;background-color:transparent;border-color:#0275d8}.btn-outline-primary:hover{color:#fff;background-color:#0275d8;border-color:#0275d8}.btn-outline-primary.focus,.btn-outline-primary:focus{-webkit-box-shadow:0 0 0 2px rgba(2,117,216,.5);box-shadow:0 0 0 2px rgba(2,117,216,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#0275d8;background-color:transparent}.btn-outline-primary.active,.btn-outline-primary:active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#0275d8;border-color:#0275d8}.btn-outline-secondary{color:#ccc;background-image:none;background-color:transparent;border-color:#ccc}.btn-outline-secondary:hover{color:#fff;background-color:#ccc;border-color:#ccc}.btn-outline-secondary.focus,.btn-outline-secondary:focus{-webkit-box-shadow:0 0 0 2px rgba(204,204,204,.5);box-shadow:0 0 0 2px rgba(204,204,204,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#ccc;background-color:transparent}.btn-outline-secondary.active,.btn-outline-secondary:active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#ccc;border-color:#ccc}.btn-outline-info{color:#5bc0de;background-image:none;background-color:transparent;border-color:#5bc0de}.btn-outline-info:hover{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.btn-outline-info.focus,.btn-outline-info:focus{-webkit-box-shadow:0 0 0 2px rgba(91,192,222,.5);box-shadow:0 0 0 2px rgba(91,192,222,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#5bc0de;background-color:transparent}.btn-outline-info.active,.btn-outline-info:active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#5bc0de;border-color:#5bc0de}.btn-outline-success{color:#5cb85c;background-image:none;background-color:transparent;border-color:#5cb85c}.btn-outline-success:hover{color:#fff;background-color:#5cb85c;border-color:#5cb85c}.btn-outline-success.focus,.btn-outline-success:focus{-webkit-box-shadow:0 0 0 2px rgba(92,184,92,.5);box-shadow:0 0 0 2px rgba(92,184,92,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#5cb85c;background-color:transparent}.btn-outline-success.active,.btn-outline-success:active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#5cb85c;border-color:#5cb85c}.btn-outline-warning{color:#f0ad4e;background-image:none;background-color:transparent;border-color:#f0ad4e}.btn-outline-warning:hover{color:#fff;background-color:#f0ad4e;border-color:#f0ad4e}.btn-outline-warning.focus,.btn-outline-warning:focus{-webkit-box-shadow:0 0 0 2px rgba(240,173,78,.5);box-shadow:0 0 0 2px rgba(240,173,78,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#f0ad4e;background-color:transparent}.btn-outline-warning.active,.btn-outline-warning:active,.show>.btn-outline-warning.dropdown-toggle{color:#fff;background-color:#f0ad4e;border-color:#f0ad4e}.btn-outline-danger{color:#d9534f;background-image:none;background-color:transparent;border-color:#d9534f}.btn-outline-danger:hover{color:#fff;background-color:#d9534f;border-color:#d9534f}.btn-outline-danger.focus,.btn-outline-danger:focus{-webkit-box-shadow:0 0 0 2px rgba(217,83,79,.5);box-shadow:0 0 0 2px rgba(217,83,79,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#d9534f;background-color:transparent}.btn-outline-danger.active,.btn-outline-danger:active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#d9534f;border-color:#d9534f}.btn-link{font-weight:400;color:#0275d8;border-radius:0}.btn-link,.btn-link.active,.btn-link:active,.btn-link:disabled{background-color:transparent}.btn-link,.btn-link:active,.btn-link:focus{border-color:transparent}.btn-link:hover{border-color:transparent}.btn-link:focus,.btn-link:hover{color:#014c8c;text-decoration:underline;background-color:transparent}.btn-link:disabled{color:#636c72}.btn-link:disabled:focus,.btn-link:disabled:hover{text-decoration:none}.btn-group-lg>.btn,.btn-lg{padding:.75rem 1.5rem;font-size:1.25rem;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.show{opacity:1}.collapse{display:none}.collapse.show{display:block}tr.collapse.show{display:table-row}tbody.collapse.show{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.dropdown,.dropup{position:relative}.dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:.3em;vertical-align:middle;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-left:.3em solid transparent}.dropdown-toggle:focus{outline:0}.dropup .dropdown-toggle::after{border-top:0;border-bottom:.3em solid}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#292b2c;text-align:left;list-style:none;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-divider{height:1px;margin:.5rem 0;overflow:hidden;background-color:#eceeef}.dropdown-item{display:block;width:100%;padding:3px 1.5rem;clear:both;font-weight:400;color:#292b2c;text-align:inherit;white-space:nowrap;background:0 0;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#1d1e1f;text-decoration:none;background-color:#f7f7f9}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#0275d8}.dropdown-item.disabled,.dropdown-item:disabled{color:#636c72;cursor:not-allowed;background-color:transparent}.show>.dropdown-menu{display:block}.show>a{outline:0}.dropdown-menu-right{right:0;left:auto}.dropdown-menu-left{right:auto;left:0}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#636c72;white-space:nowrap}.dropdown-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:990}.dropup .dropdown-menu{top:auto;bottom:100%;margin-bottom:.125rem}.btn-group,.btn-group-vertical{position:relative;display:-webkit-inline-box;display:-webkit-inline-flex;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-webkit-box-flex:0;-webkit-flex:0 1 auto;-ms-flex:0 1 auto;flex:0 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:2}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group,.btn-group-vertical .btn+.btn,.btn-group-vertical .btn+.btn-group,.btn-group-vertical .btn-group+.btn,.btn-group-vertical .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-bottom-left-radius:0;border-top-left-radius:0}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn+.dropdown-toggle-split::after{margin-left:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:1.125rem;padding-left:1.125rem}.btn-group-vertical{display:-webkit-inline-box;display:-webkit-inline-flex;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-box-align:start;-webkit-align-items:flex-start;-ms-flex-align:start;align-items:flex-start;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center}.btn-group-vertical .btn,.btn-group-vertical .btn-group{width:100%}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-right-radius:0;border-top-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-right-radius:0;border-top-left-radius:0}[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;width:100%}.input-group .form-control{position:relative;z-index:2;-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;margin-bottom:0}.input-group .form-control:active,.input-group .form-control:focus,.input-group .form-control:hover{z-index:3}.input-group .form-control,.input-group-addon,.input-group-btn{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center}.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{white-space:nowrap;vertical-align:middle}.input-group-addon{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.25;color:#464a4c;text-align:center;background-color:#eceeef;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.input-group-addon.form-control-sm,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.input-group-addon.btn{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-addon.form-control-lg,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.input-group-addon.btn{padding:.75rem 1.5rem;font-size:1.25rem;border-radius:.3rem}.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0}.input-group .form-control:not(:last-child),.input-group-addon:not(:last-child),.input-group-btn:not(:first-child)>.btn-group:not(:last-child)>.btn,.input-group-btn:not(:first-child)>.btn:not(:last-child):not(.dropdown-toggle),.input-group-btn:not(:last-child)>.btn,.input-group-btn:not(:last-child)>.btn-group>.btn,.input-group-btn:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-top-right-radius:0}.input-group-addon:not(:last-child){border-right:0}.input-group .form-control:not(:first-child),.input-group-addon:not(:first-child),.input-group-btn:not(:first-child)>.btn,.input-group-btn:not(:first-child)>.btn-group>.btn,.input-group-btn:not(:first-child)>.dropdown-toggle,.input-group-btn:not(:last-child)>.btn-group:not(:first-child)>.btn,.input-group-btn:not(:last-child)>.btn:not(:first-child){border-bottom-left-radius:0;border-top-left-radius:0}.form-control+.input-group-addon:not(:first-child){border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative;-webkit-box-flex:1;-webkit-flex:1 1 0%;-ms-flex:1 1 0%;flex:1 1 0%}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:3}.input-group-btn:not(:last-child)>.btn,.input-group-btn:not(:last-child)>.btn-group{margin-right:-1px}.input-group-btn:not(:first-child)>.btn,.input-group-btn:not(:first-child)>.btn-group{z-index:2;margin-left:-1px}.input-group-btn:not(:first-child)>.btn-group:active,.input-group-btn:not(:first-child)>.btn-group:focus,.input-group-btn:not(:first-child)>.btn-group:hover,.input-group-btn:not(:first-child)>.btn:active,.input-group-btn:not(:first-child)>.btn:focus,.input-group-btn:not(:first-child)>.btn:hover{z-index:3}.custom-control{position:relative;display:-webkit-inline-box;display:-webkit-inline-flex;display:-ms-inline-flexbox;display:inline-flex;min-height:1.5rem;padding-left:1.5rem;margin-right:1rem;cursor:pointer}.custom-control-input{position:absolute;z-index:-1;opacity:0}.custom-control-input:checked~.custom-control-indicator{color:#fff;background-color:#0275d8}.custom-control-input:focus~.custom-control-indicator{-webkit-box-shadow:0 0 0 1px #fff,0 0 0 3px #0275d8;box-shadow:0 0 0 1px #fff,0 0 0 3px #0275d8}.custom-control-input:active~.custom-control-indicator{color:#fff;background-color:#8fcafe}.custom-control-input:disabled~.custom-control-indicator{cursor:not-allowed;background-color:#eceeef}.custom-control-input:disabled~.custom-control-description{color:#636c72;cursor:not-allowed}.custom-control-indicator{position:absolute;top:.25rem;left:0;display:block;width:1rem;height:1rem;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#ddd;background-repeat:no-repeat;background-position:center center;-webkit-background-size:50% 50%;background-size:50% 50%}.custom-checkbox .custom-control-indicator{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-indicator{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26 2.974 7.25 8 2.193z'/%3E%3C/svg%3E")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-indicator{background-color:#0275d8;background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 4'%3E%3Cpath stroke='%23fff' d='M0 2h4'/%3E%3C/svg%3E")}.custom-radio .custom-control-indicator{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-indicator{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3E%3Ccircle r='3' fill='%23fff'/%3E%3C/svg%3E")}.custom-controls-stacked{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column}.custom-controls-stacked .custom-control{margin-bottom:.25rem}.custom-controls-stacked .custom-control+.custom-control{margin-left:0}.custom-select{display:inline-block;max-width:100%;height:calc(2.25rem + 2px);padding:.375rem 1.75rem .375rem .75rem;line-height:1.25;color:#464a4c;vertical-align:middle;background:#fff url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3E%3Cpath fill='%23333' d='M2 0L0 2h4zm0 5L0 3h4z'/%3E%3C/svg%3E") no-repeat right .75rem center;-webkit-background-size:8px 10px;background-size:8px 10px;border:1px solid rgba(0,0,0,.15);border-radius:.25rem;-moz-appearance:none;-webkit-appearance:none}.custom-select:focus{border-color:#5cb3fd;outline:0}.custom-select:focus::-ms-value{color:#464a4c;background-color:#fff}.custom-select:disabled{color:#636c72;cursor:not-allowed;background-color:#eceeef}.custom-select::-ms-expand{opacity:0}.custom-select-sm{padding-top:.375rem;padding-bottom:.375rem;font-size:75%}.custom-file{position:relative;display:inline-block;max-width:100%;height:2.5rem;margin-bottom:0;cursor:pointer}.custom-file-input{min-width:14rem;max-width:100%;height:2.5rem;margin:0;filter:alpha(opacity=0);opacity:0}.custom-file-control{position:absolute;top:0;right:0;left:0;z-index:5;height:2.5rem;padding:.5rem 1rem;line-height:1.5;color:#464a4c;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#fff;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.custom-file-control:lang(en)::after{content:"Choose file..."}.custom-file-control::before{position:absolute;top:-1px;right:-1px;bottom:-1px;z-index:6;display:block;height:2.5rem;padding:.5rem 1rem;line-height:1.5;color:#464a4c;background-color:#eceeef;border:1px solid rgba(0,0,0,.15);border-radius:0 .25rem .25rem 0}.custom-file-control:lang(en)::before{content:"Browse"}.nav{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5em 1em}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#636c72;cursor:not-allowed}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-right-radius:.25rem;border-top-left-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#eceeef #eceeef #ddd}.nav-tabs .nav-link.disabled{color:#636c72;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#464a4c;background-color:#fff;border-color:#ddd #ddd #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-right-radius:0;border-top-left-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-item.show .nav-link,.nav-pills .nav-link.active{color:#fff;cursor:default;background-color:#0275d8}.nav-fill .nav-item{-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-webkit-box-flex:1;-webkit-flex:1 1 100%;-ms-flex:1 1 100%;flex:1 1 100%;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;padding:.5rem 1rem}.navbar-brand{display:inline-block;padding-top:.25rem;padding-bottom:.25rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-text{display:inline-block;padding-top:.425rem;padding-bottom:.425rem}.navbar-toggler{-webkit-align-self:flex-start;-ms-flex-item-align:start;align-self:flex-start;padding:.25rem .75rem;font-size:1.25rem;line-height:1;background:0 0;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;-webkit-background-size:100% 100%;background-size:100% 100%}.navbar-toggler-left{position:absolute;left:1rem}.navbar-toggler-right{position:absolute;right:1rem}@media (max-width:575px){.navbar-toggleable .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable>.container{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-toggleable{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable .navbar-toggler{display:none}}@media (max-width:767px){.navbar-toggleable-sm .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-sm>.container{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-toggleable-sm{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-sm .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-sm>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-sm .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-sm .navbar-toggler{display:none}}@media (max-width:991px){.navbar-toggleable-md .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-md>.container{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-toggleable-md{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-md .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-md>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-md .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-md .navbar-toggler{display:none}}@media (max-width:1199px){.navbar-toggleable-lg .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-lg>.container{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-toggleable-lg{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-lg .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-lg>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-lg .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-lg .navbar-toggler{display:none}}.navbar-toggleable-xl{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-xl .navbar-nav .dropdown-menu{position:static;float:none}.navbar-toggleable-xl>.container{padding-right:0;padding-left:0}.navbar-toggleable-xl .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row}.navbar-toggleable-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-toggleable-xl>.container{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center}.navbar-toggleable-xl .navbar-collapse{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important;width:100%}.navbar-toggleable-xl .navbar-toggler{display:none}.navbar-light .navbar-brand,.navbar-light .navbar-toggler{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover,.navbar-light .navbar-toggler:focus,.navbar-light .navbar-toggler:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.open,.navbar-light .navbar-nav .open>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 32 32' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(0, 0, 0, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 8h24M4 16h24M4 24h24'/%3E%3C/svg%3E")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-inverse .navbar-brand,.navbar-inverse .navbar-toggler{color:#fff}.navbar-inverse .navbar-brand:focus,.navbar-inverse .navbar-brand:hover,.navbar-inverse .navbar-toggler:focus,.navbar-inverse .navbar-toggler:hover{color:#fff}.navbar-inverse .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-inverse .navbar-nav .nav-link:focus,.navbar-inverse .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-inverse .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-inverse .navbar-nav .active>.nav-link,.navbar-inverse .navbar-nav .nav-link.active,.navbar-inverse .navbar-nav .nav-link.open,.navbar-inverse .navbar-nav .open>.nav-link{color:#fff}.navbar-inverse .navbar-toggler{border-color:rgba(255,255,255,.1)}.navbar-inverse .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 32 32' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(255, 255, 255, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 8h24M4 16h24M4 24h24'/%3E%3C/svg%3E")}.navbar-inverse .navbar-text{color:rgba(255,255,255,.5)}.card{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;background-color:#fff;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card-block{-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card>.list-group:first-child .list-group-item:first-child{border-top-right-radius:.25rem;border-top-left-radius:.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:#f7f7f9;border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-footer{padding:.75rem 1.25rem;background-color:#f7f7f9;border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-primary{background-color:#0275d8;border-color:#0275d8}.card-primary .card-footer,.card-primary .card-header{background-color:transparent}.card-success{background-color:#5cb85c;border-color:#5cb85c}.card-success .card-footer,.card-success .card-header{background-color:transparent}.card-info{background-color:#5bc0de;border-color:#5bc0de}.card-info .card-footer,.card-info .card-header{background-color:transparent}.card-warning{background-color:#f0ad4e;border-color:#f0ad4e}.card-warning .card-footer,.card-warning .card-header{background-color:transparent}.card-danger{background-color:#d9534f;border-color:#d9534f}.card-danger .card-footer,.card-danger .card-header{background-color:transparent}.card-outline-primary{background-color:transparent;border-color:#0275d8}.card-outline-secondary{background-color:transparent;border-color:#ccc}.card-outline-info{background-color:transparent;border-color:#5bc0de}.card-outline-success{background-color:transparent;border-color:#5cb85c}.card-outline-warning{background-color:transparent;border-color:#f0ad4e}.card-outline-danger{background-color:transparent;border-color:#d9534f}.card-inverse{color:rgba(255,255,255,.65)}.card-inverse .card-footer,.card-inverse .card-header{background-color:transparent;border-color:rgba(255,255,255,.2)}.card-inverse .card-blockquote,.card-inverse .card-footer,.card-inverse .card-header,.card-inverse .card-title{color:#fff}.card-inverse .card-blockquote .blockquote-footer,.card-inverse .card-link,.card-inverse .card-subtitle,.card-inverse .card-text{color:rgba(255,255,255,.65)}.card-inverse .card-link:focus,.card-inverse .card-link:hover{color:#fff}.card-blockquote{padding:0;margin-bottom:0;border-left:0}.card-img{border-radius:calc(.25rem - 1px)}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img-top{border-top-right-radius:calc(.25rem - 1px);border-top-left-radius:calc(.25rem - 1px)}.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}@media (min-width:576px){.card-deck{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-deck .card{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-flex:1;-webkit-flex:1 0 0%;-ms-flex:1 0 0%;flex:1 0 0%;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column}.card-deck .card:not(:first-child){margin-left:15px}.card-deck .card:not(:last-child){margin-right:15px}}@media (min-width:576px){.card-group{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group .card{-webkit-box-flex:1;-webkit-flex:1 0 0%;-ms-flex:1 0 0%;flex:1 0 0%}.card-group .card+.card{margin-left:0;border-left:0}.card-group .card:first-child{border-bottom-right-radius:0;border-top-right-radius:0}.card-group .card:first-child .card-img-top{border-top-right-radius:0}.card-group .card:first-child .card-img-bottom{border-bottom-right-radius:0}.card-group .card:last-child{border-bottom-left-radius:0;border-top-left-radius:0}.card-group .card:last-child .card-img-top{border-top-left-radius:0}.card-group .card:last-child .card-img-bottom{border-bottom-left-radius:0}.card-group .card:not(:first-child):not(:last-child){border-radius:0}.card-group .card:not(:first-child):not(:last-child) .card-img-bottom,.card-group .card:not(:first-child):not(:last-child) .card-img-top{border-radius:0}}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem}.card-columns .card{display:inline-block;width:100%;margin-bottom:.75rem}}.breadcrumb{padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#eceeef;border-radius:.25rem}.breadcrumb::after{display:block;content:"";clear:both}.breadcrumb-item{float:left}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;padding-left:.5rem;color:#636c72;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#636c72}.pagination{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-item:first-child .page-link{margin-left:0;border-bottom-left-radius:.25rem;border-top-left-radius:.25rem}.page-item:last-child .page-link{border-bottom-right-radius:.25rem;border-top-right-radius:.25rem}.page-item.active .page-link{z-index:2;color:#fff;background-color:#0275d8;border-color:#0275d8}.page-item.disabled .page-link{color:#636c72;pointer-events:none;cursor:not-allowed;background-color:#fff;border-color:#ddd}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#0275d8;background-color:#fff;border:1px solid #ddd}.page-link:focus,.page-link:hover{color:#014c8c;text-decoration:none;background-color:#eceeef;border-color:#ddd}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem}.pagination-lg .page-item:first-child .page-link{border-bottom-left-radius:.3rem;border-top-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-bottom-right-radius:.3rem;border-top-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem}.pagination-sm .page-item:first-child .page-link{border-bottom-left-radius:.2rem;border-top-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-bottom-right-radius:.2rem;border-top-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}a.badge:focus,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-default{background-color:#636c72}.badge-default[href]:focus,.badge-default[href]:hover{background-color:#4b5257}.badge-primary{background-color:#0275d8}.badge-primary[href]:focus,.badge-primary[href]:hover{background-color:#025aa5}.badge-success{background-color:#5cb85c}.badge-success[href]:focus,.badge-success[href]:hover{background-color:#449d44}.badge-info{background-color:#5bc0de}.badge-info[href]:focus,.badge-info[href]:hover{background-color:#31b0d5}.badge-warning{background-color:#f0ad4e}.badge-warning[href]:focus,.badge-warning[href]:hover{background-color:#ec971f}.badge-danger{background-color:#d9534f}.badge-danger[href]:focus,.badge-danger[href]:hover{background-color:#c9302c}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#eceeef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-hr{border-top-color:#d0d5d8}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible .close{position:relative;top:-.75rem;right:-1.25rem;padding:.75rem 1.25rem;color:inherit}.alert-success{background-color:#dff0d8;border-color:#d0e9c6;color:#3c763d}.alert-success hr{border-top-color:#c1e2b3}.alert-success .alert-link{color:#2b542c}.alert-info{background-color:#d9edf7;border-color:#bcdff1;color:#31708f}.alert-info hr{border-top-color:#a6d5ec}.alert-info .alert-link{color:#245269}.alert-warning{background-color:#fcf8e3;border-color:#faf2cc;color:#8a6d3b}.alert-warning hr{border-top-color:#f7ecb5}.alert-warning .alert-link{color:#66512c}.alert-danger{background-color:#f2dede;border-color:#ebcccc;color:#a94442}.alert-danger hr{border-top-color:#e4b9b9}.alert-danger .alert-link{color:#843534}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;overflow:hidden;font-size:.75rem;line-height:1rem;text-align:center;background-color:#eceeef;border-radius:.25rem}.progress-bar{height:1rem;color:#fff;background-color:#0275d8}.progress-bar-striped{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);-webkit-background-size:1rem 1rem;background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;-o-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}.media{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:start;-webkit-align-items:flex-start;-ms-flex-align:start;align-items:flex-start}.media-body{-webkit-box-flex:1;-webkit-flex:1 1 0%;-ms-flex:1 1 0%;flex:1 1 0%}.list-group{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#464a4c;text-align:inherit}.list-group-item-action .list-group-item-heading{color:#292b2c}.list-group-item-action:focus,.list-group-item-action:hover{color:#464a4c;text-decoration:none;background-color:#f7f7f9}.list-group-item-action:active{color:#292b2c;background-color:#eceeef}.list-group-item{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-flow:row wrap;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;padding:.75rem 1.25rem;margin-bottom:-1px;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-right-radius:.25rem;border-top-left-radius:.25rem}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.list-group-item:focus,.list-group-item:hover{text-decoration:none}.list-group-item.disabled,.list-group-item:disabled{color:#636c72;cursor:not-allowed;background-color:#fff}.list-group-item.disabled .list-group-item-heading,.list-group-item:disabled .list-group-item-heading{color:inherit}.list-group-item.disabled .list-group-item-text,.list-group-item:disabled .list-group-item-text{color:#636c72}.list-group-item.active{z-index:2;color:#fff;background-color:#0275d8;border-color:#0275d8}.list-group-item.active .list-group-item-heading,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active .list-group-item-heading>small{color:inherit}.list-group-item.active .list-group-item-text{color:#daeeff}.list-group-flush .list-group-item{border-right:0;border-left:0;border-radius:0}.list-group-flush:first-child .list-group-item:first-child{border-top:0}.list-group-flush:last-child .list-group-item:last-child{border-bottom:0}.list-group-item-success{color:#3c763d;background-color:#dff0d8}a.list-group-item-success,button.list-group-item-success{color:#3c763d}a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit}a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#3c763d;background-color:#d0e9c6}a.list-group-item-success.active,button.list-group-item-success.active{color:#fff;background-color:#3c763d;border-color:#3c763d}.list-group-item-info{color:#31708f;background-color:#d9edf7}a.list-group-item-info,button.list-group-item-info{color:#31708f}a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit}a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#31708f;background-color:#c4e3f3}a.list-group-item-info.active,button.list-group-item-info.active{color:#fff;background-color:#31708f;border-color:#31708f}.list-group-item-warning{color:#8a6d3b;background-color:#fcf8e3}a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b}a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit}a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#8a6d3b;background-color:#faf2cc}a.list-group-item-warning.active,button.list-group-item-warning.active{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b}.list-group-item-danger{color:#a94442;background-color:#f2dede}a.list-group-item-danger,button.list-group-item-danger{color:#a94442}a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit}a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#a94442;background-color:#ebcccc}a.list-group-item-danger.active,button.list-group-item-danger.active{color:#fff;background-color:#a94442;border-color:#a94442}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:focus,.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.75}button.close{padding:0;cursor:pointer;background:0 0;border:0;-webkit-appearance:none}.modal-open{overflow:hidden}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;outline:0}.modal.fade .modal-dialog{-webkit-transition:-webkit-transform .3s ease-out;transition:-webkit-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out,-o-transform .3s ease-out;-webkit-transform:translate(0,-25%);-o-transform:translate(0,-25%);transform:translate(0,-25%)}.modal.show .modal-dialog{-webkit-transform:translate(0,0);-o-transform:translate(0,0);transform:translate(0,0)}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;padding:15px;border-bottom:1px solid #eceeef}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-webkit-box-flex:1;-webkit-flex:1 1 auto;-ms-flex:1 1 auto;flex:1 1 auto;padding:15px}.modal-footer{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:end;-webkit-justify-content:flex-end;-ms-flex-pack:end;justify-content:flex-end;padding:15px;border-top:1px solid #eceeef}.modal-footer>:not(:first-child){margin-left:.25rem}.modal-footer>:not(:last-child){margin-right:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:30px auto}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg{max-width:800px}}.tooltip{position:absolute;z-index:1070;display:block;font-family:-apple-system,system-ui,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-style:normal;font-weight:400;letter-spacing:normal;line-break:auto;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;white-space:normal;word-break:normal;word-spacing:normal;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip.bs-tether-element-attached-bottom,.tooltip.tooltip-top{padding:5px 0;margin-top:-3px}.tooltip.bs-tether-element-attached-bottom .tooltip-inner::before,.tooltip.tooltip-top .tooltip-inner::before{bottom:0;left:50%;margin-left:-5px;content:"";border-width:5px 5px 0;border-top-color:#000}.tooltip.bs-tether-element-attached-left,.tooltip.tooltip-right{padding:0 5px;margin-left:3px}.tooltip.bs-tether-element-attached-left .tooltip-inner::before,.tooltip.tooltip-right .tooltip-inner::before{top:50%;left:0;margin-top:-5px;content:"";border-width:5px 5px 5px 0;border-right-color:#000}.tooltip.bs-tether-element-attached-top,.tooltip.tooltip-bottom{padding:5px 0;margin-top:3px}.tooltip.bs-tether-element-attached-top .tooltip-inner::before,.tooltip.tooltip-bottom .tooltip-inner::before{top:0;left:50%;margin-left:-5px;content:"";border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bs-tether-element-attached-right,.tooltip.tooltip-left{padding:0 5px;margin-left:-3px}.tooltip.bs-tether-element-attached-right .tooltip-inner::before,.tooltip.tooltip-left .tooltip-inner::before{top:50%;right:0;margin-top:-5px;content:"";border-width:5px 0 5px 5px;border-left-color:#000}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.tooltip-inner::before{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;padding:1px;font-family:-apple-system,system-ui,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-style:normal;font-weight:400;letter-spacing:normal;line-break:auto;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;white-space:normal;word-break:normal;word-spacing:normal;font-size:.875rem;word-wrap:break-word;background-color:#fff;-webkit-background-clip:padding-box;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover.bs-tether-element-attached-bottom,.popover.popover-top{margin-top:-10px}.popover.bs-tether-element-attached-bottom::after,.popover.bs-tether-element-attached-bottom::before,.popover.popover-top::after,.popover.popover-top::before{left:50%;border-bottom-width:0}.popover.bs-tether-element-attached-bottom::before,.popover.popover-top::before{bottom:-11px;margin-left:-11px;border-top-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-bottom::after,.popover.popover-top::after{bottom:-10px;margin-left:-10px;border-top-color:#fff}.popover.bs-tether-element-attached-left,.popover.popover-right{margin-left:10px}.popover.bs-tether-element-attached-left::after,.popover.bs-tether-element-attached-left::before,.popover.popover-right::after,.popover.popover-right::before{top:50%;border-left-width:0}.popover.bs-tether-element-attached-left::before,.popover.popover-right::before{left:-11px;margin-top:-11px;border-right-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-left::after,.popover.popover-right::after{left:-10px;margin-top:-10px;border-right-color:#fff}.popover.bs-tether-element-attached-top,.popover.popover-bottom{margin-top:10px}.popover.bs-tether-element-attached-top::after,.popover.bs-tether-element-attached-top::before,.popover.popover-bottom::after,.popover.popover-bottom::before{left:50%;border-top-width:0}.popover.bs-tether-element-attached-top::before,.popover.popover-bottom::before{top:-11px;margin-left:-11px;border-bottom-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-top::after,.popover.popover-bottom::after{top:-10px;margin-left:-10px;border-bottom-color:#f7f7f7}.popover.bs-tether-element-attached-top .popover-title::before,.popover.popover-bottom .popover-title::before{position:absolute;top:0;left:50%;display:block;width:20px;margin-left:-10px;content:"";border-bottom:1px solid #f7f7f7}.popover.bs-tether-element-attached-right,.popover.popover-left{margin-left:-10px}.popover.bs-tether-element-attached-right::after,.popover.bs-tether-element-attached-right::before,.popover.popover-left::after,.popover.popover-left::before{top:50%;border-right-width:0}.popover.bs-tether-element-attached-right::before,.popover.popover-left::before{right:-11px;margin-top:-11px;border-left-color:rgba(0,0,0,.25)}.popover.bs-tether-element-attached-right::after,.popover.popover-left::after{right:-10px;margin-top:-10px;border-left-color:#fff}.popover-title{padding:8px 14px;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-right-radius:calc(.3rem - 1px);border-top-left-radius:calc(.3rem - 1px)}.popover-title:empty{display:none}.popover-content{padding:9px 14px}.popover::after,.popover::before{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid}.popover::before{content:"";border-width:11px}.popover::after{content:"";border-width:10px}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-item{position:relative;display:none;width:100%}@media (-webkit-transform-3d){.carousel-item{-webkit-transition:-webkit-transform .6s ease-in-out;transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out,-o-transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}}@supports ((-webkit-transform:translate3d(0,0,0)) or (transform:translate3d(0,0,0))){.carousel-item{-webkit-transition:-webkit-transform .6s ease-in-out;transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out,-o-transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex}.carousel-item-next,.carousel-item-prev{position:absolute;top:0}@media (-webkit-transform-3d){.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.active.carousel-item-right,.carousel-item-next{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.active.carousel-item-left,.carousel-item-prev{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}}@supports ((-webkit-transform:translate3d(0,0,0)) or (transform:translate3d(0,0,0))){.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.active.carousel-item-right,.carousel-item-next{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.active.carousel-item-left,.carousel-item-prev{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-align:center;-webkit-align-items:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:transparent no-repeat center center;-webkit-background-size:100% 100%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M4 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3E%3C/svg%3E")}.carousel-control-next-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M1.5 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3E%3C/svg%3E")}.carousel-indicators{position:absolute;right:0;bottom:10px;left:0;z-index:15;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{position:relative;-webkit-box-flex:1;-webkit-flex:1 0 auto;-ms-flex:1 0 auto;flex:1 0 auto;max-width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:rgba(255,255,255,.5)}.carousel-indicators li::before{position:absolute;top:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators li::after{position:absolute;bottom:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators .active{background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-faded{background-color:#f7f7f7}.bg-primary{background-color:#0275d8!important}a.bg-primary:focus,a.bg-primary:hover{background-color:#025aa5!important}.bg-success{background-color:#5cb85c!important}a.bg-success:focus,a.bg-success:hover{background-color:#449d44!important}.bg-info{background-color:#5bc0de!important}a.bg-info:focus,a.bg-info:hover{background-color:#31b0d5!important}.bg-warning{background-color:#f0ad4e!important}a.bg-warning:focus,a.bg-warning:hover{background-color:#ec971f!important}.bg-danger{background-color:#d9534f!important}a.bg-danger:focus,a.bg-danger:hover{background-color:#c9302c!important}.bg-inverse{background-color:#292b2c!important}a.bg-inverse:focus,a.bg-inverse:hover{background-color:#101112!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.rounded{border-radius:.25rem}.rounded-top{border-top-right-radius:.25rem;border-top-left-radius:.25rem}.rounded-right{border-bottom-right-radius:.25rem;border-top-right-radius:.25rem}.rounded-bottom{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.rounded-left{border-bottom-left-radius:.25rem;border-top-left-radius:.25rem}.rounded-circle{border-radius:50%}.rounded-0{border-radius:0}.clearfix::after{display:block;content:"";clear:both}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-cell{display:table-cell!important}.d-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-webkit-box!important;display:-webkit-flex!important;display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-webkit-inline-box!important;display:-webkit-inline-flex!important;display:-ms-inline-flexbox!important;display:inline-flex!important}}.flex-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-sm-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-sm-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-sm-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-sm-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-sm-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-sm-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-md-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-md-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-md-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-md-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-md-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-md-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-lg-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-lg-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-lg-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-lg-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-lg-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-lg-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-first{-webkit-box-ordinal-group:0;-webkit-order:-1;-ms-flex-order:-1;order:-1}.flex-xl-last{-webkit-box-ordinal-group:2;-webkit-order:1;-ms-flex-order:1;order:1}.flex-xl-unordered{-webkit-box-ordinal-group:1;-webkit-order:0;-ms-flex-order:0;order:0}.flex-xl-row{-webkit-box-orient:horizontal!important;-webkit-box-direction:normal!important;-webkit-flex-direction:row!important;-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-webkit-flex-direction:column!important;-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-webkit-box-orient:horizontal!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:row-reverse!important;-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-webkit-box-orient:vertical!important;-webkit-box-direction:reverse!important;-webkit-flex-direction:column-reverse!important;-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-webkit-flex-wrap:wrap!important;-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-webkit-flex-wrap:nowrap!important;-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-webkit-flex-wrap:wrap-reverse!important;-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-xl-start{-webkit-box-pack:start!important;-webkit-justify-content:flex-start!important;-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-webkit-box-pack:end!important;-webkit-justify-content:flex-end!important;-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-webkit-box-pack:center!important;-webkit-justify-content:center!important;-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-webkit-box-pack:justify!important;-webkit-justify-content:space-between!important;-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-webkit-justify-content:space-around!important;-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-webkit-box-align:start!important;-webkit-align-items:flex-start!important;-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-webkit-box-align:end!important;-webkit-align-items:flex-end!important;-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-webkit-box-align:center!important;-webkit-align-items:center!important;-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-webkit-box-align:baseline!important;-webkit-align-items:baseline!important;-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-webkit-box-align:stretch!important;-webkit-align-items:stretch!important;-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-webkit-align-content:flex-start!important;-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-webkit-align-content:flex-end!important;-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-webkit-align-content:center!important;-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-webkit-align-content:space-between!important;-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-webkit-align-content:space-around!important;-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-webkit-align-content:stretch!important;-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-webkit-align-self:auto!important;-ms-flex-item-align:auto!important;-ms-grid-row-align:auto!important;align-self:auto!important}.align-self-xl-start{-webkit-align-self:flex-start!important;-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-webkit-align-self:flex-end!important;-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-webkit-align-self:center!important;-ms-flex-item-align:center!important;-ms-grid-row-align:center!important;align-self:center!important}.align-self-xl-baseline{-webkit-align-self:baseline!important;-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-webkit-align-self:stretch!important;-ms-flex-item-align:stretch!important;-ms-grid-row-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1030}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.m-0{margin:0 0!important}.mt-0{margin-top:0!important}.mr-0{margin-right:0!important}.mb-0{margin-bottom:0!important}.ml-0{margin-left:0!important}.mx-0{margin-right:0!important;margin-left:0!important}.my-0{margin-top:0!important;margin-bottom:0!important}.m-1{margin:.25rem .25rem!important}.mt-1{margin-top:.25rem!important}.mr-1{margin-right:.25rem!important}.mb-1{margin-bottom:.25rem!important}.ml-1{margin-left:.25rem!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-2{margin:.5rem .5rem!important}.mt-2{margin-top:.5rem!important}.mr-2{margin-right:.5rem!important}.mb-2{margin-bottom:.5rem!important}.ml-2{margin-left:.5rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-3{margin:1rem 1rem!important}.mt-3{margin-top:1rem!important}.mr-3{margin-right:1rem!important}.mb-3{margin-bottom:1rem!important}.ml-3{margin-left:1rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-4{margin:1.5rem 1.5rem!important}.mt-4{margin-top:1.5rem!important}.mr-4{margin-right:1.5rem!important}.mb-4{margin-bottom:1.5rem!important}.ml-4{margin-left:1.5rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-5{margin:3rem 3rem!important}.mt-5{margin-top:3rem!important}.mr-5{margin-right:3rem!important}.mb-5{margin-bottom:3rem!important}.ml-5{margin-left:3rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-0{padding:0 0!important}.pt-0{padding-top:0!important}.pr-0{padding-right:0!important}.pb-0{padding-bottom:0!important}.pl-0{padding-left:0!important}.px-0{padding-right:0!important;padding-left:0!important}.py-0{padding-top:0!important;padding-bottom:0!important}.p-1{padding:.25rem .25rem!important}.pt-1{padding-top:.25rem!important}.pr-1{padding-right:.25rem!important}.pb-1{padding-bottom:.25rem!important}.pl-1{padding-left:.25rem!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-2{padding:.5rem .5rem!important}.pt-2{padding-top:.5rem!important}.pr-2{padding-right:.5rem!important}.pb-2{padding-bottom:.5rem!important}.pl-2{padding-left:.5rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-3{padding:1rem 1rem!important}.pt-3{padding-top:1rem!important}.pr-3{padding-right:1rem!important}.pb-3{padding-bottom:1rem!important}.pl-3{padding-left:1rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-4{padding:1.5rem 1.5rem!important}.pt-4{padding-top:1.5rem!important}.pr-4{padding-right:1.5rem!important}.pb-4{padding-bottom:1.5rem!important}.pl-4{padding-left:1.5rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-5{padding:3rem 3rem!important}.pt-5{padding-top:3rem!important}.pr-5{padding-right:3rem!important}.pb-5{padding-bottom:3rem!important}.pl-5{padding-left:3rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-auto{margin:auto!important}.mt-auto{margin-top:auto!important}.mr-auto{margin-right:auto!important}.mb-auto{margin-bottom:auto!important}.ml-auto{margin-left:auto!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}@media (min-width:576px){.m-sm-0{margin:0 0!important}.mt-sm-0{margin-top:0!important}.mr-sm-0{margin-right:0!important}.mb-sm-0{margin-bottom:0!important}.ml-sm-0{margin-left:0!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.m-sm-1{margin:.25rem .25rem!important}.mt-sm-1{margin-top:.25rem!important}.mr-sm-1{margin-right:.25rem!important}.mb-sm-1{margin-bottom:.25rem!important}.ml-sm-1{margin-left:.25rem!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-sm-2{margin:.5rem .5rem!important}.mt-sm-2{margin-top:.5rem!important}.mr-sm-2{margin-right:.5rem!important}.mb-sm-2{margin-bottom:.5rem!important}.ml-sm-2{margin-left:.5rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-sm-3{margin:1rem 1rem!important}.mt-sm-3{margin-top:1rem!important}.mr-sm-3{margin-right:1rem!important}.mb-sm-3{margin-bottom:1rem!important}.ml-sm-3{margin-left:1rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-sm-4{margin:1.5rem 1.5rem!important}.mt-sm-4{margin-top:1.5rem!important}.mr-sm-4{margin-right:1.5rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.ml-sm-4{margin-left:1.5rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-sm-5{margin:3rem 3rem!important}.mt-sm-5{margin-top:3rem!important}.mr-sm-5{margin-right:3rem!important}.mb-sm-5{margin-bottom:3rem!important}.ml-sm-5{margin-left:3rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-sm-0{padding:0 0!important}.pt-sm-0{padding-top:0!important}.pr-sm-0{padding-right:0!important}.pb-sm-0{padding-bottom:0!important}.pl-sm-0{padding-left:0!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.p-sm-1{padding:.25rem .25rem!important}.pt-sm-1{padding-top:.25rem!important}.pr-sm-1{padding-right:.25rem!important}.pb-sm-1{padding-bottom:.25rem!important}.pl-sm-1{padding-left:.25rem!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-sm-2{padding:.5rem .5rem!important}.pt-sm-2{padding-top:.5rem!important}.pr-sm-2{padding-right:.5rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pl-sm-2{padding-left:.5rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-sm-3{padding:1rem 1rem!important}.pt-sm-3{padding-top:1rem!important}.pr-sm-3{padding-right:1rem!important}.pb-sm-3{padding-bottom:1rem!important}.pl-sm-3{padding-left:1rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-sm-4{padding:1.5rem 1.5rem!important}.pt-sm-4{padding-top:1.5rem!important}.pr-sm-4{padding-right:1.5rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pl-sm-4{padding-left:1.5rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-sm-5{padding:3rem 3rem!important}.pt-sm-5{padding-top:3rem!important}.pr-sm-5{padding-right:3rem!important}.pb-sm-5{padding-bottom:3rem!important}.pl-sm-5{padding-left:3rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto{margin-top:auto!important}.mr-sm-auto{margin-right:auto!important}.mb-sm-auto{margin-bottom:auto!important}.ml-sm-auto{margin-left:auto!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:768px){.m-md-0{margin:0 0!important}.mt-md-0{margin-top:0!important}.mr-md-0{margin-right:0!important}.mb-md-0{margin-bottom:0!important}.ml-md-0{margin-left:0!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.m-md-1{margin:.25rem .25rem!important}.mt-md-1{margin-top:.25rem!important}.mr-md-1{margin-right:.25rem!important}.mb-md-1{margin-bottom:.25rem!important}.ml-md-1{margin-left:.25rem!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-md-2{margin:.5rem .5rem!important}.mt-md-2{margin-top:.5rem!important}.mr-md-2{margin-right:.5rem!important}.mb-md-2{margin-bottom:.5rem!important}.ml-md-2{margin-left:.5rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-md-3{margin:1rem 1rem!important}.mt-md-3{margin-top:1rem!important}.mr-md-3{margin-right:1rem!important}.mb-md-3{margin-bottom:1rem!important}.ml-md-3{margin-left:1rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-md-4{margin:1.5rem 1.5rem!important}.mt-md-4{margin-top:1.5rem!important}.mr-md-4{margin-right:1.5rem!important}.mb-md-4{margin-bottom:1.5rem!important}.ml-md-4{margin-left:1.5rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-md-5{margin:3rem 3rem!important}.mt-md-5{margin-top:3rem!important}.mr-md-5{margin-right:3rem!important}.mb-md-5{margin-bottom:3rem!important}.ml-md-5{margin-left:3rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-md-0{padding:0 0!important}.pt-md-0{padding-top:0!important}.pr-md-0{padding-right:0!important}.pb-md-0{padding-bottom:0!important}.pl-md-0{padding-left:0!important}.px-md-0{padding-right:0!important;padding-left:0!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.p-md-1{padding:.25rem .25rem!important}.pt-md-1{padding-top:.25rem!important}.pr-md-1{padding-right:.25rem!important}.pb-md-1{padding-bottom:.25rem!important}.pl-md-1{padding-left:.25rem!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-md-2{padding:.5rem .5rem!important}.pt-md-2{padding-top:.5rem!important}.pr-md-2{padding-right:.5rem!important}.pb-md-2{padding-bottom:.5rem!important}.pl-md-2{padding-left:.5rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-md-3{padding:1rem 1rem!important}.pt-md-3{padding-top:1rem!important}.pr-md-3{padding-right:1rem!important}.pb-md-3{padding-bottom:1rem!important}.pl-md-3{padding-left:1rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-md-4{padding:1.5rem 1.5rem!important}.pt-md-4{padding-top:1.5rem!important}.pr-md-4{padding-right:1.5rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pl-md-4{padding-left:1.5rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-md-5{padding:3rem 3rem!important}.pt-md-5{padding-top:3rem!important}.pr-md-5{padding-right:3rem!important}.pb-md-5{padding-bottom:3rem!important}.pl-md-5{padding-left:3rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto{margin-top:auto!important}.mr-md-auto{margin-right:auto!important}.mb-md-auto{margin-bottom:auto!important}.ml-md-auto{margin-left:auto!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:992px){.m-lg-0{margin:0 0!important}.mt-lg-0{margin-top:0!important}.mr-lg-0{margin-right:0!important}.mb-lg-0{margin-bottom:0!important}.ml-lg-0{margin-left:0!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.m-lg-1{margin:.25rem .25rem!important}.mt-lg-1{margin-top:.25rem!important}.mr-lg-1{margin-right:.25rem!important}.mb-lg-1{margin-bottom:.25rem!important}.ml-lg-1{margin-left:.25rem!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-lg-2{margin:.5rem .5rem!important}.mt-lg-2{margin-top:.5rem!important}.mr-lg-2{margin-right:.5rem!important}.mb-lg-2{margin-bottom:.5rem!important}.ml-lg-2{margin-left:.5rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-lg-3{margin:1rem 1rem!important}.mt-lg-3{margin-top:1rem!important}.mr-lg-3{margin-right:1rem!important}.mb-lg-3{margin-bottom:1rem!important}.ml-lg-3{margin-left:1rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-lg-4{margin:1.5rem 1.5rem!important}.mt-lg-4{margin-top:1.5rem!important}.mr-lg-4{margin-right:1.5rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.ml-lg-4{margin-left:1.5rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-lg-5{margin:3rem 3rem!important}.mt-lg-5{margin-top:3rem!important}.mr-lg-5{margin-right:3rem!important}.mb-lg-5{margin-bottom:3rem!important}.ml-lg-5{margin-left:3rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-lg-0{padding:0 0!important}.pt-lg-0{padding-top:0!important}.pr-lg-0{padding-right:0!important}.pb-lg-0{padding-bottom:0!important}.pl-lg-0{padding-left:0!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.p-lg-1{padding:.25rem .25rem!important}.pt-lg-1{padding-top:.25rem!important}.pr-lg-1{padding-right:.25rem!important}.pb-lg-1{padding-bottom:.25rem!important}.pl-lg-1{padding-left:.25rem!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-lg-2{padding:.5rem .5rem!important}.pt-lg-2{padding-top:.5rem!important}.pr-lg-2{padding-right:.5rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pl-lg-2{padding-left:.5rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-lg-3{padding:1rem 1rem!important}.pt-lg-3{padding-top:1rem!important}.pr-lg-3{padding-right:1rem!important}.pb-lg-3{padding-bottom:1rem!important}.pl-lg-3{padding-left:1rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-lg-4{padding:1.5rem 1.5rem!important}.pt-lg-4{padding-top:1.5rem!important}.pr-lg-4{padding-right:1.5rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pl-lg-4{padding-left:1.5rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-lg-5{padding:3rem 3rem!important}.pt-lg-5{padding-top:3rem!important}.pr-lg-5{padding-right:3rem!important}.pb-lg-5{padding-bottom:3rem!important}.pl-lg-5{padding-left:3rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto{margin-top:auto!important}.mr-lg-auto{margin-right:auto!important}.mb-lg-auto{margin-bottom:auto!important}.ml-lg-auto{margin-left:auto!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0 0!important}.mt-xl-0{margin-top:0!important}.mr-xl-0{margin-right:0!important}.mb-xl-0{margin-bottom:0!important}.ml-xl-0{margin-left:0!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.m-xl-1{margin:.25rem .25rem!important}.mt-xl-1{margin-top:.25rem!important}.mr-xl-1{margin-right:.25rem!important}.mb-xl-1{margin-bottom:.25rem!important}.ml-xl-1{margin-left:.25rem!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-xl-2{margin:.5rem .5rem!important}.mt-xl-2{margin-top:.5rem!important}.mr-xl-2{margin-right:.5rem!important}.mb-xl-2{margin-bottom:.5rem!important}.ml-xl-2{margin-left:.5rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-xl-3{margin:1rem 1rem!important}.mt-xl-3{margin-top:1rem!important}.mr-xl-3{margin-right:1rem!important}.mb-xl-3{margin-bottom:1rem!important}.ml-xl-3{margin-left:1rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-xl-4{margin:1.5rem 1.5rem!important}.mt-xl-4{margin-top:1.5rem!important}.mr-xl-4{margin-right:1.5rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.ml-xl-4{margin-left:1.5rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-xl-5{margin:3rem 3rem!important}.mt-xl-5{margin-top:3rem!important}.mr-xl-5{margin-right:3rem!important}.mb-xl-5{margin-bottom:3rem!important}.ml-xl-5{margin-left:3rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-xl-0{padding:0 0!important}.pt-xl-0{padding-top:0!important}.pr-xl-0{padding-right:0!important}.pb-xl-0{padding-bottom:0!important}.pl-xl-0{padding-left:0!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.p-xl-1{padding:.25rem .25rem!important}.pt-xl-1{padding-top:.25rem!important}.pr-xl-1{padding-right:.25rem!important}.pb-xl-1{padding-bottom:.25rem!important}.pl-xl-1{padding-left:.25rem!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-xl-2{padding:.5rem .5rem!important}.pt-xl-2{padding-top:.5rem!important}.pr-xl-2{padding-right:.5rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pl-xl-2{padding-left:.5rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-xl-3{padding:1rem 1rem!important}.pt-xl-3{padding-top:1rem!important}.pr-xl-3{padding-right:1rem!important}.pb-xl-3{padding-bottom:1rem!important}.pl-xl-3{padding-left:1rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-xl-4{padding:1.5rem 1.5rem!important}.pt-xl-4{padding-top:1.5rem!important}.pr-xl-4{padding-right:1.5rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pl-xl-4{padding-left:1.5rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-xl-5{padding:3rem 3rem!important}.pt-xl-5{padding-top:3rem!important}.pr-xl-5{padding-right:3rem!important}.pb-xl-5{padding-bottom:3rem!important}.pl-xl-5{padding-left:3rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto{margin-top:auto!important}.mr-xl-auto{margin-right:auto!important}.mb-xl-auto{margin-bottom:auto!important}.ml-xl-auto{margin-left:auto!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}}.text-justify{text-align:justify!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-normal{font-weight:400}.font-weight-bold{font-weight:700}.font-italic{font-style:italic}.text-white{color:#fff!important}.text-muted{color:#636c72!important}a.text-muted:focus,a.text-muted:hover{color:#4b5257!important}.text-primary{color:#0275d8!important}a.text-primary:focus,a.text-primary:hover{color:#025aa5!important}.text-success{color:#5cb85c!important}a.text-success:focus,a.text-success:hover{color:#449d44!important}.text-info{color:#5bc0de!important}a.text-info:focus,a.text-info:hover{color:#31b0d5!important}.text-warning{color:#f0ad4e!important}a.text-warning:focus,a.text-warning:hover{color:#ec971f!important}.text-danger{color:#d9534f!important}a.text-danger:focus,a.text-danger:hover{color:#c9302c!important}.text-gray-dark{color:#292b2c!important}a.text-gray-dark:focus,a.text-gray-dark:hover{color:#101112!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.invisible{visibility:hidden!important}.hidden-xs-up{display:none!important}@media (max-width:575px){.hidden-xs-down{display:none!important}}@media (min-width:576px){.hidden-sm-up{display:none!important}}@media (max-width:767px){.hidden-sm-down{display:none!important}}@media (min-width:768px){.hidden-md-up{display:none!important}}@media (max-width:991px){.hidden-md-down{display:none!important}}@media (min-width:992px){.hidden-lg-up{display:none!important}}@media (max-width:1199px){.hidden-lg-down{display:none!important}}@media (min-width:1200px){.hidden-xl-up{display:none!important}}.hidden-xl-down{display:none!important}.visible-print-block{display:none!important}@media print{.visible-print-block{display:block!important}}.visible-print-inline{display:none!important}@media print{.visible-print-inline{display:inline!important}}.visible-print-inline-block{display:none!important}@media print{.visible-print-inline-block{display:inline-block!important}}@media print{.hidden-print{display:none!important}}/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/archivebox/themes/static/external.png b/archivebox/themes/static/external.png new file mode 100755 index 0000000000000000000000000000000000000000..7e1a5f02aebccd4dcc6b1b0e3040c66ee84270a8 GIT binary patch literal 1647 zcmV-#29WuQP)gwIy-SzeLx3{-TOH0Pa z#^mJWg@uK_zP?jaQ@FUeMn*<ssP45C&jS0Z#=KL|K+q*ZaT6T^=ZwNzx>f$;{MezMmFr z-hyp&pfeg-EEbE!VzF2(7K_DVk;{Eq7SipHWdCbiob5_l5zgwP+;_-S8WPHi#`iyW z(v(2RfaCG2w8fhTLg82%;|yPEGBw{8Lj0lBOs#fPD z0xTl};WJsQGjbahmJlz6v?G;inIw(73980_q z`d%=(mLkC*iCv-ZJo+UDa)Y~&b%mbIJ28$M5`4gFr46&Z?!L`zQe3aQrAs1=ee8qz9E1kOAI z6T=WD0&f#W;B*SpfpMZrVGb}#)F{jY#)t}qxxfffpD-WTPgEzw0DFnrgg9UyQJD}6 z>>=tB;(=kJDq#sQNYo@O1BQr-gr&d$QID`3=qIWXQUJX~EkYWgkGMW<5f)MbJ;Zhb zMpj4%bcyYE@!4@ZcN3DaUi$(S%LCX|c`(_%u&m@q9Sl#B_}VnWH7FfA&d zc6|ygYJl5|&L@^11ICj-aeIMJoyUwA)e2a(@&~sUSUF%F{}r}?rv0YbO`wvrMBv8@ zTeblf0A7%`vLtZP3buL*tWmA!9}!kh!B$FxM@mhR5~qj}_R|F~7iz8-usZQy^q!y) zdD)?kbL8=7d8b(Xl(3!ne8Oho@3!1BvKsE(oa0_nEX~>1xFKsOHsMEEV&W;%DP7je zvrxR`sQZcm%hq`G9CPd~+c9v}{~?kWeqR;)vP15z=)95oXMFLs=CN?6{*oqJiI3_F zyiY!+_{8Af?RYZXah`Kludr9&ro3cf6WH_7$&$Z4EqEh1d}K9.sorting_1,table.dataTable.order-column tbody tr>.sorting_2,table.dataTable.order-column tbody tr>.sorting_3,table.dataTable.display tbody tr>.sorting_1,table.dataTable.display tbody tr>.sorting_2,table.dataTable.display tbody tr>.sorting_3{background-color:#fafafa}table.dataTable.order-column tbody tr.selected>.sorting_1,table.dataTable.order-column tbody tr.selected>.sorting_2,table.dataTable.order-column tbody tr.selected>.sorting_3,table.dataTable.display tbody tr.selected>.sorting_1,table.dataTable.display tbody tr.selected>.sorting_2,table.dataTable.display tbody tr.selected>.sorting_3{background-color:#acbad5}table.dataTable.display tbody tr.odd>.sorting_1,table.dataTable.order-column.stripe tbody tr.odd>.sorting_1{background-color:#f1f1f1}table.dataTable.display tbody tr.odd>.sorting_2,table.dataTable.order-column.stripe tbody tr.odd>.sorting_2{background-color:#f3f3f3}table.dataTable.display tbody tr.odd>.sorting_3,table.dataTable.order-column.stripe tbody tr.odd>.sorting_3{background-color:whitesmoke}table.dataTable.display tbody tr.odd.selected>.sorting_1,table.dataTable.order-column.stripe tbody tr.odd.selected>.sorting_1{background-color:#a6b4cd}table.dataTable.display tbody tr.odd.selected>.sorting_2,table.dataTable.order-column.stripe tbody tr.odd.selected>.sorting_2{background-color:#a8b5cf}table.dataTable.display tbody tr.odd.selected>.sorting_3,table.dataTable.order-column.stripe tbody tr.odd.selected>.sorting_3{background-color:#a9b7d1}table.dataTable.display tbody tr.even>.sorting_1,table.dataTable.order-column.stripe tbody tr.even>.sorting_1{background-color:#fafafa}table.dataTable.display tbody tr.even>.sorting_2,table.dataTable.order-column.stripe tbody tr.even>.sorting_2{background-color:#fcfcfc}table.dataTable.display tbody tr.even>.sorting_3,table.dataTable.order-column.stripe tbody tr.even>.sorting_3{background-color:#fefefe}table.dataTable.display tbody tr.even.selected>.sorting_1,table.dataTable.order-column.stripe tbody tr.even.selected>.sorting_1{background-color:#acbad5}table.dataTable.display tbody tr.even.selected>.sorting_2,table.dataTable.order-column.stripe tbody tr.even.selected>.sorting_2{background-color:#aebcd6}table.dataTable.display tbody tr.even.selected>.sorting_3,table.dataTable.order-column.stripe tbody tr.even.selected>.sorting_3{background-color:#afbdd8}table.dataTable.display tbody tr:hover>.sorting_1,table.dataTable.order-column.hover tbody tr:hover>.sorting_1{background-color:#eaeaea}table.dataTable.display tbody tr:hover>.sorting_2,table.dataTable.order-column.hover tbody tr:hover>.sorting_2{background-color:#ececec}table.dataTable.display tbody tr:hover>.sorting_3,table.dataTable.order-column.hover tbody tr:hover>.sorting_3{background-color:#efefef}table.dataTable.display tbody tr:hover.selected>.sorting_1,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_1{background-color:#a2aec7}table.dataTable.display tbody tr:hover.selected>.sorting_2,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_2{background-color:#a3b0c9}table.dataTable.display tbody tr:hover.selected>.sorting_3,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_3{background-color:#a5b2cb}table.dataTable.no-footer{border-bottom:1px solid #111}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}table.dataTable.compact thead th,table.dataTable.compact thead td{padding:4px 17px 4px 4px}table.dataTable.compact tfoot th,table.dataTable.compact tfoot td{padding:4px}table.dataTable.compact tbody th,table.dataTable.compact tbody td{padding:4px}table.dataTable th.dt-left,table.dataTable td.dt-left{text-align:left}table.dataTable th.dt-center,table.dataTable td.dt-center,table.dataTable td.dataTables_empty{text-align:center}table.dataTable th.dt-right,table.dataTable td.dt-right{text-align:right}table.dataTable th.dt-justify,table.dataTable td.dt-justify{text-align:justify}table.dataTable th.dt-nowrap,table.dataTable td.dt-nowrap{white-space:nowrap}table.dataTable thead th.dt-head-left,table.dataTable thead td.dt-head-left,table.dataTable tfoot th.dt-head-left,table.dataTable tfoot td.dt-head-left{text-align:left}table.dataTable thead th.dt-head-center,table.dataTable thead td.dt-head-center,table.dataTable tfoot th.dt-head-center,table.dataTable tfoot td.dt-head-center{text-align:center}table.dataTable thead th.dt-head-right,table.dataTable thead td.dt-head-right,table.dataTable tfoot th.dt-head-right,table.dataTable tfoot td.dt-head-right{text-align:right}table.dataTable thead th.dt-head-justify,table.dataTable thead td.dt-head-justify,table.dataTable tfoot th.dt-head-justify,table.dataTable tfoot td.dt-head-justify{text-align:justify}table.dataTable thead th.dt-head-nowrap,table.dataTable thead td.dt-head-nowrap,table.dataTable tfoot th.dt-head-nowrap,table.dataTable tfoot td.dt-head-nowrap{white-space:nowrap}table.dataTable tbody th.dt-body-left,table.dataTable tbody td.dt-body-left{text-align:left}table.dataTable tbody th.dt-body-center,table.dataTable tbody td.dt-body-center{text-align:center}table.dataTable tbody th.dt-body-right,table.dataTable tbody td.dt-body-right{text-align:right}table.dataTable tbody th.dt-body-justify,table.dataTable tbody td.dt-body-justify{text-align:justify}table.dataTable tbody th.dt-body-nowrap,table.dataTable tbody td.dt-body-nowrap{white-space:nowrap}table.dataTable,table.dataTable th,table.dataTable td{box-sizing:content-box}.dataTables_wrapper{position:relative;clear:both;*zoom:1;zoom:1}.dataTables_wrapper .dataTables_length{float:left}.dataTables_wrapper .dataTables_filter{float:right;text-align:right}.dataTables_wrapper .dataTables_filter input{margin-left:0.5em}.dataTables_wrapper .dataTables_info{clear:both;float:left;padding-top:0.755em}.dataTables_wrapper .dataTables_paginate{float:right;text-align:right;padding-top:0.25em}.dataTables_wrapper .dataTables_paginate .paginate_button{box-sizing:border-box;display:inline-block;min-width:1.5em;padding:0.5em 1em;margin-left:2px;text-align:center;text-decoration:none !important;cursor:pointer;*cursor:hand;color:#333 !important;border:1px solid transparent;border-radius:2px}.dataTables_wrapper .dataTables_paginate .paginate_button.current,.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{color:#333 !important;border:1px solid #979797;background-color:white;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #fff), color-stop(100%, #dcdcdc));background:-webkit-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:-moz-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:-ms-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:-o-linear-gradient(top, #fff 0%, #dcdcdc 100%);background:linear-gradient(to bottom, #fff 0%, #dcdcdc 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button.disabled,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{cursor:default;color:#666 !important;border:1px solid transparent;background:transparent;box-shadow:none}.dataTables_wrapper .dataTables_paginate .paginate_button:hover{color:white !important;border:1px solid #111;background-color:#585858;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));background:-webkit-linear-gradient(top, #585858 0%, #111 100%);background:-moz-linear-gradient(top, #585858 0%, #111 100%);background:-ms-linear-gradient(top, #585858 0%, #111 100%);background:-o-linear-gradient(top, #585858 0%, #111 100%);background:linear-gradient(to bottom, #585858 0%, #111 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button:active{outline:none;background-color:#2b2b2b;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);box-shadow:inset 0 0 3px #111}.dataTables_wrapper .dataTables_paginate .ellipsis{padding:0 1em}.dataTables_wrapper .dataTables_processing{position:absolute;top:50%;left:50%;width:100%;height:40px;margin-left:-50%;margin-top:-25px;padding-top:20px;text-align:center;font-size:1.2em;background-color:white;background:-webkit-gradient(linear, left top, right top, color-stop(0%, rgba(255,255,255,0)), color-stop(25%, rgba(255,255,255,0.9)), color-stop(75%, rgba(255,255,255,0.9)), color-stop(100%, rgba(255,255,255,0)));background:-webkit-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:-moz-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:-ms-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:-o-linear-gradient(left, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%);background:linear-gradient(to right, rgba(255,255,255,0) 0%, rgba(255,255,255,0.9) 25%, rgba(255,255,255,0.9) 75%, rgba(255,255,255,0) 100%)}.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter,.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_processing,.dataTables_wrapper .dataTables_paginate{color:#333}.dataTables_wrapper .dataTables_scroll{clear:both}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody{*margin-top:-1px;-webkit-overflow-scrolling:touch}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td{vertical-align:middle}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td>div.dataTables_sizing{height:0;overflow:hidden;margin:0 !important;padding:0 !important}.dataTables_wrapper.no-footer .dataTables_scrollBody{border-bottom:1px solid #111}.dataTables_wrapper.no-footer div.dataTables_scrollHead table.dataTable,.dataTables_wrapper.no-footer div.dataTables_scrollBody>table{border-bottom:none}.dataTables_wrapper:after{visibility:hidden;display:block;content:"";clear:both;height:0}@media screen and (max-width: 767px){.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_paginate{float:none;text-align:center}.dataTables_wrapper .dataTables_paginate{margin-top:0.5em}}@media screen and (max-width: 640px){.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter{float:none;text-align:center}.dataTables_wrapper .dataTables_filter{margin-top:0.5em}} diff --git a/archivebox/themes/static/jquery.dataTables.min.js b/archivebox/themes/static/jquery.dataTables.min.js new file mode 100644 index 00000000..07af1c39 --- /dev/null +++ b/archivebox/themes/static/jquery.dataTables.min.js @@ -0,0 +1,166 @@ +/*! + DataTables 1.10.19 + ©2008-2018 SpryMedia Ltd - datatables.net/license +*/ +(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Z(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()), +d[c]=e,"o"===b[1]&&Z(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Z(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Ca(a){var b=n.defaults.oLanguage,c=b.sDecimal;c&&Da(c);if(a){var d=a.sZeroRecords;!a.sEmptyTable&&(d&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(d&&"Loading..."===b.sLoadingRecords)&&F(a, +a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&c!==a&&Da(a)}}function fb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%": +"");"boolean"===typeof a.scrollX&&(a.scrollX=a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b").css({position:"fixed",top:0,left:-1*h(E).scrollLeft(),height:1,width:1, +overflow:"hidden"}).append(h("
").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(h("
").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,n.__browser);a.oScroll.iBarWidth=n.__browser.barWidth} +function ib(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==e;)a.hasOwnProperty(d)&&(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ea(a,b){var c=n.defaults.column,d=a.aoColumns.length,c=h.extend({},n.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},n.models.oSearch,c[d]);ka(a,d,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b], +d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=e.attr("width")||null;var f=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(gb(c),J(n.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),c.sClass&&e.addClass(c.sClass),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=S(g),i=b.mRender? +S(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b._setter=null;b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return N(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone, +b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function $(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Fa(a);for(var c=0,d=b.length;cq[f])d(l.length+q[f],m);else if("string"=== +typeof q[f]){j=0;for(i=l.length;jb&&a[e]--; -1!=d&&c===k&&a.splice(d, +1)}function da(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=Ia(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c").appendTo(g));b=0;for(c=l.length;btr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(m.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(m.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,m=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!mb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:m;for(j=j?0:g;j",{"class":e?d[0]:""}).append(h("",{valign:"top",colSpan:V(a),"class":a.oClasses.sRowEmpty}).html(c))[0];r(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ka(a),g,m,i]);r(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ka(a),g,m,i]);d=h(a.nTBody);d.children().detach(); +d.append(h(b));r(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;c.bSort&&nb(a);d?ga(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;P(a);a._drawHold=!1}function ob(a){var b=a.oClasses,c=h(a.nTable),c=h("
").insertBefore(c),d=a.oFeatures,e=h("
",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore= +a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,m,l,q,k=0;k")[0];m=f[k+1];if("'"==m||'"'==m){l="";for(q=2;f[k+q]!=m;)l+=f[k+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(m=l.split("."),i.id=m[0].substr(1,m[0].length-1),i.className=m[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;k+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=pb(a);else if("f"==j&& +d.bFilter)g=qb(a);else if("r"==j&&d.bProcessing)g=rb(a);else if("t"==j)g=sb(a);else if("i"==j&&d.bInfo)g=tb(a);else if("p"==j&&d.bPaginate)g=ub(a);else if(0!==n.ext.feature.length){i=n.ext.feature;q=0;for(m=i.length;q',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_", +g):j+g,b=h("
",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("
").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).on("change.DT",function(){Ra(a,h(this).val());P(a)});h(a.nTable).on("length.dt.DT",function(b,c,d){a=== +c&&h("select",i).val(d)});return i[0]}function ub(a){var b=a.sPaginationType,c=n.ext.pager[b],d="function"===typeof c,e=function(a){P(a)},b=h("
").addClass(a.oClasses.sPaging+b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;lf&&(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]} +function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");r(a,null,"processing",[a,b])}function sb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),m=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("
",{"class":f.sScrollWrapper}).append(h("
",{"class":f.sScrollHead}).css({overflow:"hidden", +position:"relative",border:0,width:d?!d?null:v(d):"100%"}).append(h("
",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("
",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:v(d)}).append(b));l&&i.append(h("
",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:v(d):"100%"}).append(h("
", +{"class":f.sScrollFootInner}).append(m.removeAttr("id").css("margin-left",0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),k=b[0],f=b[1],t=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(t.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=t;a.aoDrawCallback.push({fn:la,sName:"scrolling"});return i[0]}function la(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth, +f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,m=j.children("table"),j=a.nScrollBody,l=h(j),q=j.style,t=h(a.nScrollFoot).children("div"),n=t.children("table"),o=h(a.nTHead),p=h(a.nTable),s=p[0],r=s.style,u=a.nTFoot?h(a.nTFoot):null,x=a.oBrowser,U=x.bScrollOversize,Xb=D(a.aoColumns,"nTh"),Q,L,R,w,Ua=[],y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!== +L&&a.scrollBarVis!==k)a.scrollBarVis=L,$(a);else{a.scrollBarVis=L;p.children("thead, tfoot").remove();u&&(R=u.clone().prependTo(p),Q=u.find("tr"),R=R.find("tr"));w=o.clone().prependTo(p);o=o.find("tr");L=w.find("tr");w.find("th, td").removeAttr("tabindex");c||(q.width="100%",f[0].style.width="100%");h.each(ra(a,w),function(b,c){B=aa(a,b);c.style.width=a.aoColumns[B].sWidth});u&&I(function(a){a.style.width=""},R);f=p.outerWidth();if(""===c){r.width="100%";if(U&&(p.find("tbody").height()>j.offsetHeight|| +"scroll"==l.css("overflow-y")))r.width=v(p.outerWidth()-b);f=p.outerWidth()}else""!==d&&(r.width=v(d),f=p.outerWidth());I(C,L);I(function(a){z.push(a.innerHTML);Ua.push(v(h(a).css("width")))},L);I(function(a,b){if(h.inArray(a,Xb)!==-1)a.style.width=Ua[b]},o);h(L).height(0);u&&(I(C,R),I(function(a){A.push(a.innerHTML);y.push(v(h(a).css("width")))},R),I(function(a,b){a.style.width=y[b]},Q),h(R).height(0));I(function(a,b){a.innerHTML='
'+z[b]+"
";a.childNodes[0].style.height= +"0";a.childNodes[0].style.overflow="hidden";a.style.width=Ua[b]},L);u&&I(function(a,b){a.innerHTML='
'+A[b]+"
";a.childNodes[0].style.height="0";a.childNodes[0].style.overflow="hidden";a.style.width=y[b]},R);if(p.outerWidth()j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(U&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))r.width=v(Q-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else Q="100%";q.width=v(Q); +g.width=v(Q);u&&(a.nScrollFoot.style.width=v(Q));!e&&U&&(q.height=v(s.offsetHeight+b));c=p.outerWidth();m[0].style.width=v(c);i.width=v(c);d=p.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+(x.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";u&&(n[0].style.width=v(c),t[0].style.width=v(c),t[0].style[e]=d?b+"px":"0px");p.children("colgroup").insertBefore(p.children("thead"));l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=0}}function I(a,b,c){for(var d=0,e=0, +f=b.length,g,j;e").appendTo(j.find("tbody"));j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");m=ra(a,j.find("thead")[0]);for(n=0;n").css({width:o.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(n=0;n").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()").css("width",v(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Gb(a, +b){var c=Hb(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Hb(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;fd&&(d=c.length,e=f);return e}function v(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function X(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var m=[];f=function(a){a.length&& +!h.isArray(a[0])?m.push(a):h.merge(m,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;ae?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];return ce?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,n=f[a]._aSortData,o=f[b]._aSortData;for(j=0;jg?1:0})}a.bSorted=!0}function Jb(a){for(var b,c,d=a.aoColumns,e=X(a),a=a.oLanguage.oAria,f=0,g=d.length;f/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0e?e+1:3));e=0;for(f=d.length;ee?e+1:3))}a.aLastSort=d}function Ib(a,b){var c=a.aoColumns[b],d=n.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,ba(a,b)));for(var f,g=n.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j=f.length?[0,c[1]]:c)}));b.search!==k&&h.extend(a.oPreviousSearch,Cb(b.search));if(b.columns){d=0;for(e=b.columns.length;d=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Na(a,b){var c=a.renderer,d=n.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"=== +typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function ia(a,b){var c=[],c=Lb.numbers_length,d=Math.floor(c/2);b<=c?c=Y(0,b):a<=d?(c=Y(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=Y(b-(c-2),b):(c=Y(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function Da(a){h.each({num:function(b){return za(b,a)},"num-fmt":function(b){return za(b,a,Ya)},"html-num":function(b){return za(b, +a,Aa)},"html-num-fmt":function(b){return za(b,a,Aa,Ya)}},function(b,c){x.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(x.type.search[b+a]=x.type.search.html)})}function Mb(a){return function(){var b=[ya(this[n.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return n.ext.internal[a].apply(this,b)}}var n=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new s(ya(this[x.iApiIndex])):new s(this)}; +this.fnAddData=function(a,b){var c=this.api(!0),d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&la(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a, +b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data(): +c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]}; +this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return ya(this[x.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust(); +(d===k||d)&&h.draw();return 0};this.fnVersionCheck=x.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=x.internal;for(var e in n.ext.internal)e&&(this[e]=Mb(e));this.each(function(){var e={},g=1").appendTo(q)); +p.nTHead=b[0];b=q.children("tbody");b.length===0&&(b=h("").appendTo(q));p.nTBody=b[0];b=q.children("tfoot");if(b.length===0&&a.length>0&&(p.oScroll.sX!==""||p.oScroll.sY!==""))b=h("").appendTo(q);if(b.length===0||b.children().length===0)q.addClass(u.sNoFooter);else if(b.length>0){p.nTFoot=b[0];ea(p.aoFooter,p.nTFoot)}if(g.aaData)for(j=0;j/g,Zb=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,$b=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Ya=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Ob=function(a){var b=parseInt(a,10);return!isNaN(b)&& +isFinite(a)?b:null},Pb=function(a,b){Za[b]||(Za[b]=RegExp(Qa(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Za[b],"."):a},$a=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Pb(a,b));c&&d&&(a=a.replace(Ya,""));return!isNaN(parseFloat(a))&&isFinite(a)},Qb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===typeof a)?null:$a(a.replace(Aa,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;ea.length)){b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d")[0],Wb=va.textContent!==k,Yb= +/<.*?>/g,Oa=n.util.throttle,Sb=[],w=Array.prototype,ac=function(a){var b,c,d=n.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};s=function(a,b){if(!(this instanceof +s))return new s(a,b);var c=[],d=function(a){(a=ac(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;ea?new s(b[a],this[a]):null},filter:function(a){var b=[];if(w.filter)b=w.filter.call(this,a,this);else for(var c=0,d=this.length;c").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=V(d),e.push(c[0]))};f(a,b);c._details&&c._details.detach();c._details=h(e); +c._detailsShow&&c._details.insertAfter(c.nTr)}return this});o(["row().child.show()","row().child().show()"],function(){Ub(this,!0);return this});o(["row().child.hide()","row().child().hide()"],function(){Ub(this,!1);return this});o(["row().child.remove()","row().child().remove()"],function(){db(this);return this});o("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var bc=/^([^:]+):(name|visIdx|visible)$/,Vb=function(a,b, +c,d,e){for(var c=[],d=0,f=e.length;d=0?b:g.length+b];if(typeof a==="function"){var e=Ba(c,f);return h.map(g,function(b,f){return a(f,Vb(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(bc): +"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var n=h.map(g,function(a,b){return a.bVisible?b:null});return[n[n.length+b]]}return[aa(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null});default:return[]}if(a.nodeName&&a._DT_CellIndex)return[a._DT_CellIndex.column];b=h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray();if(b.length||!a.nodeName)return b;b=h(a).closest("*[data-dt-column]");return b.length?[b.data("dt-column")]:[]},c,f)}, +1);c.selector.cols=a;c.selector.opts=b;return c});u("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});u("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});u("columns().data()","column().data()",function(){return this.iterator("column-rows",Vb,1)});u("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData}, +1)});u("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});u("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,e,"anCells",b)},1)});u("columns().visible()","column().visible()",function(a,b){var c=this.iterator("column",function(b,c){if(a===k)return b.aoColumns[c].bVisible;var f=b.aoColumns,g=f[c],j=b.aoData, +i,m,l;if(a!==k&&g.bVisible!==a){if(a){var n=h.inArray(!0,D(f,"bVisible"),c+1);i=0;for(m=j.length;id;return!0};n.isDataTable= +n.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;if(a instanceof n.Api)return!0;h.each(n.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:null;if(e.nTable===b||f===b||g===b)c=!0});return c};n.tables=n.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(n.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new s(c):c};n.camelToHungarian=J;o("$()",function(a,b){var c= +this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){o(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0]=h.map(a[0].split(/\s/),function(a){return!a.match(/\.dt\b/)?a+".dt":a}).join(" ");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});o("clear()",function(){return this.iterator("table",function(a){oa(a)})});o("settings()",function(){return new s(this.context,this.context)});o("init()",function(){var a= +this.context;return a.length?a[0].oInit:null});o("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});o("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),o;b.bDestroying=!0;r(b,"aoDestroyCallback","destroy",[b]);a||(new s(b)).columns().visible(!0);k.off(".DT").find(":not(tbody *)").off(".DT"); +h(E).off(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];wa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable), +(o=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%o])}));c=h.inArray(b,n.settings);-1!==c&&n.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){o(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});o("i18n()",function(a,b,c){var d=this.context[0],a=S(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]: +a._);return a.replace("%d",c)});n.version="1.10.19";n.settings=[];n.models={};n.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};n.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};n.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null, +sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};n.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1, +bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+ +a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"}, +oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({}, +n.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};Z(n.defaults);n.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null}; +Z(n.defaults.column);n.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[], +aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button", +iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?1*this._iRecordsTotal: +this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};n.ext=x={buttons:{}, +classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:n.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:n.version};h.extend(x,{afnFiltering:x.search,aTypes:x.type.detect,ofnSearch:x.type.search,oSort:x.type.order,afnSortData:x.order,aoFeatures:x.feature,oApi:x.internal,oStdClasses:x.classes,oPagination:x.pager}); +h.extend(n.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled", +sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"", +sJUIHeader:"",sJUIFooter:""});var Lb=n.ext.pager;h.extend(Lb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[ia(a,b)]},simple_numbers:function(a,b){return["previous",ia(a,b),"next"]},full_numbers:function(a,b){return["first","previous",ia(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",ia(a,b),"last"]},_numbers:ia,numbers_length:7});h.extend(!0,n.ext.renderer,{pageButton:{_:function(a,b,c,d,e, +f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||{},m,l,n=0,o=function(b,d){var k,s,u,r,v=function(b){Ta(a,b.data.action,true)};k=0;for(s=d.length;k").appendTo(b);o(u,r)}else{m=null;l="";switch(r){case "ellipsis":b.append('…');break;case "first":m=j.sFirst;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":m=j.sPrevious;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":m= +j.sNext;l=r+(e",{"class":g.sPageButton+" "+l,"aria-controls":a.sTableId,"aria-label":i[r],"data-dt-idx":n,tabindex:a.iTabIndex,id:c===0&&typeof r==="string"?a.sTableId+"_"+r:null}).html(m).appendTo(b);Wa(u,{action:r},v);n++}}}},s;try{s=h(b).find(H.activeElement).data("dt-idx")}catch(u){}o(h(b).empty(),d);s!==k&&h(b).find("[data-dt-idx="+ +s+"]").focus()}}});h.extend(n.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&!Zb.test(a))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Qb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Qb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)|| +"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(n.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Nb," ").replace(Aa,""):""},string:function(a){return M(a)?a:"string"===typeof a?a.replace(Nb," "):a}});var za=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Pb(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(x.type.order,{"date-pre":function(a){a=Date.parse(a);return isNaN(a)?-Infinity:a},"html-pre":function(a){return M(a)? +"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return ab?1:0},"string-desc":function(a,b){return ab?-1:0}});Da("");h.extend(!0,n.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc: +c.sSortingClass)}})},jqueryui:function(a,b,c,d){h("
").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]== +"asc"?d.sSortJUIAsc:h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});var eb=function(a){return"string"===typeof a?a.replace(//g,">").replace(/"/g,"""):a};n.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return eb(f);h=h.toFixed(c);f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g, +a)+f+(e||"")}}},text:function(){return{display:eb,filter:eb}}};h.extend(n.ext.internal,{_fnExternApiFunc:Mb,_fnBuildAjax:sa,_fnAjaxUpdate:mb,_fnAjaxParameters:vb,_fnAjaxUpdateDraw:wb,_fnAjaxDataSrc:ta,_fnAddColumn:Ea,_fnColumnOptions:ka,_fnAdjustColumnSizing:$,_fnVisibleToColumnIndex:aa,_fnColumnIndexToVisible:ba,_fnVisbleColumns:V,_fnGetColumns:ma,_fnColumnTypes:Ga,_fnApplyColumnDefs:jb,_fnHungarianMap:Z,_fnCamelToHungarian:J,_fnLanguageCompat:Ca,_fnBrowserDetect:hb,_fnAddData:O,_fnAddTr:na,_fnNodeToDataIndex:function(a, +b){return b._DT_RowIndex!==k?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:kb,_fnSplitObjNotation:Ja,_fnGetObjectDataFn:S,_fnSetObjectDataFn:N,_fnGetDataMaster:Ka,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:da,_fnGetRowElements:Ia,_fnCreateTr:Ha,_fnBuildHead:lb,_fnDrawHead:fa,_fnDraw:P,_fnReDraw:T,_fnAddOptionsHtml:ob,_fnDetectHeader:ea,_fnGetUniqueThs:ra,_fnFeatureHtmlFilter:qb,_fnFilterComplete:ga,_fnFilterCustom:zb, +_fnFilterColumn:yb,_fnFilter:xb,_fnFilterCreateSearch:Pa,_fnEscapeRegex:Qa,_fnFilterData:Ab,_fnFeatureHtmlInfo:tb,_fnUpdateInfo:Db,_fnInfoMacros:Eb,_fnInitialise:ha,_fnInitComplete:ua,_fnLengthChange:Ra,_fnFeatureHtmlLength:pb,_fnFeatureHtmlPaginate:ub,_fnPageChange:Ta,_fnFeatureHtmlProcessing:rb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:sb,_fnScrollDraw:la,_fnApplyToChildren:I,_fnCalculateColumnWidths:Fa,_fnThrottle:Oa,_fnConvertToWidth:Fb,_fnGetWidestNode:Gb,_fnGetMaxLenString:Hb,_fnStringToCss:v, +_fnSortFlatten:X,_fnSort:nb,_fnSortAria:Jb,_fnSortListener:Va,_fnSortAttachListener:Ma,_fnSortingClasses:wa,_fnSortData:Ib,_fnSaveState:xa,_fnLoadState:Kb,_fnSettingsFromNode:ya,_fnLog:K,_fnMap:F,_fnBindAction:Wa,_fnCallbackReg:z,_fnCallbackFire:r,_fnLengthOverflow:Sa,_fnRenderer:Na,_fnDataSource:y,_fnRowAttributes:La,_fnExtend:Xa,_fnCalculateEnd:function(){}});h.fn.dataTable=n;n.$=h;h.fn.dataTableSettings=n.settings;h.fn.dataTableExt=n.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()}; +h.each(n,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable}); diff --git a/archivebox/themes/static/jquery.min.js b/archivebox/themes/static/jquery.min.js new file mode 100644 index 00000000..4d9b3a25 --- /dev/null +++ b/archivebox/themes/static/jquery.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.3.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(e,t){"use strict";var n=[],r=e.document,i=Object.getPrototypeOf,o=n.slice,a=n.concat,s=n.push,u=n.indexOf,l={},c=l.toString,f=l.hasOwnProperty,p=f.toString,d=p.call(Object),h={},g=function e(t){return"function"==typeof t&&"number"!=typeof t.nodeType},y=function e(t){return null!=t&&t===t.window},v={type:!0,src:!0,noModule:!0};function m(e,t,n){var i,o=(t=t||r).createElement("script");if(o.text=e,n)for(i in v)n[i]&&(o[i]=n[i]);t.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?l[c.call(e)]||"object":typeof e}var b="3.3.1",w=function(e,t){return new w.fn.init(e,t)},T=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;w.fn=w.prototype={jquery:"3.3.1",constructor:w,length:0,toArray:function(){return o.call(this)},get:function(e){return null==e?o.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=w.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return w.each(this,e)},map:function(e){return this.pushStack(w.map(this,function(t,n){return e.call(t,n,t)}))},slice:function(){return this.pushStack(o.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(n>=0&&n0&&t-1 in e)}var E=function(e){var t,n,r,i,o,a,s,u,l,c,f,p,d,h,g,y,v,m,x,b="sizzle"+1*new Date,w=e.document,T=0,C=0,E=ae(),k=ae(),S=ae(),D=function(e,t){return e===t&&(f=!0),0},N={}.hasOwnProperty,A=[],j=A.pop,q=A.push,L=A.push,H=A.slice,O=function(e,t){for(var n=0,r=e.length;n+~]|"+M+")"+M+"*"),z=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),X=new RegExp(W),U=new RegExp("^"+R+"$"),V={ID:new RegExp("^#("+R+")"),CLASS:new RegExp("^\\.("+R+")"),TAG:new RegExp("^("+R+"|[*])"),ATTR:new RegExp("^"+I),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+P+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},G=/^(?:input|select|textarea|button)$/i,Y=/^h\d$/i,Q=/^[^{]+\{\s*\[native \w/,J=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,K=/[+~]/,Z=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ee=function(e,t,n){var r="0x"+t-65536;return r!==r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},te=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ne=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},re=function(){p()},ie=me(function(e){return!0===e.disabled&&("form"in e||"label"in e)},{dir:"parentNode",next:"legend"});try{L.apply(A=H.call(w.childNodes),w.childNodes),A[w.childNodes.length].nodeType}catch(e){L={apply:A.length?function(e,t){q.apply(e,H.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function oe(e,t,r,i){var o,s,l,c,f,h,v,m=t&&t.ownerDocument,T=t?t.nodeType:9;if(r=r||[],"string"!=typeof e||!e||1!==T&&9!==T&&11!==T)return r;if(!i&&((t?t.ownerDocument||t:w)!==d&&p(t),t=t||d,g)){if(11!==T&&(f=J.exec(e)))if(o=f[1]){if(9===T){if(!(l=t.getElementById(o)))return r;if(l.id===o)return r.push(l),r}else if(m&&(l=m.getElementById(o))&&x(t,l)&&l.id===o)return r.push(l),r}else{if(f[2])return L.apply(r,t.getElementsByTagName(e)),r;if((o=f[3])&&n.getElementsByClassName&&t.getElementsByClassName)return L.apply(r,t.getElementsByClassName(o)),r}if(n.qsa&&!S[e+" "]&&(!y||!y.test(e))){if(1!==T)m=t,v=e;else if("object"!==t.nodeName.toLowerCase()){(c=t.getAttribute("id"))?c=c.replace(te,ne):t.setAttribute("id",c=b),s=(h=a(e)).length;while(s--)h[s]="#"+c+" "+ve(h[s]);v=h.join(","),m=K.test(e)&&ge(t.parentNode)||t}if(v)try{return L.apply(r,m.querySelectorAll(v)),r}catch(e){}finally{c===b&&t.removeAttribute("id")}}}return u(e.replace(B,"$1"),t,r,i)}function ae(){var e=[];function t(n,i){return e.push(n+" ")>r.cacheLength&&delete t[e.shift()],t[n+" "]=i}return t}function se(e){return e[b]=!0,e}function ue(e){var t=d.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function le(e,t){var n=e.split("|"),i=n.length;while(i--)r.attrHandle[n[i]]=t}function ce(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function fe(e){return function(t){return"input"===t.nodeName.toLowerCase()&&t.type===e}}function pe(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function de(e){return function(t){return"form"in t?t.parentNode&&!1===t.disabled?"label"in t?"label"in t.parentNode?t.parentNode.disabled===e:t.disabled===e:t.isDisabled===e||t.isDisabled!==!e&&ie(t)===e:t.disabled===e:"label"in t&&t.disabled===e}}function he(e){return se(function(t){return t=+t,se(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}function ge(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}n=oe.support={},o=oe.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return!!t&&"HTML"!==t.nodeName},p=oe.setDocument=function(e){var t,i,a=e?e.ownerDocument||e:w;return a!==d&&9===a.nodeType&&a.documentElement?(d=a,h=d.documentElement,g=!o(d),w!==d&&(i=d.defaultView)&&i.top!==i&&(i.addEventListener?i.addEventListener("unload",re,!1):i.attachEvent&&i.attachEvent("onunload",re)),n.attributes=ue(function(e){return e.className="i",!e.getAttribute("className")}),n.getElementsByTagName=ue(function(e){return e.appendChild(d.createComment("")),!e.getElementsByTagName("*").length}),n.getElementsByClassName=Q.test(d.getElementsByClassName),n.getById=ue(function(e){return h.appendChild(e).id=b,!d.getElementsByName||!d.getElementsByName(b).length}),n.getById?(r.filter.ID=function(e){var t=e.replace(Z,ee);return function(e){return e.getAttribute("id")===t}},r.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&g){var n=t.getElementById(e);return n?[n]:[]}}):(r.filter.ID=function(e){var t=e.replace(Z,ee);return function(e){var n="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return n&&n.value===t}},r.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&g){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),r.find.TAG=n.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):n.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},r.find.CLASS=n.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&g)return t.getElementsByClassName(e)},v=[],y=[],(n.qsa=Q.test(d.querySelectorAll))&&(ue(function(e){h.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&y.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||y.push("\\["+M+"*(?:value|"+P+")"),e.querySelectorAll("[id~="+b+"-]").length||y.push("~="),e.querySelectorAll(":checked").length||y.push(":checked"),e.querySelectorAll("a#"+b+"+*").length||y.push(".#.+[+~]")}),ue(function(e){e.innerHTML="";var t=d.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&y.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&y.push(":enabled",":disabled"),h.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&y.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),y.push(",.*:")})),(n.matchesSelector=Q.test(m=h.matches||h.webkitMatchesSelector||h.mozMatchesSelector||h.oMatchesSelector||h.msMatchesSelector))&&ue(function(e){n.disconnectedMatch=m.call(e,"*"),m.call(e,"[s!='']:x"),v.push("!=",W)}),y=y.length&&new RegExp(y.join("|")),v=v.length&&new RegExp(v.join("|")),t=Q.test(h.compareDocumentPosition),x=t||Q.test(h.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return f=!0,0;var r=!e.compareDocumentPosition-!t.compareDocumentPosition;return r||(1&(r=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!n.sortDetached&&t.compareDocumentPosition(e)===r?e===d||e.ownerDocument===w&&x(w,e)?-1:t===d||t.ownerDocument===w&&x(w,t)?1:c?O(c,e)-O(c,t):0:4&r?-1:1)}:function(e,t){if(e===t)return f=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===d?-1:t===d?1:i?-1:o?1:c?O(c,e)-O(c,t):0;if(i===o)return ce(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?ce(a[r],s[r]):a[r]===w?-1:s[r]===w?1:0},d):d},oe.matches=function(e,t){return oe(e,null,null,t)},oe.matchesSelector=function(e,t){if((e.ownerDocument||e)!==d&&p(e),t=t.replace(z,"='$1']"),n.matchesSelector&&g&&!S[t+" "]&&(!v||!v.test(t))&&(!y||!y.test(t)))try{var r=m.call(e,t);if(r||n.disconnectedMatch||e.document&&11!==e.document.nodeType)return r}catch(e){}return oe(t,d,null,[e]).length>0},oe.contains=function(e,t){return(e.ownerDocument||e)!==d&&p(e),x(e,t)},oe.attr=function(e,t){(e.ownerDocument||e)!==d&&p(e);var i=r.attrHandle[t.toLowerCase()],o=i&&N.call(r.attrHandle,t.toLowerCase())?i(e,t,!g):void 0;return void 0!==o?o:n.attributes||!g?e.getAttribute(t):(o=e.getAttributeNode(t))&&o.specified?o.value:null},oe.escape=function(e){return(e+"").replace(te,ne)},oe.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},oe.uniqueSort=function(e){var t,r=[],i=0,o=0;if(f=!n.detectDuplicates,c=!n.sortStable&&e.slice(0),e.sort(D),f){while(t=e[o++])t===e[o]&&(i=r.push(o));while(i--)e.splice(r[i],1)}return c=null,e},i=oe.getText=function(e){var t,n="",r=0,o=e.nodeType;if(o){if(1===o||9===o||11===o){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=i(e)}else if(3===o||4===o)return e.nodeValue}else while(t=e[r++])n+=i(t);return n},(r=oe.selectors={cacheLength:50,createPseudo:se,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(Z,ee),e[3]=(e[3]||e[4]||e[5]||"").replace(Z,ee),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||oe.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&oe.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return V.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=a(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(Z,ee).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=E[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&E(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=oe.attr(r,e);return null==i?"!="===t:!t||(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i.replace($," ")+" ").indexOf(n)>-1:"|="===t&&(i===n||i.slice(0,n.length+1)===n+"-"))}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,u){var l,c,f,p,d,h,g=o!==a?"nextSibling":"previousSibling",y=t.parentNode,v=s&&t.nodeName.toLowerCase(),m=!u&&!s,x=!1;if(y){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===v:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?y.firstChild:y.lastChild],a&&m){x=(d=(l=(c=(f=(p=y)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1])&&l[2],p=d&&y.childNodes[d];while(p=++d&&p&&p[g]||(x=d=0)||h.pop())if(1===p.nodeType&&++x&&p===t){c[e]=[T,d,x];break}}else if(m&&(x=d=(l=(c=(f=(p=t)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1]),!1===x)while(p=++d&&p&&p[g]||(x=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===v:1===p.nodeType)&&++x&&(m&&((c=(f=p[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]=[T,x]),p===t))break;return(x-=i)===r||x%r==0&&x/r>=0}}},PSEUDO:function(e,t){var n,i=r.pseudos[e]||r.setFilters[e.toLowerCase()]||oe.error("unsupported pseudo: "+e);return i[b]?i(t):i.length>1?(n=[e,e,"",t],r.setFilters.hasOwnProperty(e.toLowerCase())?se(function(e,n){var r,o=i(e,t),a=o.length;while(a--)e[r=O(e,o[a])]=!(n[r]=o[a])}):function(e){return i(e,0,n)}):i}},pseudos:{not:se(function(e){var t=[],n=[],r=s(e.replace(B,"$1"));return r[b]?se(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),t[0]=null,!n.pop()}}),has:se(function(e){return function(t){return oe(e,t).length>0}}),contains:se(function(e){return e=e.replace(Z,ee),function(t){return(t.textContent||t.innerText||i(t)).indexOf(e)>-1}}),lang:se(function(e){return U.test(e||"")||oe.error("unsupported lang: "+e),e=e.replace(Z,ee).toLowerCase(),function(t){var n;do{if(n=g?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return(n=n.toLowerCase())===e||0===n.indexOf(e+"-")}while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===h},focus:function(e){return e===d.activeElement&&(!d.hasFocus||d.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:de(!1),disabled:de(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!r.pseudos.empty(e)},header:function(e){return Y.test(e.nodeName)},input:function(e){return G.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:he(function(){return[0]}),last:he(function(e,t){return[t-1]}),eq:he(function(e,t,n){return[n<0?n+t:n]}),even:he(function(e,t){for(var n=0;n=0;)e.push(r);return e}),gt:he(function(e,t,n){for(var r=n<0?n+t:n;++r1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function be(e,t,n){for(var r=0,i=t.length;r-1&&(o[l]=!(a[l]=f))}}else v=we(v===a?v.splice(h,v.length):v),i?i(null,a,v,u):L.apply(a,v)})}function Ce(e){for(var t,n,i,o=e.length,a=r.relative[e[0].type],s=a||r.relative[" "],u=a?1:0,c=me(function(e){return e===t},s,!0),f=me(function(e){return O(t,e)>-1},s,!0),p=[function(e,n,r){var i=!a&&(r||n!==l)||((t=n).nodeType?c(e,n,r):f(e,n,r));return t=null,i}];u1&&xe(p),u>1&&ve(e.slice(0,u-1).concat({value:" "===e[u-2].type?"*":""})).replace(B,"$1"),n,u0,i=e.length>0,o=function(o,a,s,u,c){var f,h,y,v=0,m="0",x=o&&[],b=[],w=l,C=o||i&&r.find.TAG("*",c),E=T+=null==w?1:Math.random()||.1,k=C.length;for(c&&(l=a===d||a||c);m!==k&&null!=(f=C[m]);m++){if(i&&f){h=0,a||f.ownerDocument===d||(p(f),s=!g);while(y=e[h++])if(y(f,a||d,s)){u.push(f);break}c&&(T=E)}n&&((f=!y&&f)&&v--,o&&x.push(f))}if(v+=m,n&&m!==v){h=0;while(y=t[h++])y(x,b,a,s);if(o){if(v>0)while(m--)x[m]||b[m]||(b[m]=j.call(u));b=we(b)}L.apply(u,b),c&&!o&&b.length>0&&v+t.length>1&&oe.uniqueSort(u)}return c&&(T=E,l=w),x};return n?se(o):o}return s=oe.compile=function(e,t){var n,r=[],i=[],o=S[e+" "];if(!o){t||(t=a(e)),n=t.length;while(n--)(o=Ce(t[n]))[b]?r.push(o):i.push(o);(o=S(e,Ee(i,r))).selector=e}return o},u=oe.select=function(e,t,n,i){var o,u,l,c,f,p="function"==typeof e&&e,d=!i&&a(e=p.selector||e);if(n=n||[],1===d.length){if((u=d[0]=d[0].slice(0)).length>2&&"ID"===(l=u[0]).type&&9===t.nodeType&&g&&r.relative[u[1].type]){if(!(t=(r.find.ID(l.matches[0].replace(Z,ee),t)||[])[0]))return n;p&&(t=t.parentNode),e=e.slice(u.shift().value.length)}o=V.needsContext.test(e)?0:u.length;while(o--){if(l=u[o],r.relative[c=l.type])break;if((f=r.find[c])&&(i=f(l.matches[0].replace(Z,ee),K.test(u[0].type)&&ge(t.parentNode)||t))){if(u.splice(o,1),!(e=i.length&&ve(u)))return L.apply(n,i),n;break}}}return(p||s(e,d))(i,t,!g,n,!t||K.test(e)&&ge(t.parentNode)||t),n},n.sortStable=b.split("").sort(D).join("")===b,n.detectDuplicates=!!f,p(),n.sortDetached=ue(function(e){return 1&e.compareDocumentPosition(d.createElement("fieldset"))}),ue(function(e){return e.innerHTML="","#"===e.firstChild.getAttribute("href")})||le("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),n.attributes&&ue(function(e){return e.innerHTML="",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||le("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ue(function(e){return null==e.getAttribute("disabled")})||le(P,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),oe}(e);w.find=E,w.expr=E.selectors,w.expr[":"]=w.expr.pseudos,w.uniqueSort=w.unique=E.uniqueSort,w.text=E.getText,w.isXMLDoc=E.isXML,w.contains=E.contains,w.escapeSelector=E.escape;var k=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&w(e).is(n))break;r.push(e)}return r},S=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},D=w.expr.match.needsContext;function N(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var A=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,t,n){return g(t)?w.grep(e,function(e,r){return!!t.call(e,r,e)!==n}):t.nodeType?w.grep(e,function(e){return e===t!==n}):"string"!=typeof t?w.grep(e,function(e){return u.call(t,e)>-1!==n}):w.filter(t,e,n)}w.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?w.find.matchesSelector(r,e)?[r]:[]:w.find.matches(e,w.grep(t,function(e){return 1===e.nodeType}))},w.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(w(e).filter(function(){for(t=0;t1?w.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&D.test(e)?w(e):e||[],!1).length}});var q,L=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(w.fn.init=function(e,t,n){var i,o;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(i="<"===e[0]&&">"===e[e.length-1]&&e.length>=3?[null,e,null]:L.exec(e))||!i[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(i[1]){if(t=t instanceof w?t[0]:t,w.merge(this,w.parseHTML(i[1],t&&t.nodeType?t.ownerDocument||t:r,!0)),A.test(i[1])&&w.isPlainObject(t))for(i in t)g(this[i])?this[i](t[i]):this.attr(i,t[i]);return this}return(o=r.getElementById(i[2]))&&(this[0]=o,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):g(e)?void 0!==n.ready?n.ready(e):e(w):w.makeArray(e,this)}).prototype=w.fn,q=w(r);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};w.fn.extend({has:function(e){var t=w(e,this),n=t.length;return this.filter(function(){for(var e=0;e-1:1===n.nodeType&&w.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(o.length>1?w.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?u.call(w(e),this[0]):u.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(w.uniqueSort(w.merge(this.get(),w(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}w.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return k(e,"parentNode")},parentsUntil:function(e,t,n){return k(e,"parentNode",n)},next:function(e){return P(e,"nextSibling")},prev:function(e){return P(e,"previousSibling")},nextAll:function(e){return k(e,"nextSibling")},prevAll:function(e){return k(e,"previousSibling")},nextUntil:function(e,t,n){return k(e,"nextSibling",n)},prevUntil:function(e,t,n){return k(e,"previousSibling",n)},siblings:function(e){return S((e.parentNode||{}).firstChild,e)},children:function(e){return S(e.firstChild)},contents:function(e){return N(e,"iframe")?e.contentDocument:(N(e,"template")&&(e=e.content||e),w.merge([],e.childNodes))}},function(e,t){w.fn[e]=function(n,r){var i=w.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=w.filter(r,i)),this.length>1&&(O[e]||w.uniqueSort(i),H.test(e)&&i.reverse()),this.pushStack(i)}});var M=/[^\x20\t\r\n\f]+/g;function R(e){var t={};return w.each(e.match(M)||[],function(e,n){t[n]=!0}),t}w.Callbacks=function(e){e="string"==typeof e?R(e):w.extend({},e);var t,n,r,i,o=[],a=[],s=-1,u=function(){for(i=i||e.once,r=t=!0;a.length;s=-1){n=a.shift();while(++s-1)o.splice(n,1),n<=s&&s--}),this},has:function(e){return e?w.inArray(e,o)>-1:o.length>0},empty:function(){return o&&(o=[]),this},disable:function(){return i=a=[],o=n="",this},disabled:function(){return!o},lock:function(){return i=a=[],n||t||(o=n=""),this},locked:function(){return!!i},fireWith:function(e,n){return i||(n=[e,(n=n||[]).slice?n.slice():n],a.push(n),t||u()),this},fire:function(){return l.fireWith(this,arguments),this},fired:function(){return!!r}};return l};function I(e){return e}function W(e){throw e}function $(e,t,n,r){var i;try{e&&g(i=e.promise)?i.call(e).done(t).fail(n):e&&g(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}w.extend({Deferred:function(t){var n=[["notify","progress",w.Callbacks("memory"),w.Callbacks("memory"),2],["resolve","done",w.Callbacks("once memory"),w.Callbacks("once memory"),0,"resolved"],["reject","fail",w.Callbacks("once memory"),w.Callbacks("once memory"),1,"rejected"]],r="pending",i={state:function(){return r},always:function(){return o.done(arguments).fail(arguments),this},"catch":function(e){return i.then(null,e)},pipe:function(){var e=arguments;return w.Deferred(function(t){w.each(n,function(n,r){var i=g(e[r[4]])&&e[r[4]];o[r[1]](function(){var e=i&&i.apply(this,arguments);e&&g(e.promise)?e.promise().progress(t.notify).done(t.resolve).fail(t.reject):t[r[0]+"With"](this,i?[e]:arguments)})}),e=null}).promise()},then:function(t,r,i){var o=0;function a(t,n,r,i){return function(){var s=this,u=arguments,l=function(){var e,l;if(!(t=o&&(r!==W&&(s=void 0,u=[e]),n.rejectWith(s,u))}};t?c():(w.Deferred.getStackHook&&(c.stackTrace=w.Deferred.getStackHook()),e.setTimeout(c))}}return w.Deferred(function(e){n[0][3].add(a(0,e,g(i)?i:I,e.notifyWith)),n[1][3].add(a(0,e,g(t)?t:I)),n[2][3].add(a(0,e,g(r)?r:W))}).promise()},promise:function(e){return null!=e?w.extend(e,i):i}},o={};return w.each(n,function(e,t){var a=t[2],s=t[5];i[t[1]]=a.add,s&&a.add(function(){r=s},n[3-e][2].disable,n[3-e][3].disable,n[0][2].lock,n[0][3].lock),a.add(t[3].fire),o[t[0]]=function(){return o[t[0]+"With"](this===o?void 0:this,arguments),this},o[t[0]+"With"]=a.fireWith}),i.promise(o),t&&t.call(o,o),o},when:function(e){var t=arguments.length,n=t,r=Array(n),i=o.call(arguments),a=w.Deferred(),s=function(e){return function(n){r[e]=this,i[e]=arguments.length>1?o.call(arguments):n,--t||a.resolveWith(r,i)}};if(t<=1&&($(e,a.done(s(n)).resolve,a.reject,!t),"pending"===a.state()||g(i[n]&&i[n].then)))return a.then();while(n--)$(i[n],s(n),a.reject);return a.promise()}});var B=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;w.Deferred.exceptionHook=function(t,n){e.console&&e.console.warn&&t&&B.test(t.name)&&e.console.warn("jQuery.Deferred exception: "+t.message,t.stack,n)},w.readyException=function(t){e.setTimeout(function(){throw t})};var F=w.Deferred();w.fn.ready=function(e){return F.then(e)["catch"](function(e){w.readyException(e)}),this},w.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--w.readyWait:w.isReady)||(w.isReady=!0,!0!==e&&--w.readyWait>0||F.resolveWith(r,[w]))}}),w.ready.then=F.then;function _(){r.removeEventListener("DOMContentLoaded",_),e.removeEventListener("load",_),w.ready()}"complete"===r.readyState||"loading"!==r.readyState&&!r.documentElement.doScroll?e.setTimeout(w.ready):(r.addEventListener("DOMContentLoaded",_),e.addEventListener("load",_));var z=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===x(n)){i=!0;for(s in n)z(e,t,s,n[s],!0,o,a)}else if(void 0!==r&&(i=!0,g(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(w(e),n)})),t))for(;s1,null,!0)},removeData:function(e){return this.each(function(){K.remove(this,e)})}}),w.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=J.get(e,t),n&&(!r||Array.isArray(n)?r=J.access(e,t,w.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=w.queue(e,t),r=n.length,i=n.shift(),o=w._queueHooks(e,t),a=function(){w.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return J.get(e,n)||J.access(e,n,{empty:w.Callbacks("once memory").add(function(){J.remove(e,[t+"queue",n])})})}}),w.fn.extend({queue:function(e,t){var n=2;return"string"!=typeof e&&(t=e,e="fx",n--),arguments.length\x20\t\r\n\f]+)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};ge.optgroup=ge.option,ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td;function ye(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&N(e,t)?w.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n-1)i&&i.push(o);else if(l=w.contains(o.ownerDocument,o),a=ye(f.appendChild(o),"script"),l&&ve(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}!function(){var e=r.createDocumentFragment().appendChild(r.createElement("div")),t=r.createElement("input");t.setAttribute("type","radio"),t.setAttribute("checked","checked"),t.setAttribute("name","t"),e.appendChild(t),h.checkClone=e.cloneNode(!0).cloneNode(!0).lastChild.checked,e.innerHTML="",h.noCloneChecked=!!e.cloneNode(!0).lastChild.defaultValue}();var be=r.documentElement,we=/^key/,Te=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ce=/^([^.]*)(?:\.(.+)|)/;function Ee(){return!0}function ke(){return!1}function Se(){try{return r.activeElement}catch(e){}}function De(e,t,n,r,i,o){var a,s;if("object"==typeof t){"string"!=typeof n&&(r=r||n,n=void 0);for(s in t)De(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=ke;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return w().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=w.guid++)),e.each(function(){w.event.add(this,t,i,r,n)})}w.event={global:{},add:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,y=J.get(e);if(y){n.handler&&(n=(o=n).handler,i=o.selector),i&&w.find.matchesSelector(be,i),n.guid||(n.guid=w.guid++),(u=y.events)||(u=y.events={}),(a=y.handle)||(a=y.handle=function(t){return"undefined"!=typeof w&&w.event.triggered!==t.type?w.event.dispatch.apply(e,arguments):void 0}),l=(t=(t||"").match(M)||[""]).length;while(l--)d=g=(s=Ce.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=w.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=w.event.special[d]||{},c=w.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&w.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(e,r,h,a)||e.addEventListener&&e.addEventListener(d,a)),f.add&&(f.add.call(e,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),w.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,y=J.hasData(e)&&J.get(e);if(y&&(u=y.events)){l=(t=(t||"").match(M)||[""]).length;while(l--)if(s=Ce.exec(t[l])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){f=w.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,y.handle)||w.removeEvent(e,d,y.handle),delete u[d])}else for(d in u)w.event.remove(e,d+t[l],n,r,!0);w.isEmptyObject(u)&&J.remove(e,"handle events")}},dispatch:function(e){var t=w.event.fix(e),n,r,i,o,a,s,u=new Array(arguments.length),l=(J.get(this,"events")||{})[t.type]||[],c=w.event.special[t.type]||{};for(u[0]=t,n=1;n=1))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n-1:w.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u\x20\t\r\n\f]*)[^>]*)\/>/gi,Ae=/\s*$/g;function Le(e,t){return N(e,"table")&&N(11!==t.nodeType?t:t.firstChild,"tr")?w(e).children("tbody")[0]||e:e}function He(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Oe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Pe(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(J.hasData(e)&&(o=J.access(e),a=J.set(t,o),l=o.events)){delete a.handle,a.events={};for(i in l)for(n=0,r=l[i].length;n1&&"string"==typeof y&&!h.checkClone&&je.test(y))return e.each(function(i){var o=e.eq(i);v&&(t[0]=y.call(this,i,o.html())),Re(o,t,n,r)});if(p&&(i=xe(t,e[0].ownerDocument,!1,e,r),o=i.firstChild,1===i.childNodes.length&&(i=o),o||r)){for(u=(s=w.map(ye(i,"script"),He)).length;f")},clone:function(e,t,n){var r,i,o,a,s=e.cloneNode(!0),u=w.contains(e.ownerDocument,e);if(!(h.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||w.isXMLDoc(e)))for(a=ye(s),r=0,i=(o=ye(e)).length;r0&&ve(a,!u&&ye(e,"script")),s},cleanData:function(e){for(var t,n,r,i=w.event.special,o=0;void 0!==(n=e[o]);o++)if(Y(n)){if(t=n[J.expando]){if(t.events)for(r in t.events)i[r]?w.event.remove(n,r):w.removeEvent(n,r,t.handle);n[J.expando]=void 0}n[K.expando]&&(n[K.expando]=void 0)}}}),w.fn.extend({detach:function(e){return Ie(this,e,!0)},remove:function(e){return Ie(this,e)},text:function(e){return z(this,function(e){return void 0===e?w.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Re(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Le(this,e).appendChild(e)})},prepend:function(){return Re(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Le(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Re(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Re(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(w.cleanData(ye(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return w.clone(this,e,t)})},html:function(e){return z(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ae.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=w.htmlPrefilter(e);try{for(;n=0&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))),u}function et(e,t,n){var r=$e(e),i=Fe(e,t,r),o="border-box"===w.css(e,"boxSizing",!1,r),a=o;if(We.test(i)){if(!n)return i;i="auto"}return a=a&&(h.boxSizingReliable()||i===e.style[t]),("auto"===i||!parseFloat(i)&&"inline"===w.css(e,"display",!1,r))&&(i=e["offset"+t[0].toUpperCase()+t.slice(1)],a=!0),(i=parseFloat(i)||0)+Ze(e,t,n||(o?"border":"content"),a,r,i)+"px"}w.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Fe(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=G(t),u=Xe.test(t),l=e.style;if(u||(t=Je(s)),a=w.cssHooks[t]||w.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"==(o=typeof n)&&(i=ie.exec(n))&&i[1]&&(n=ue(e,t,i),o="number"),null!=n&&n===n&&("number"===o&&(n+=i&&i[3]||(w.cssNumber[s]?"":"px")),h.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=G(t);return Xe.test(t)||(t=Je(s)),(a=w.cssHooks[t]||w.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Fe(e,t,r)),"normal"===i&&t in Ve&&(i=Ve[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),w.each(["height","width"],function(e,t){w.cssHooks[t]={get:function(e,n,r){if(n)return!ze.test(w.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?et(e,t,r):se(e,Ue,function(){return et(e,t,r)})},set:function(e,n,r){var i,o=$e(e),a="border-box"===w.css(e,"boxSizing",!1,o),s=r&&Ze(e,t,r,a,o);return a&&h.scrollboxSize()===o.position&&(s-=Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-parseFloat(o[t])-Ze(e,t,"border",!1,o)-.5)),s&&(i=ie.exec(n))&&"px"!==(i[3]||"px")&&(e.style[t]=n,n=w.css(e,t)),Ke(e,n,s)}}}),w.cssHooks.marginLeft=_e(h.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Fe(e,"marginLeft"))||e.getBoundingClientRect().left-se(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),w.each({margin:"",padding:"",border:"Width"},function(e,t){w.cssHooks[e+t]={expand:function(n){for(var r=0,i={},o="string"==typeof n?n.split(" "):[n];r<4;r++)i[e+oe[r]+t]=o[r]||o[r-2]||o[0];return i}},"margin"!==e&&(w.cssHooks[e+t].set=Ke)}),w.fn.extend({css:function(e,t){return z(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=$e(e),i=t.length;a1)}});function tt(e,t,n,r,i){return new tt.prototype.init(e,t,n,r,i)}w.Tween=tt,tt.prototype={constructor:tt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||w.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(w.cssNumber[n]?"":"px")},cur:function(){var e=tt.propHooks[this.prop];return e&&e.get?e.get(this):tt.propHooks._default.get(this)},run:function(e){var t,n=tt.propHooks[this.prop];return this.options.duration?this.pos=t=w.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):tt.propHooks._default.set(this),this}},tt.prototype.init.prototype=tt.prototype,tt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=w.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){w.fx.step[e.prop]?w.fx.step[e.prop](e):1!==e.elem.nodeType||null==e.elem.style[w.cssProps[e.prop]]&&!w.cssHooks[e.prop]?e.elem[e.prop]=e.now:w.style(e.elem,e.prop,e.now+e.unit)}}},tt.propHooks.scrollTop=tt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},w.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},w.fx=tt.prototype.init,w.fx.step={};var nt,rt,it=/^(?:toggle|show|hide)$/,ot=/queueHooks$/;function at(){rt&&(!1===r.hidden&&e.requestAnimationFrame?e.requestAnimationFrame(at):e.setTimeout(at,w.fx.interval),w.fx.tick())}function st(){return e.setTimeout(function(){nt=void 0}),nt=Date.now()}function ut(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=oe[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function lt(e,t,n){for(var r,i=(pt.tweeners[t]||[]).concat(pt.tweeners["*"]),o=0,a=i.length;o1)},removeAttr:function(e){return this.each(function(){w.removeAttr(this,e)})}}),w.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?w.prop(e,t,n):(1===o&&w.isXMLDoc(e)||(i=w.attrHooks[t.toLowerCase()]||(w.expr.match.bool.test(t)?dt:void 0)),void 0!==n?null===n?void w.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=w.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!h.radioValue&&"radio"===t&&N(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(M);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),dt={set:function(e,t,n){return!1===t?w.removeAttr(e,n):e.setAttribute(n,n),n}},w.each(w.expr.match.bool.source.match(/\w+/g),function(e,t){var n=ht[t]||w.find.attr;ht[t]=function(e,t,r){var i,o,a=t.toLowerCase();return r||(o=ht[a],ht[a]=i,i=null!=n(e,t,r)?a:null,ht[a]=o),i}});var gt=/^(?:input|select|textarea|button)$/i,yt=/^(?:a|area)$/i;w.fn.extend({prop:function(e,t){return z(this,w.prop,e,t,arguments.length>1)},removeProp:function(e){return this.each(function(){delete this[w.propFix[e]||e]})}}),w.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&w.isXMLDoc(e)||(t=w.propFix[t]||t,i=w.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=w.find.attr(e,"tabindex");return t?parseInt(t,10):gt.test(e.nodeName)||yt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),h.optSelected||(w.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),w.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){w.propFix[this.toLowerCase()]=this});function vt(e){return(e.match(M)||[]).join(" ")}function mt(e){return e.getAttribute&&e.getAttribute("class")||""}function xt(e){return Array.isArray(e)?e:"string"==typeof e?e.match(M)||[]:[]}w.fn.extend({addClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){w(this).addClass(e.call(this,t,mt(this)))});if((t=xt(e)).length)while(n=this[u++])if(i=mt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=t[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){w(this).removeClass(e.call(this,t,mt(this)))});if(!arguments.length)return this.attr("class","");if((t=xt(e)).length)while(n=this[u++])if(i=mt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=t[a++])while(r.indexOf(" "+o+" ")>-1)r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(e,t){var n=typeof e,r="string"===n||Array.isArray(e);return"boolean"==typeof t&&r?t?this.addClass(e):this.removeClass(e):g(e)?this.each(function(n){w(this).toggleClass(e.call(this,n,mt(this),t),t)}):this.each(function(){var t,i,o,a;if(r){i=0,o=w(this),a=xt(e);while(t=a[i++])o.hasClass(t)?o.removeClass(t):o.addClass(t)}else void 0!==e&&"boolean"!==n||((t=mt(this))&&J.set(this,"__className__",t),this.setAttribute&&this.setAttribute("class",t||!1===e?"":J.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&(" "+vt(mt(n))+" ").indexOf(t)>-1)return!0;return!1}});var bt=/\r/g;w.fn.extend({val:function(e){var t,n,r,i=this[0];{if(arguments.length)return r=g(e),this.each(function(n){var i;1===this.nodeType&&(null==(i=r?e.call(this,n,w(this).val()):e)?i="":"number"==typeof i?i+="":Array.isArray(i)&&(i=w.map(i,function(e){return null==e?"":e+""})),(t=w.valHooks[this.type]||w.valHooks[this.nodeName.toLowerCase()])&&"set"in t&&void 0!==t.set(this,i,"value")||(this.value=i))});if(i)return(t=w.valHooks[i.type]||w.valHooks[i.nodeName.toLowerCase()])&&"get"in t&&void 0!==(n=t.get(i,"value"))?n:"string"==typeof(n=i.value)?n.replace(bt,""):null==n?"":n}}}),w.extend({valHooks:{option:{get:function(e){var t=w.find.attr(e,"value");return null!=t?t:vt(w.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r-1)&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),w.each(["radio","checkbox"],function(){w.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=w.inArray(w(e).val(),t)>-1}},h.checkOn||(w.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),h.focusin="onfocusin"in e;var wt=/^(?:focusinfocus|focusoutblur)$/,Tt=function(e){e.stopPropagation()};w.extend(w.event,{trigger:function(t,n,i,o){var a,s,u,l,c,p,d,h,v=[i||r],m=f.call(t,"type")?t.type:t,x=f.call(t,"namespace")?t.namespace.split("."):[];if(s=h=u=i=i||r,3!==i.nodeType&&8!==i.nodeType&&!wt.test(m+w.event.triggered)&&(m.indexOf(".")>-1&&(m=(x=m.split(".")).shift(),x.sort()),c=m.indexOf(":")<0&&"on"+m,t=t[w.expando]?t:new w.Event(m,"object"==typeof t&&t),t.isTrigger=o?2:3,t.namespace=x.join("."),t.rnamespace=t.namespace?new RegExp("(^|\\.)"+x.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,t.result=void 0,t.target||(t.target=i),n=null==n?[t]:w.makeArray(n,[t]),d=w.event.special[m]||{},o||!d.trigger||!1!==d.trigger.apply(i,n))){if(!o&&!d.noBubble&&!y(i)){for(l=d.delegateType||m,wt.test(l+m)||(s=s.parentNode);s;s=s.parentNode)v.push(s),u=s;u===(i.ownerDocument||r)&&v.push(u.defaultView||u.parentWindow||e)}a=0;while((s=v[a++])&&!t.isPropagationStopped())h=s,t.type=a>1?l:d.bindType||m,(p=(J.get(s,"events")||{})[t.type]&&J.get(s,"handle"))&&p.apply(s,n),(p=c&&s[c])&&p.apply&&Y(s)&&(t.result=p.apply(s,n),!1===t.result&&t.preventDefault());return t.type=m,o||t.isDefaultPrevented()||d._default&&!1!==d._default.apply(v.pop(),n)||!Y(i)||c&&g(i[m])&&!y(i)&&((u=i[c])&&(i[c]=null),w.event.triggered=m,t.isPropagationStopped()&&h.addEventListener(m,Tt),i[m](),t.isPropagationStopped()&&h.removeEventListener(m,Tt),w.event.triggered=void 0,u&&(i[c]=u)),t.result}},simulate:function(e,t,n){var r=w.extend(new w.Event,n,{type:e,isSimulated:!0});w.event.trigger(r,null,t)}}),w.fn.extend({trigger:function(e,t){return this.each(function(){w.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return w.event.trigger(e,t,n,!0)}}),h.focusin||w.each({focus:"focusin",blur:"focusout"},function(e,t){var n=function(e){w.event.simulate(t,e.target,w.event.fix(e))};w.event.special[t]={setup:function(){var r=this.ownerDocument||this,i=J.access(r,t);i||r.addEventListener(e,n,!0),J.access(r,t,(i||0)+1)},teardown:function(){var r=this.ownerDocument||this,i=J.access(r,t)-1;i?J.access(r,t,i):(r.removeEventListener(e,n,!0),J.remove(r,t))}}});var Ct=e.location,Et=Date.now(),kt=/\?/;w.parseXML=function(t){var n;if(!t||"string"!=typeof t)return null;try{n=(new e.DOMParser).parseFromString(t,"text/xml")}catch(e){n=void 0}return n&&!n.getElementsByTagName("parsererror").length||w.error("Invalid XML: "+t),n};var St=/\[\]$/,Dt=/\r?\n/g,Nt=/^(?:submit|button|image|reset|file)$/i,At=/^(?:input|select|textarea|keygen)/i;function jt(e,t,n,r){var i;if(Array.isArray(t))w.each(t,function(t,i){n||St.test(e)?r(e,i):jt(e+"["+("object"==typeof i&&null!=i?t:"")+"]",i,n,r)});else if(n||"object"!==x(t))r(e,t);else for(i in t)jt(e+"["+i+"]",t[i],n,r)}w.param=function(e,t){var n,r=[],i=function(e,t){var n=g(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(Array.isArray(e)||e.jquery&&!w.isPlainObject(e))w.each(e,function(){i(this.name,this.value)});else for(n in e)jt(n,e[n],t,i);return r.join("&")},w.fn.extend({serialize:function(){return w.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=w.prop(this,"elements");return e?w.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!w(this).is(":disabled")&&At.test(this.nodeName)&&!Nt.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=w(this).val();return null==n?null:Array.isArray(n)?w.map(n,function(e){return{name:t.name,value:e.replace(Dt,"\r\n")}}):{name:t.name,value:n.replace(Dt,"\r\n")}}).get()}});var qt=/%20/g,Lt=/#.*$/,Ht=/([?&])_=[^&]*/,Ot=/^(.*?):[ \t]*([^\r\n]*)$/gm,Pt=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Mt=/^(?:GET|HEAD)$/,Rt=/^\/\//,It={},Wt={},$t="*/".concat("*"),Bt=r.createElement("a");Bt.href=Ct.href;function Ft(e){return function(t,n){"string"!=typeof t&&(n=t,t="*");var r,i=0,o=t.toLowerCase().match(M)||[];if(g(n))while(r=o[i++])"+"===r[0]?(r=r.slice(1)||"*",(e[r]=e[r]||[]).unshift(n)):(e[r]=e[r]||[]).push(n)}}function _t(e,t,n,r){var i={},o=e===Wt;function a(s){var u;return i[s]=!0,w.each(e[s]||[],function(e,s){var l=s(t,n,r);return"string"!=typeof l||o||i[l]?o?!(u=l):void 0:(t.dataTypes.unshift(l),a(l),!1)}),u}return a(t.dataTypes[0])||!i["*"]&&a("*")}function zt(e,t){var n,r,i=w.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&w.extend(!0,e,r),e}function Xt(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}function Ut(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}w.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Ct.href,type:"GET",isLocal:Pt.test(Ct.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":$t,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":w.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?zt(zt(e,w.ajaxSettings),t):zt(w.ajaxSettings,e)},ajaxPrefilter:Ft(It),ajaxTransport:Ft(Wt),ajax:function(t,n){"object"==typeof t&&(n=t,t=void 0),n=n||{};var i,o,a,s,u,l,c,f,p,d,h=w.ajaxSetup({},n),g=h.context||h,y=h.context&&(g.nodeType||g.jquery)?w(g):w.event,v=w.Deferred(),m=w.Callbacks("once memory"),x=h.statusCode||{},b={},T={},C="canceled",E={readyState:0,getResponseHeader:function(e){var t;if(c){if(!s){s={};while(t=Ot.exec(a))s[t[1].toLowerCase()]=t[2]}t=s[e.toLowerCase()]}return null==t?null:t},getAllResponseHeaders:function(){return c?a:null},setRequestHeader:function(e,t){return null==c&&(e=T[e.toLowerCase()]=T[e.toLowerCase()]||e,b[e]=t),this},overrideMimeType:function(e){return null==c&&(h.mimeType=e),this},statusCode:function(e){var t;if(e)if(c)E.always(e[E.status]);else for(t in e)x[t]=[x[t],e[t]];return this},abort:function(e){var t=e||C;return i&&i.abort(t),k(0,t),this}};if(v.promise(E),h.url=((t||h.url||Ct.href)+"").replace(Rt,Ct.protocol+"//"),h.type=n.method||n.type||h.method||h.type,h.dataTypes=(h.dataType||"*").toLowerCase().match(M)||[""],null==h.crossDomain){l=r.createElement("a");try{l.href=h.url,l.href=l.href,h.crossDomain=Bt.protocol+"//"+Bt.host!=l.protocol+"//"+l.host}catch(e){h.crossDomain=!0}}if(h.data&&h.processData&&"string"!=typeof h.data&&(h.data=w.param(h.data,h.traditional)),_t(It,h,n,E),c)return E;(f=w.event&&h.global)&&0==w.active++&&w.event.trigger("ajaxStart"),h.type=h.type.toUpperCase(),h.hasContent=!Mt.test(h.type),o=h.url.replace(Lt,""),h.hasContent?h.data&&h.processData&&0===(h.contentType||"").indexOf("application/x-www-form-urlencoded")&&(h.data=h.data.replace(qt,"+")):(d=h.url.slice(o.length),h.data&&(h.processData||"string"==typeof h.data)&&(o+=(kt.test(o)?"&":"?")+h.data,delete h.data),!1===h.cache&&(o=o.replace(Ht,"$1"),d=(kt.test(o)?"&":"?")+"_="+Et+++d),h.url=o+d),h.ifModified&&(w.lastModified[o]&&E.setRequestHeader("If-Modified-Since",w.lastModified[o]),w.etag[o]&&E.setRequestHeader("If-None-Match",w.etag[o])),(h.data&&h.hasContent&&!1!==h.contentType||n.contentType)&&E.setRequestHeader("Content-Type",h.contentType),E.setRequestHeader("Accept",h.dataTypes[0]&&h.accepts[h.dataTypes[0]]?h.accepts[h.dataTypes[0]]+("*"!==h.dataTypes[0]?", "+$t+"; q=0.01":""):h.accepts["*"]);for(p in h.headers)E.setRequestHeader(p,h.headers[p]);if(h.beforeSend&&(!1===h.beforeSend.call(g,E,h)||c))return E.abort();if(C="abort",m.add(h.complete),E.done(h.success),E.fail(h.error),i=_t(Wt,h,n,E)){if(E.readyState=1,f&&y.trigger("ajaxSend",[E,h]),c)return E;h.async&&h.timeout>0&&(u=e.setTimeout(function(){E.abort("timeout")},h.timeout));try{c=!1,i.send(b,k)}catch(e){if(c)throw e;k(-1,e)}}else k(-1,"No Transport");function k(t,n,r,s){var l,p,d,b,T,C=n;c||(c=!0,u&&e.clearTimeout(u),i=void 0,a=s||"",E.readyState=t>0?4:0,l=t>=200&&t<300||304===t,r&&(b=Xt(h,E,r)),b=Ut(h,b,E,l),l?(h.ifModified&&((T=E.getResponseHeader("Last-Modified"))&&(w.lastModified[o]=T),(T=E.getResponseHeader("etag"))&&(w.etag[o]=T)),204===t||"HEAD"===h.type?C="nocontent":304===t?C="notmodified":(C=b.state,p=b.data,l=!(d=b.error))):(d=C,!t&&C||(C="error",t<0&&(t=0))),E.status=t,E.statusText=(n||C)+"",l?v.resolveWith(g,[p,C,E]):v.rejectWith(g,[E,C,d]),E.statusCode(x),x=void 0,f&&y.trigger(l?"ajaxSuccess":"ajaxError",[E,h,l?p:d]),m.fireWith(g,[E,C]),f&&(y.trigger("ajaxComplete",[E,h]),--w.active||w.event.trigger("ajaxStop")))}return E},getJSON:function(e,t,n){return w.get(e,t,n,"json")},getScript:function(e,t){return w.get(e,void 0,t,"script")}}),w.each(["get","post"],function(e,t){w[t]=function(e,n,r,i){return g(n)&&(i=i||r,r=n,n=void 0),w.ajax(w.extend({url:e,type:t,dataType:i,data:n,success:r},w.isPlainObject(e)&&e))}}),w._evalUrl=function(e){return w.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},w.fn.extend({wrapAll:function(e){var t;return this[0]&&(g(e)&&(e=e.call(this[0])),t=w(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(e){return g(e)?this.each(function(t){w(this).wrapInner(e.call(this,t))}):this.each(function(){var t=w(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=g(e);return this.each(function(n){w(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(e){return this.parent(e).not("body").each(function(){w(this).replaceWith(this.childNodes)}),this}}),w.expr.pseudos.hidden=function(e){return!w.expr.pseudos.visible(e)},w.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},w.ajaxSettings.xhr=function(){try{return new e.XMLHttpRequest}catch(e){}};var Vt={0:200,1223:204},Gt=w.ajaxSettings.xhr();h.cors=!!Gt&&"withCredentials"in Gt,h.ajax=Gt=!!Gt,w.ajaxTransport(function(t){var n,r;if(h.cors||Gt&&!t.crossDomain)return{send:function(i,o){var a,s=t.xhr();if(s.open(t.type,t.url,t.async,t.username,t.password),t.xhrFields)for(a in t.xhrFields)s[a]=t.xhrFields[a];t.mimeType&&s.overrideMimeType&&s.overrideMimeType(t.mimeType),t.crossDomain||i["X-Requested-With"]||(i["X-Requested-With"]="XMLHttpRequest");for(a in i)s.setRequestHeader(a,i[a]);n=function(e){return function(){n&&(n=r=s.onload=s.onerror=s.onabort=s.ontimeout=s.onreadystatechange=null,"abort"===e?s.abort():"error"===e?"number"!=typeof s.status?o(0,"error"):o(s.status,s.statusText):o(Vt[s.status]||s.status,s.statusText,"text"!==(s.responseType||"text")||"string"!=typeof s.responseText?{binary:s.response}:{text:s.responseText},s.getAllResponseHeaders()))}},s.onload=n(),r=s.onerror=s.ontimeout=n("error"),void 0!==s.onabort?s.onabort=r:s.onreadystatechange=function(){4===s.readyState&&e.setTimeout(function(){n&&r()})},n=n("abort");try{s.send(t.hasContent&&t.data||null)}catch(e){if(n)throw e}},abort:function(){n&&n()}}}),w.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),w.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return w.globalEval(e),e}}}),w.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),w.ajaxTransport("script",function(e){if(e.crossDomain){var t,n;return{send:function(i,o){t=w(" - - - - -
-
-
-
- - Logo - ArchiveBox: Add - -
-
- Main Index   |   - Admin   |   - Docs -
-
-
-
-
-

-
{% csrf_token %} - Add new links...
-
- -
-
- - + tr td a.favicon img { + padding-left: 6px; + padding-right: 12px; + vertical-align: -4px; + } + tr td a.title { + font-size: 1.4em; + text-decoration: none; + color: black; + } + tr td a.title small { + background-color: #efefef; + border-radius: 4px; + float: right; + } + input[type="search"]::-webkit-search-cancel-button { + -webkit-appearance: searchfield-cancel-button; + } + .title-col { + text-align: left; + } + .title-col a { + color: black; + } + + + + + + + + +
+
+
+
+ + Logo + ArchiveBox: Add + +
+
+ Main Index   |   + Admin   |   + Docs +
+
+
+
+
+ {{ stdout | safe }} +

+
+ {% csrf_token %} Add new links...
+
+ +
+ + Go back to Snapshot list +
+ diff --git a/archivebox/util.py b/archivebox/util.py index 87c98263..50511313 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -20,6 +20,7 @@ from .config import ( CHECK_SSL_VALIDITY, WGET_USER_AGENT, CHROME_OPTIONS, + COLOR_DICT ) try: @@ -69,6 +70,8 @@ URL_REGEX = re.compile( re.IGNORECASE, ) +COLOR_REGEX = re.compile(r'\[(?P\d+)(;(?P\d+)(;(?P\d+))?)?m') + def enforce_types(func): """ @@ -195,6 +198,27 @@ def chrome_args(**options) -> List[str]: return cmd_args +def ansi_to_html(text): + """ + Based on: https://stackoverflow.com/questions/19212665/python-converting-ansi-color-codes-to-html + """ + TEMPLATE = '
' + text = text.replace('[m', '') + + def single_sub(match): + argsdict = match.groupdict() + if argsdict['arg_3'] is None: + if argsdict['arg_2'] is None: + bold, color = 0, argsdict['arg_1'] + else: + bold, color = argsdict['arg_1'], argsdict['arg_2'] + else: + bold, color = argsdict['arg_3'], argsdict['arg_2'] + + return TEMPLATE.format(COLOR_DICT[color][0]) + + return COLOR_REGEX.sub(single_sub, text) + class ExtendedEncoder(pyjson.JSONEncoder): """ From 364c5752d827c87a927bed00e89e4e3d7c5b6e4a Mon Sep 17 00:00:00 2001 From: Cristian Date: Wed, 1 Jul 2020 12:29:56 -0500 Subject: [PATCH 211/365] feat: Handle empty URL case --- archivebox/core/views.py | 27 +- archivebox/themes/default/add_links.html | 426 +++++++++++------------ 2 files changed, 218 insertions(+), 235 deletions(-) diff --git a/archivebox/core/views.py b/archivebox/core/views.py index b7911674..5efa79cd 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -57,19 +57,22 @@ class AddLinks(View): def post(self, request): url = request.POST['url'] - print(f'[+] Adding URL: {url}') - add_stdout = StringIO() - with redirect_stdout(add_stdout): - extracted_links = add( - import_str=url, - update_all=False, - out_dir=OUTPUT_DIR, - ) - print(add_stdout.getvalue()) + if url: + print(f'[+] Adding URL: {url}') + add_stdout = StringIO() + with redirect_stdout(add_stdout): + extracted_links = add( + import_str=url, + update_all=False, + out_dir=OUTPUT_DIR, + ) + print(add_stdout.getvalue()) - context = { - "stdout": ansi_to_html(add_stdout.getvalue()) - } + context = { + "stdout": ansi_to_html(add_stdout.getvalue()) + } + else: + context = {"stdout": "Please enter a URL"} return render(template_name=self.template, request=request, context=context) diff --git a/archivebox/themes/default/add_links.html b/archivebox/themes/default/add_links.html index db09322f..6c625594 100644 --- a/archivebox/themes/default/add_links.html +++ b/archivebox/themes/default/add_links.html @@ -2,231 +2,211 @@ - - Archived Sites - - - - - - - - - -
-
-
-
- - Logo - ArchiveBox: Add - -
-
- Main Index   |   - Admin   |   - Docs -
-
-
-
-
- {{ stdout | safe }} -

-
- {% csrf_token %} Add new links...
-
- -
+ tr td a.favicon img { + padding-left: 6px; + padding-right: 12px; + vertical-align: -4px; + } + tr td a.title { + font-size: 1.4em; + text-decoration:none; + color:black; + } + tr td a.title small { + background-color: #efefef; + border-radius: 4px; + float:right + } + input[type=search]::-webkit-search-cancel-button { + -webkit-appearance: searchfield-cancel-button; + } + .title-col { + text-align: left; + } + .title-col a { + color: black; + } + + + + + + + + +
+
+
+
+ + Logo + ArchiveBox: Add + +
+
+ Main Index   |   + Admin   |   + Docs +
+
+
+
+
+ {{ stdout | safe }} +

+
{% csrf_token %} + Add new links...
+
+ +
+
- Go back to Snapshot list -
- + Go back to Snapshot list + + From 8840ad72bbc2006c9e02690b814b6524679ef79f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 03:12:30 -0400 Subject: [PATCH 212/365] remove circular import possibilities --- archivebox/config/__init__.py | 8 ++++++++ archivebox/core/admin.py | 2 +- archivebox/util.py | 25 ++++++++++++++----------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py index fa979211..f06b0f3d 100644 --- a/archivebox/config/__init__.py +++ b/archivebox/config/__init__.py @@ -21,6 +21,14 @@ from .stubs import ( ConfigDefaultDict, ) +# precedence order for config: +# 1. cli args +# 2. shell environment vars +# 3. config file +# 4. defaults + +# env USE_COLO=false archivebox add '...' +# env SHOW_PROGRESS=1 archivebox add '...' # ****************************************************************************** # Documentation: https://github.com/pirate/ArchiveBox/wiki/Configuration diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index 5cf71796..7942c6c2 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.utils.html import format_html -from archivebox.util import htmldecode, urldecode +from util import htmldecode, urldecode from core.models import Snapshot from cli.logging import printable_filesize diff --git a/archivebox/util.py b/archivebox/util.py index 50511313..717e1185 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -14,15 +14,6 @@ from dateutil import parser as dateparser import requests from base32_crockford import encode as base32_encode # type: ignore -from .config import ( - TIMEOUT, - STATICFILE_EXTENSIONS, - CHECK_SSL_VALIDITY, - WGET_USER_AGENT, - CHROME_OPTIONS, - COLOR_DICT -) - try: import chardet detect_encoding = lambda rawdata: chardet.detect(rawdata)["encoding"] @@ -49,7 +40,6 @@ base_url = lambda url: without_scheme(url) # uniq base url used to dedupe links without_www = lambda url: url.replace('://www.', '://', 1) without_trailing_slash = lambda url: url[:-1] if url[-1] == '/' else url.replace('/?', '?') hashurl = lambda url: base32_encode(int(sha256(base_url(url).encode('utf-8')).hexdigest(), 16))[:20] -is_static_file = lambda url: extension(url).lower() in STATICFILE_EXTENSIONS # TODO: the proper way is with MIME type detection, not using extension urlencode = lambda s: s and quote(s, encoding='utf-8', errors='replace') urldecode = lambda s: s and unquote(s) @@ -70,7 +60,14 @@ URL_REGEX = re.compile( re.IGNORECASE, ) +<<<<<<< HEAD COLOR_REGEX = re.compile(r'\[(?P\d+)(;(?P\d+)(;(?P\d+))?)?m') +======= +def is_static_file(url: str): + # TODO: the proper way is with MIME type detection + ext, not only extension + from .config import STATICFILE_EXTENSIONS + return extension(url).lower() in STATICFILE_EXTENSIONS +>>>>>>> c1fe068... remove circular import possibilities def enforce_types(func): @@ -155,8 +152,10 @@ def parse_date(date: Any) -> Optional[datetime]: @enforce_types -def download_url(url: str, timeout: int=TIMEOUT) -> str: +def download_url(url: str, timeout: int=None) -> str: """Download the contents of a remote url and return the text""" + from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT + timeout = timeout or TIMEOUT response = requests.get( url, headers={'User-Agent': WGET_USER_AGENT}, @@ -170,6 +169,8 @@ def download_url(url: str, timeout: int=TIMEOUT) -> str: def chrome_args(**options) -> List[str]: """helper to build up a chrome shell command with arguments""" + from .config import CHROME_OPTIONS + options = {**CHROME_OPTIONS, **options} cmd_args = [options['CHROME_BINARY']] @@ -202,6 +203,8 @@ def ansi_to_html(text): """ Based on: https://stackoverflow.com/questions/19212665/python-converting-ansi-color-codes-to-html """ + from .config import COLOR_DICT + TEMPLATE = '
' text = text.replace('[m', '') From 2ece5c20cfb11eff27078faa316aa4af075e5ad9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 03:14:07 -0400 Subject: [PATCH 213/365] bump docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d6d43042..2061184e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d6d43042893a017e0d43723da0b9890422102554 +Subproject commit 2061184e3ea6a35d8e32cb4ca6d24a1afc06706f From 3ec97e55283ed88be6ea3df89266378dda5fe09f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 03:22:37 -0400 Subject: [PATCH 214/365] fix git conflict commited by accident --- archivebox/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/archivebox/util.py b/archivebox/util.py index 717e1185..4ba1e3dd 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -60,14 +60,12 @@ URL_REGEX = re.compile( re.IGNORECASE, ) -<<<<<<< HEAD COLOR_REGEX = re.compile(r'\[(?P\d+)(;(?P\d+)(;(?P\d+))?)?m') -======= + def is_static_file(url: str): # TODO: the proper way is with MIME type detection + ext, not only extension from .config import STATICFILE_EXTENSIONS return extension(url).lower() in STATICFILE_EXTENSIONS ->>>>>>> c1fe068... remove circular import possibilities def enforce_types(func): @@ -204,7 +202,7 @@ def ansi_to_html(text): Based on: https://stackoverflow.com/questions/19212665/python-converting-ansi-color-codes-to-html """ from .config import COLOR_DICT - + TEMPLATE = '
' text = text.replace('[m', '') From 322be6b29233eee1b77626aab78d9e43b76261b0 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 03:53:39 -0400 Subject: [PATCH 215/365] move main into cli init and remove circular import layer --- archivebox/__init__.py | 6 ---- archivebox/__main__.py | 9 ++---- archivebox/cli/__init__.py | 55 ++++++++++++++++++++++++++++++- archivebox/cli/archivebox.py | 63 ------------------------------------ setup.py | 11 +++---- 5 files changed, 61 insertions(+), 83 deletions(-) delete mode 100755 archivebox/cli/archivebox.py diff --git a/archivebox/__init__.py b/archivebox/__init__.py index 56b6f16e..b0c00b61 100644 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -1,7 +1 @@ __package__ = 'archivebox' - -from . import core -from . import cli - -# The main CLI source code, is in 'archivebox/main.py' -from .main import * diff --git a/archivebox/__main__.py b/archivebox/__main__.py index 3386d46d..55e94415 100755 --- a/archivebox/__main__.py +++ b/archivebox/__main__.py @@ -3,13 +3,8 @@ __package__ = 'archivebox' import sys -from .cli import archivebox - - -def main(): - archivebox.main(args=sys.argv[1:], stdin=sys.stdin) +from .cli import main if __name__ == '__main__': - archivebox.main(args=sys.argv[1:], stdin=sys.stdin) - + main(args=sys.argv[1:], stdin=sys.stdin) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 7972c02e..ece64f8b 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -1,8 +1,13 @@ __package__ = 'archivebox.cli' +__command__ = 'archivebox' import os +import argparse + +from typing import Optional, Dict, List, IO + +from ..config import OUTPUT_DIR -from typing import Dict, List, Optional, IO from importlib import import_module CLI_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -24,6 +29,7 @@ is_valid_cli_module = lambda module, subcommand: ( and module.__command__.split(' ')[-1] == subcommand ) + def list_subcommands() -> Dict[str, str]: """find and import all valid archivebox_.py files in CLI_DIR""" @@ -57,6 +63,53 @@ def run_subcommand(subcommand: str, SUBCOMMANDS = list_subcommands() + +def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: + subcommands = list_subcommands() + parser = argparse.ArgumentParser( + prog=__command__, + description='ArchiveBox: The self-hosted internet archive', + add_help=False, + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + '--help', '-h', + action='store_true', + help=subcommands['help'], + ) + group.add_argument( + '--version', + action='store_true', + help=subcommands['version'], + ) + group.add_argument( + "subcommand", + type=str, + help= "The name of the subcommand to run", + nargs='?', + choices=subcommands.keys(), + default=None, + ) + parser.add_argument( + "subcommand_args", + help="Arguments for the subcommand", + nargs=argparse.REMAINDER, + ) + command = parser.parse_args(args or ()) + + if command.help or command.subcommand is None: + command.subcommand = 'help' + if command.version: + command.subcommand = 'version' + + run_subcommand( + subcommand=command.subcommand, + subcommand_args=command.subcommand_args, + stdin=stdin, + pwd=pwd or OUTPUT_DIR, + ) + + __all__ = ( 'SUBCOMMANDS', 'list_subcommands', diff --git a/archivebox/cli/archivebox.py b/archivebox/cli/archivebox.py deleted file mode 100755 index c8281937..00000000 --- a/archivebox/cli/archivebox.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -# archivebox [command] - -__package__ = 'archivebox.cli' -__command__ = 'archivebox' - -import sys -import argparse - -from typing import Optional, List, IO - -from . import list_subcommands, run_subcommand -from ..config import OUTPUT_DIR - - -def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: - subcommands = list_subcommands() - parser = argparse.ArgumentParser( - prog=__command__, - description='ArchiveBox: The self-hosted internet archive', - add_help=False, - ) - group = parser.add_mutually_exclusive_group() - group.add_argument( - '--help', '-h', - action='store_true', - help=subcommands['help'], - ) - group.add_argument( - '--version', - action='store_true', - help=subcommands['version'], - ) - group.add_argument( - "subcommand", - type=str, - help= "The name of the subcommand to run", - nargs='?', - choices=subcommands.keys(), - default=None, - ) - parser.add_argument( - "subcommand_args", - help="Arguments for the subcommand", - nargs=argparse.REMAINDER, - ) - command = parser.parse_args(args or ()) - - if command.help or command.subcommand is None: - command.subcommand = 'help' - if command.version: - command.subcommand = 'version' - - run_subcommand( - subcommand=command.subcommand, - subcommand_args=command.subcommand_args, - stdin=stdin, - pwd=pwd or OUTPUT_DIR, - ) - - -if __name__ == '__main__': - main(args=sys.argv[1:], stdin=sys.stdin) diff --git a/setup.py b/setup.py index 8ac00c44..049528fb 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import os import setuptools from pathlib import Path @@ -10,9 +9,9 @@ README = (BASE_DIR / "README.md").read_text() VERSION = (SOURCE_DIR / "VERSION").read_text().strip() # To see when setup.py gets called (uncomment for debugging) -import sys -print(SOURCE_DIR, f" (v{VERSION})") -print('>', sys.executable, *sys.argv) +# import sys +# print(SOURCE_DIR, f" (v{VERSION})") +# print('>', sys.executable, *sys.argv) # raise SystemExit(0) setuptools.setup( @@ -69,10 +68,10 @@ setuptools.setup( # 'redis': ['redis', 'django-redis'], # 'pywb': ['pywb', 'redis'], }, - packages=[PKG_NAME], + packages=setuptools.find_packages(), entry_points={ "console_scripts": [ - f"{PKG_NAME} = {PKG_NAME}.__main__:main", + f"{PKG_NAME} = {PKG_NAME}.cli:main", ], }, include_package_data=True, From 0c48449aa64c58fc350a40d39c3062e90e457a2d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 04:00:51 -0400 Subject: [PATCH 216/365] fix subcommand and args not being passed --- archivebox/cli/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index ece64f8b..8d06855a 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -2,6 +2,7 @@ __package__ = 'archivebox.cli' __command__ = 'archivebox' import os +import sys import argparse from typing import Optional, Dict, List, IO @@ -65,6 +66,7 @@ SUBCOMMANDS = list_subcommands() def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: + args = sys.argv[1:] if args is None else args subcommands = list_subcommands() parser = argparse.ArgumentParser( prog=__command__, From 528fc8f1f64bae28e54b416be5bb578dc2e38ccb Mon Sep 17 00:00:00 2001 From: Cristian Date: Thu, 2 Jul 2020 12:11:23 -0500 Subject: [PATCH 217/365] fix: Improve encoding detection for rss+xml content types --- archivebox/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/archivebox/util.py b/archivebox/util.py index 4ba1e3dd..8fdda389 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -160,6 +160,15 @@ def download_url(url: str, timeout: int=None) -> str: verify=CHECK_SSL_VALIDITY, timeout=timeout, ) + if response.headers.get('Content-Type') == 'application/rss+xml': + # Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py + _TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?''' + _XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P[\w-]+)') + _BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE) + _BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE) + match = _BODY_ENCODING_STR_RE.search(response.text[:1024]) + if match: + response.encoding = match.group('xmlcharset') return response.text From f373df7bd43ebe2c557f16c9e0c139975b63396c Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 13:23:40 -0400 Subject: [PATCH 218/365] update helptext to clarify adding links --- archivebox/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index a1aba118..f1fb98ce 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -377,11 +377,11 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None: else: print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links), **ANSI)) print() - print(' {lightred}Hint:{reset}To view your archive index, open:'.format(**ANSI)) - print(' {}'.format(os.path.join(out_dir, HTML_INDEX_FILENAME))) + print(' {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI)) + print(' archivebox server # then visit http://127.0.0.1:8000') print() print(' To add new links, you can run:') - print(" archivebox add 'https://example.com'") + print(" archivebox add ~/some/path/or/url/to/list_of_links.txt") print() print(' For more usage and examples, run:') print(' archivebox help') From 7c428f40c8b74df85c6088ad7fcd5b62c4e10556 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 Jul 2020 13:31:05 -0400 Subject: [PATCH 219/365] fix stdin link importing --- archivebox/cli/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 8d06855a..087f11b5 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -64,9 +64,14 @@ def run_subcommand(subcommand: str, SUBCOMMANDS = list_subcommands() +class NotProvided: + pass + + +def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None: + args = sys.argv[1:] if args is NotProvided else args + stdin = sys.stdin if stdin is NotProvided else stdin -def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None: - args = sys.argv[1:] if args is None else args subcommands = list_subcommands() parser = argparse.ArgumentParser( prog=__command__, From 8bdfa18a3f8eb10dfd05337f7c488d20bda31bcc Mon Sep 17 00:00:00 2001 From: Cristian Date: Thu, 2 Jul 2020 15:54:25 -0500 Subject: [PATCH 220/365] feat: Allow feed loading from the add links view --- archivebox/core/forms.py | 7 +++++ archivebox/core/views.py | 33 +++++++++++++++++------- archivebox/themes/default/add_links.html | 10 +++++-- 3 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 archivebox/core/forms.py diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py new file mode 100644 index 00000000..5f67e2c6 --- /dev/null +++ b/archivebox/core/forms.py @@ -0,0 +1,7 @@ +from django import forms + +CHOICES = (('url', 'URL'), ('feed', 'Feed')) + +class AddLinkForm(forms.Form): + url = forms.URLField() + source = forms.ChoiceField(choices=CHOICES, widget=forms.RadioSelect, initial='url') diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 5efa79cd..0c5efff2 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -22,6 +22,8 @@ from ..config import ( from ..util import base_url, ansi_to_html from .. main import add +from .forms import AddLinkForm + class MainIndex(View): template = 'main_index.html' @@ -51,28 +53,39 @@ class AddLinks(View): if not request.user.is_authenticated and not PUBLIC_INDEX: return redirect(f'/admin/login/?next={request.path}') - context = {} + context = { + "form": AddLinkForm() + } return render(template_name=self.template, request=request, context=context) def post(self, request): - url = request.POST['url'] - if url: + #url = request.POST['url'] + #if url: + form = AddLinkForm(request.POST) + if form.is_valid(): + url = form.cleaned_data["url"] print(f'[+] Adding URL: {url}') + if form.cleaned_data["source"] == "url": + key = "import_str" + else: + key = "import_path" + input_kwargs = { + key: url, + "update_all": False, + "out_dir": OUTPUT_DIR, + } add_stdout = StringIO() with redirect_stdout(add_stdout): - extracted_links = add( - import_str=url, - update_all=False, - out_dir=OUTPUT_DIR, - ) + extracted_links = add(**input_kwargs) print(add_stdout.getvalue()) context = { - "stdout": ansi_to_html(add_stdout.getvalue()) + "stdout": ansi_to_html(add_stdout.getvalue()), + "form": AddLinkForm() } else: - context = {"stdout": "Please enter a URL"} + context = {"form": form} return render(template_name=self.template, request=request, context=context) diff --git a/archivebox/themes/default/add_links.html b/archivebox/themes/default/add_links.html index 6c625594..7143c576 100644 --- a/archivebox/themes/default/add_links.html +++ b/archivebox/themes/default/add_links.html @@ -159,6 +159,12 @@ .title-col a { color: black; } + .ul-form { + list-style: none; + } + .ul-form li { + list-style: none; + } @@ -199,9 +205,9 @@
{{ stdout | safe }}

-
{% csrf_token %} + {% csrf_token %} Add new links...
-
+ {{ form.as_ul }}
From 63fe19e2c2d236cabae36ef441aff9fd46dd6014 Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 3 Jul 2020 11:52:57 -0500 Subject: [PATCH 221/365] feat: Add pytest and initial tests --- setup.py | 3 +++ tests/test_init.py | 40 ++++++++++++++++++++++++++++++++++++++++ tests/test_util.py | 21 +++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 tests/test_init.py create mode 100644 tests/test_util.py diff --git a/setup.py b/setup.py index 049528fb..12002580 100755 --- a/setup.py +++ b/setup.py @@ -65,6 +65,9 @@ setuptools.setup( "sphinx-rtd-theme", "recommonmark", ], + "test": [ + "pytest" + ] # 'redis': ['redis', 'django-redis'], # 'pywb': ['pywb', 'redis'], }, diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 00000000..b870a599 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,40 @@ +# archivebox init +# archivebox add + +import os +import subprocess +from pathlib import Path +import json + +import pytest + +@pytest.fixture +def process(tmp_path): + os.chdir(tmp_path) + process = subprocess.run(['archivebox', 'init'], capture_output=True) + return process + + +def test_init(tmp_path, process): + assert "Initializing a new ArchiveBox collection in this folder..." in process.stdout.decode("utf-8") + +def test_update(tmp_path, process): + os.chdir(tmp_path) + update_process = subprocess.run(['archivebox', 'init'], capture_output=True) + assert "Updating existing ArchiveBox collection in this folder" in update_process.stdout.decode("utf-8") + +def test_add_link(tmp_path, process): + os.chdir(tmp_path) + add_process = subprocess.run(['archivebox', 'add', 'http://example.com'], capture_output=True) + archived_item_path = list(tmp_path.glob('archive/**/*'))[0] + + assert "index.json" in [x.name for x in archived_item_path.iterdir()] + + with open(archived_item_path / "index.json", "r") as f: + output_json = json.load(f) + assert "IANA — IANA-managed Reserved Domains" == output_json['history']['title'][0]['output'] + + with open(tmp_path / "index.html", "r") as f: + output_html = f.read() + assert "IANA — IANA-managed Reserved Domains" in output_html + diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 00000000..19ed31c0 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,21 @@ +#@enforce_types +#def download_url(url: str, timeout: int=None) -> str: +# """Download the contents of a remote url and return the text""" +# from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT +# timeout = timeout or TIMEOUT +# response = requests.get( +# url, +# headers={'User-Agent': WGET_USER_AGENT}, +# verify=CHECK_SSL_VALIDITY, +# timeout=timeout, +# ) +# if response.headers.get('Content-Type') == 'application/rss+xml': +# # Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py +# _TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?''' +# _XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P[\w-]+)') +# _BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE) +# _BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE) +# match = _BODY_ENCODING_STR_RE.search(response.text[:1024]) +# if match: +# response.encoding = match.group('xmlcharset') +# return response.text \ No newline at end of file From 438203f4cec49e92c49976d57788be6b188f173e Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 3 Jul 2020 12:54:21 -0500 Subject: [PATCH 222/365] test: add basic download_url test --- tests/test_util.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 19ed31c0..1497de5a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,21 +1,5 @@ -#@enforce_types -#def download_url(url: str, timeout: int=None) -> str: -# """Download the contents of a remote url and return the text""" -# from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT -# timeout = timeout or TIMEOUT -# response = requests.get( -# url, -# headers={'User-Agent': WGET_USER_AGENT}, -# verify=CHECK_SSL_VALIDITY, -# timeout=timeout, -# ) -# if response.headers.get('Content-Type') == 'application/rss+xml': -# # Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py -# _TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?''' -# _XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P[\w-]+)') -# _BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE) -# _BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE) -# match = _BODY_ENCODING_STR_RE.search(response.text[:1024]) -# if match: -# response.encoding = match.group('xmlcharset') -# return response.text \ No newline at end of file +from archivebox import util + +def test_download_url_downloads_content(): + text = util.download_url("https://example.com") + assert "Example Domain" in text \ No newline at end of file From 4302ae4caa4fccbe40e67084d4b3edd315e9eb1f Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 3 Jul 2020 13:13:59 -0500 Subject: [PATCH 223/365] fix: Remove test section in setup.py --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 12002580..9ca39608 100755 --- a/setup.py +++ b/setup.py @@ -64,10 +64,8 @@ setuptools.setup( "sphinx", "sphinx-rtd-theme", "recommonmark", + "pytest", ], - "test": [ - "pytest" - ] # 'redis': ['redis', 'django-redis'], # 'pywb': ['pywb', 'redis'], }, From ffaae510779b49b44450c58c3c631a29f065ae32 Mon Sep 17 00:00:00 2001 From: apkallum Date: Fri, 3 Jul 2020 16:52:28 -0400 Subject: [PATCH 224/365] test github actions --- .github/workflows/test.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..311236c0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,25 @@ +name: Test workflow +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + architecture: x64 + + - name: Install dependencies + run: | + pip install -e .[dev] + + - name: Test with pytest + run: | + pytest -s \ No newline at end of file From d5fc13b34e0f29c67b52c05a3ba098f049830e60 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 08:36:58 -0500 Subject: [PATCH 225/365] refactor: Move pytest fixtures to its own file --- tests/__init__.py | 0 tests/fixtures.py | 10 ++++++++++ tests/test_args.py | 0 tests/test_init.py | 9 +-------- 4 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/fixtures.py create mode 100644 tests/test_args.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 00000000..9bf2640a --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,10 @@ +import os +import subprocess + +import pytest + +@pytest.fixture +def process(tmp_path): + os.chdir(tmp_path) + process = subprocess.run(['archivebox', 'init'], capture_output=True) + return process \ No newline at end of file diff --git a/tests/test_args.py b/tests/test_args.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_init.py b/tests/test_init.py index b870a599..1b80bb1b 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -6,14 +6,7 @@ import subprocess from pathlib import Path import json -import pytest - -@pytest.fixture -def process(tmp_path): - os.chdir(tmp_path) - process = subprocess.run(['archivebox', 'init'], capture_output=True) - return process - +from .fixtures import * def test_init(tmp_path, process): assert "Initializing a new ArchiveBox collection in this folder..." in process.stdout.decode("utf-8") From 8b22a2a7dd2507e164f0780fa38d73ba36912144 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 09:10:36 -0500 Subject: [PATCH 226/365] feat: Enable --depth flag (still does nothing) --- archivebox/cli/archivebox_add.py | 13 +++++++------ tests/test_args.py | 7 +++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 272fe5cf..77a11bd0 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -45,6 +45,13 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional ' ~/Desktop/sites_list.csv\n' ) ) + parser.add_argument( + "--depth", + action="store", + default=0, + type=int, + help="Recursively archive all linked pages up to this many hops away" + ) command = parser.parse_args(args or ()) import_str = accept_stdin(stdin) add( @@ -63,12 +70,6 @@ if __name__ == '__main__': # TODO: Implement these # # parser.add_argument( -# '--depth', #'-d', -# type=int, -# help='Recursively archive all linked pages up to this many hops away', -# default=0, -# ) -# parser.add_argument( # '--mirror', #'-m', # action='store_true', # help='Archive an entire site (finding all linked pages below it on the same domain)', diff --git a/tests/test_args.py b/tests/test_args.py index e69de29b..b8df1941 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -0,0 +1,7 @@ +import subprocess + +from .fixtures import * + +def test_depth_flag_is_accepted(tmp_path, process): + arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) + assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode('utf-8') \ No newline at end of file From 2db03245398f0a6c7fcda77a3ebc5688e3836396 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 09:49:28 -0500 Subject: [PATCH 227/365] feat: depth=0 crawls the current page only --- archivebox/cli/archivebox_add.py | 14 +++++++++++--- tests/test_args.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 77a11bd0..5bbccb19 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -53,14 +53,22 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional help="Recursively archive all linked pages up to this many hops away" ) command = parser.parse_args(args or ()) - import_str = accept_stdin(stdin) + #import_str = accept_stdin(stdin) add( - import_str=import_str, - import_path=command.import_path, + import_str=command.import_path, + import_path=None, update_all=command.update_all, index_only=command.index_only, out_dir=pwd or OUTPUT_DIR, ) + #if command.depth == 1: + # add( + # import_str=None, + # import_path=command.import_path, + # update_all=command.update_all, + # index_only=command.index_only, + # out_dir=pwd or OUTPUT_DIR, + # ) if __name__ == '__main__': diff --git a/tests/test_args.py b/tests/test_args.py index b8df1941..59d43fee 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -1,7 +1,15 @@ import subprocess +import json from .fixtures import * -def test_depth_flag_is_accepted(tmp_path, process): +def test_depth_flag_is_accepted(process): arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) - assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode('utf-8') \ No newline at end of file + assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode('utf-8') + +def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process): + arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) + archived_item_path = list(tmp_path.glob('archive/**/*'))[0] + with open(archived_item_path / "index.json", "r") as f: + output_json = json.load(f) + assert output_json["base_url"] == "example.com" \ No newline at end of file From 32e790979e2f37c3615b52e0ed858603abd429a5 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 10:07:44 -0500 Subject: [PATCH 228/365] feat: Enable depth=1 functionality --- archivebox/cli/archivebox_add.py | 16 ++++++++-------- tests/test_args.py | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 5bbccb19..65335679 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -61,14 +61,14 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional index_only=command.index_only, out_dir=pwd or OUTPUT_DIR, ) - #if command.depth == 1: - # add( - # import_str=None, - # import_path=command.import_path, - # update_all=command.update_all, - # index_only=command.index_only, - # out_dir=pwd or OUTPUT_DIR, - # ) + if command.depth == 1: + add( + import_str=None, + import_path=command.import_path, + update_all=command.update_all, + index_only=command.index_only, + out_dir=pwd or OUTPUT_DIR, + ) if __name__ == '__main__': diff --git a/tests/test_args.py b/tests/test_args.py index 59d43fee..e0c6020e 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -12,4 +12,11 @@ def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process): archived_item_path = list(tmp_path.glob('archive/**/*'))[0] with open(archived_item_path / "index.json", "r") as f: output_json = json.load(f) - assert output_json["base_url"] == "example.com" \ No newline at end of file + assert output_json["base_url"] == "example.com" + +def test_depth_flag_1_crawls_the_page_AND_links(tmp_path, process): + arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=1"], capture_output=True) + with open(tmp_path / "index.json", "r") as f: + archive_file = f.read() + assert "https://example.com" in archive_file + assert "https://www.iana.org/domains/example" in archive_file \ No newline at end of file From a6940092bbf37123e68e2c22418584fa9b4a2d88 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 10:25:02 -0500 Subject: [PATCH 229/365] feat: Make sure that depth can only be either 1 or 0 --- archivebox/cli/archivebox_add.py | 2 +- tests/test_args.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 65335679..2f77f754 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -49,11 +49,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional "--depth", action="store", default=0, + choices=[0,1], type=int, help="Recursively archive all linked pages up to this many hops away" ) command = parser.parse_args(args or ()) - #import_str = accept_stdin(stdin) add( import_str=command.import_path, import_path=None, diff --git a/tests/test_args.py b/tests/test_args.py index e0c6020e..91264ef2 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -5,7 +5,13 @@ from .fixtures import * def test_depth_flag_is_accepted(process): arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) - assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode('utf-8') + assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode("utf-8") + +def test_depth_flag_fails_if_it_is_not_0_or_1(process): + arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=5"], capture_output=True) + assert 'invalid choice' in arg_process.stderr.decode("utf-8") + arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=-1"], capture_output=True) + assert 'invalid choice' in arg_process.stderr.decode("utf-8") def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process): arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) @@ -19,4 +25,4 @@ def test_depth_flag_1_crawls_the_page_AND_links(tmp_path, process): with open(tmp_path / "index.json", "r") as f: archive_file = f.read() assert "https://example.com" in archive_file - assert "https://www.iana.org/domains/example" in archive_file \ No newline at end of file + assert "https://www.iana.org/domains/example" in archive_file From bca6a06f6035e7a10c9726ef40e7aed4b4b7ee34 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 11:53:02 -0500 Subject: [PATCH 230/365] test: Fix test to reflect new API changes --- tests/test_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 1b80bb1b..c5627a2f 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -25,9 +25,9 @@ def test_add_link(tmp_path, process): with open(archived_item_path / "index.json", "r") as f: output_json = json.load(f) - assert "IANA — IANA-managed Reserved Domains" == output_json['history']['title'][0]['output'] + assert "Example Domain" == output_json['history']['title'][0]['output'] with open(tmp_path / "index.html", "r") as f: output_html = f.read() - assert "IANA — IANA-managed Reserved Domains" in output_html + assert "Example Domain" in output_html From b68c13918f28246e8521080a03486dcbb7ff8537 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 12:39:36 -0500 Subject: [PATCH 231/365] feat: Disable stdin from archivebox add --- archivebox/cli/archivebox_add.py | 6 ++++-- archivebox/main.py | 3 +-- tests/test_init.py | 6 ++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 2f77f754..c729e9fb 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -10,7 +10,7 @@ from typing import List, Optional, IO from ..main import add, docstring from ..config import OUTPUT_DIR, ONLY_NEW -from .logging import SmartFormatter, accept_stdin +from .logging import SmartFormatter, reject_stdin @docstring(add.__doc__) @@ -38,9 +38,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional type=str, default=None, help=( - 'URL or path to local file containing a list of links to import. e.g.:\n' + 'URL or path to local file containing a page or list of links to import. e.g.:\n' ' https://getpocket.com/users/USERNAME/feed/all\n' ' https://example.com/some/rss/feed.xml\n' + ' https://example.com\n' ' ~/Downloads/firefox_bookmarks_export.html\n' ' ~/Desktop/sites_list.csv\n' ) @@ -54,6 +55,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional help="Recursively archive all linked pages up to this many hops away" ) command = parser.parse_args(args or ()) + reject_stdin(__command__, stdin) add( import_str=command.import_path, import_path=None, diff --git a/archivebox/main.py b/archivebox/main.py index f1fb98ce..3f05a385 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -507,8 +507,7 @@ def add(import_str: Optional[str]=None, if (import_str and import_path) or (not import_str and not import_path): stderr( - '[X] You should pass either an import path as an argument, ' - 'or pass a list of links via stdin, but not both.\n', + '[X] You should pass an import path or a page url as an argument\n', color='red', ) raise SystemExit(2) diff --git a/tests/test_init.py b/tests/test_init.py index c5627a2f..d592b0a1 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -31,3 +31,9 @@ def test_add_link(tmp_path, process): output_html = f.read() assert "Example Domain" in output_html +def test_add_link_does_not_support_stdin(tmp_path, process): + os.chdir(tmp_path) + stdin_process = subprocess.Popen(["archivebox", "add"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = stdin_process.communicate(input="example.com".encode())[0] + assert "does not accept stdin" in output.decode("utf-8") + From c1d8a74e4f2673047e31b96aa303fbd300dccc50 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 7 Jul 2020 15:46:45 -0500 Subject: [PATCH 232/365] feat: Make input sent via stdin behave the same as using args --- archivebox/cli/archivebox_add.py | 19 +++++++++++++++---- tests/test_init.py | 12 +++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index c729e9fb..c692750b 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -10,7 +10,7 @@ from typing import List, Optional, IO from ..main import add, docstring from ..config import OUTPUT_DIR, ONLY_NEW -from .logging import SmartFormatter, reject_stdin +from .logging import SmartFormatter, accept_stdin @docstring(add.__doc__) @@ -55,9 +55,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional help="Recursively archive all linked pages up to this many hops away" ) command = parser.parse_args(args or ()) - reject_stdin(__command__, stdin) + import_string = accept_stdin(stdin) + if import_string and command.import_path: + stderr( + '[X] You should pass an import path or a page url as an argument or in stdin but not both\n', + color='red', + ) + raise SystemExit(2) + elif import_string: + import_path = import_string + else: + import_path = command.import_path + add( - import_str=command.import_path, + import_str=import_path, import_path=None, update_all=command.update_all, index_only=command.index_only, @@ -66,7 +77,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional if command.depth == 1: add( import_str=None, - import_path=command.import_path, + import_path=import_path, update_all=command.update_all, index_only=command.index_only, out_dir=pwd or OUTPUT_DIR, diff --git a/tests/test_init.py b/tests/test_init.py index d592b0a1..97870459 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -31,9 +31,15 @@ def test_add_link(tmp_path, process): output_html = f.read() assert "Example Domain" in output_html -def test_add_link_does_not_support_stdin(tmp_path, process): +def test_add_link_support_stdin(tmp_path, process): os.chdir(tmp_path) stdin_process = subprocess.Popen(["archivebox", "add"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = stdin_process.communicate(input="example.com".encode())[0] - assert "does not accept stdin" in output.decode("utf-8") + stdin_process.communicate(input="http://example.com".encode()) + archived_item_path = list(tmp_path.glob('archive/**/*'))[0] + + assert "index.json" in [x.name for x in archived_item_path.iterdir()] + + with open(archived_item_path / "index.json", "r") as f: + output_json = json.load(f) + assert "Example Domain" == output_json['history']['title'][0]['output'] From f12bfeb3229345b2d4cd7c1670ba050ca1111e7c Mon Sep 17 00:00:00 2001 From: Cristian Date: Wed, 8 Jul 2020 08:17:47 -0500 Subject: [PATCH 233/365] refactor: Change add() to receive url and depth instead of import_str and import_path --- archivebox/cli/archivebox_add.py | 12 ++---------- archivebox/core/views.py | 8 +++----- archivebox/main.py | 25 ++++++++++--------------- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index c692750b..8f491d42 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -68,20 +68,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional import_path = command.import_path add( - import_str=import_path, - import_path=None, + url=import_path, + depth=command.depth, update_all=command.update_all, index_only=command.index_only, out_dir=pwd or OUTPUT_DIR, ) - if command.depth == 1: - add( - import_str=None, - import_path=import_path, - update_all=command.update_all, - index_only=command.index_only, - out_dir=pwd or OUTPUT_DIR, - ) if __name__ == '__main__': diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 0c5efff2..a721b992 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -66,12 +66,10 @@ class AddLinks(View): if form.is_valid(): url = form.cleaned_data["url"] print(f'[+] Adding URL: {url}') - if form.cleaned_data["source"] == "url": - key = "import_str" - else: - key = "import_path" + depth = 0 if form.cleaned_data["source"] == "url" else 1 input_kwargs = { - key: url, + "url": url, + "depth": depth, "update_all": False, "out_dir": OUTPUT_DIR, } diff --git a/archivebox/main.py b/archivebox/main.py index 3f05a385..a96c4250 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -496,8 +496,8 @@ def status(out_dir: str=OUTPUT_DIR) -> None: @enforce_types -def add(import_str: Optional[str]=None, - import_path: Optional[str]=None, +def add(url: str, + depth: int=0, update_all: bool=not ONLY_NEW, index_only: bool=False, out_dir: str=OUTPUT_DIR) -> List[Link]: @@ -505,17 +505,9 @@ def add(import_str: Optional[str]=None, check_data_folder(out_dir=out_dir) - if (import_str and import_path) or (not import_str and not import_path): - stderr( - '[X] You should pass an import path or a page url as an argument\n', - color='red', - ) - raise SystemExit(2) - elif import_str: - import_path = save_stdin_to_sources(import_str, out_dir=out_dir) - elif import_path: - import_path = save_file_to_sources(import_path, out_dir=out_dir) - + base_path = save_stdin_to_sources(url, out_dir=out_dir) + if depth == 1: + depth_path = save_file_to_sources(url, out_dir=out_dir) check_dependencies() # Step 1: Load list of links from the existing index @@ -523,8 +515,11 @@ def add(import_str: Optional[str]=None, all_links: List[Link] = [] new_links: List[Link] = [] all_links = load_main_index(out_dir=out_dir) - if import_path: - all_links, new_links = import_new_links(all_links, import_path, out_dir=out_dir) + all_links, new_links = import_new_links(all_links, base_path, out_dir=out_dir) + if depth == 1: + all_links, new_links_depth = import_new_links(all_links, depth_path, out_dir=out_dir) + new_links = new_links + new_links_depth + # Step 2: Write updated index with deduped old and new links back to disk write_main_index(links=all_links, out_dir=out_dir) From 4ebf929606b50afcce94f2440a7ac363cc96a887 Mon Sep 17 00:00:00 2001 From: Cristian Date: Wed, 8 Jul 2020 08:30:07 -0500 Subject: [PATCH 234/365] refactor: Change wording on CLI help --- archivebox/cli/archivebox_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 8f491d42..c4c78399 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -38,7 +38,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional type=str, default=None, help=( - 'URL or path to local file containing a page or list of links to import. e.g.:\n' + 'URL or path to local file to start the archiving process from. e.g.:\n' ' https://getpocket.com/users/USERNAME/feed/all\n' ' https://example.com/some/rss/feed.xml\n' ' https://example.com\n' From d476b130074a18e0a903743bdd3e61b5f7f397b0 Mon Sep 17 00:00:00 2001 From: Cristian Date: Wed, 8 Jul 2020 14:46:31 -0500 Subject: [PATCH 235/365] fix: Add missing permission to add view (post) --- archivebox/core/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 0c5efff2..57941264 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -60,8 +60,8 @@ class AddLinks(View): return render(template_name=self.template, request=request, context=context) def post(self, request): - #url = request.POST['url'] - #if url: + if not request.user.is_authenticated and not PUBLIC_INDEX: + return redirect(f'/admin/login/?next={request.path}') form = AddLinkForm(request.POST) if form.is_valid(): url = form.cleaned_data["url"] From 09b4438c9f5ad89c9cc46bdc3c4df131420a8b37 Mon Sep 17 00:00:00 2001 From: Apkallum Date: Wed, 8 Jul 2020 17:54:01 -0400 Subject: [PATCH 236/365] fix legacy index.html --- archivebox/themes/legacy/main_index.html | 73 +----------------------- 1 file changed, 2 insertions(+), 71 deletions(-) diff --git a/archivebox/themes/legacy/main_index.html b/archivebox/themes/legacy/main_index.html index 1b366300..e246b0d9 100644 --- a/archivebox/themes/legacy/main_index.html +++ b/archivebox/themes/legacy/main_index.html @@ -4,34 +4,6 @@ Archived Sites + + + +
+

Example Domain

+

This domain is for use in illustrative examples in documents. You may use this + domain in literature without prior coordination or asking for permission.

+

+ More information... +

+
+ + diff --git a/tests/mock_server/templates/iana.org.html b/tests/mock_server/templates/iana.org.html new file mode 100644 index 00000000..c1e60a2e --- /dev/null +++ b/tests/mock_server/templates/iana.org.html @@ -0,0 +1,390 @@ + + + + IANA — IANA-managed Reserved Domains + + + + + + + + + + + + + + + + + +
+ +
+ +
+ + +
+ + +

IANA-managed Reserved Domains

+ +

Certain domains are set aside, and nominally registered to “IANA”, for specific + policy or technical purposes.

+ +

Example domains

+ +

As described in + RFC 2606 + and + RFC 6761, + a number of domains such as + example.com + and + example.org + are maintained for documentation purposes. These domains may be used as illustrative + examples in documents without prior coordination with us. They are + not available for registration or transfer.

+ +

Test IDN top-level domains

+ +

These domains were temporarily delegated by IANA for the + IDN Evaluation + being conducted by + ICANN.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DomainDomain (A-label)LanguageScript
إختبار + + XN--KGBECHTV + + ArabicArabic
آزمایشی + + XN--HGBK6AJ7F53BBA + + PersianArabic
测试 + + XN--0ZWM56D + + ChineseHan (Simplified variant)
測試 + + XN--G6W251D + + ChineseHan (Traditional variant)
испытание + + XN--80AKHBYKNJ4F + + RussianCyrillic
परीक्षा + + XN--11B5BS3A9AJ6G + + HindiDevanagari (Nagari)
δοκιμή + + XN--JXALPDLP + + Greek, Modern (1453-)Greek
테스트 + + XN--9T4B11YI5A + + KoreanHangul (Hangŭl, Hangeul)
טעסט + + XN--DEBA0AD + + YiddishHebrew
テスト + + XN--ZCKZAH + + JapaneseKatakana
பரிட்சை + + XN--HLCJ6AYA9ESC7A + + TamilTamil
+
+ +

Policy-reserved domains

+ +

We act as both the registrant and registrar for a select number of domains + which have been reserved under policy grounds. These exclusions are + typically indicated in either technical standards (RFC documents), + or + contractual limitations.

+ +

Domains which are described as registered to IANA or ICANN on policy + grounds are not available for registration or transfer, with the exception + of + + country-name.info + domains. These domains are available for release + by the ICANN Governmental Advisory Committee Secretariat.

+ +

Other Special-Use Domains

+ +

There is additionally a + Special-Use Domain Names + registry documenting special-use domains designated by technical standards. For further information, see + Special-Use Domain Names + (RFC 6761).

+ + +
+ + + + +
+ + diff --git a/tests/test_args.py b/tests/test_args.py index 91264ef2..f52626fb 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -4,25 +4,25 @@ import json from .fixtures import * def test_depth_flag_is_accepted(process): - arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) + arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=0"], capture_output=True) assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode("utf-8") def test_depth_flag_fails_if_it_is_not_0_or_1(process): - arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=5"], capture_output=True) + arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=5"], capture_output=True) assert 'invalid choice' in arg_process.stderr.decode("utf-8") - arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=-1"], capture_output=True) + arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=-1"], capture_output=True) assert 'invalid choice' in arg_process.stderr.decode("utf-8") def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process): - arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True) + arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=0"], capture_output=True) archived_item_path = list(tmp_path.glob('archive/**/*'))[0] with open(archived_item_path / "index.json", "r") as f: output_json = json.load(f) - assert output_json["base_url"] == "example.com" + assert output_json["base_url"] == "localhost:8080/static/example.com.html" def test_depth_flag_1_crawls_the_page_AND_links(tmp_path, process): - arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=1"], capture_output=True) + arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=1"], capture_output=True) with open(tmp_path / "index.json", "r") as f: archive_file = f.read() - assert "https://example.com" in archive_file - assert "https://www.iana.org/domains/example" in archive_file + assert "http://localhost:8080/static/example.com.html" in archive_file + assert "http://localhost:8080/static/iana.org.html" in archive_file From fe80a93a0380a11a3196f194c13bf9ae13531e4e Mon Sep 17 00:00:00 2001 From: Cristian Date: Mon, 13 Jul 2020 09:43:36 -0500 Subject: [PATCH 240/365] test: Refactor init tests to use local webserver --- tests/test_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 97870459..24d3ed52 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -18,7 +18,7 @@ def test_update(tmp_path, process): def test_add_link(tmp_path, process): os.chdir(tmp_path) - add_process = subprocess.run(['archivebox', 'add', 'http://example.com'], capture_output=True) + add_process = subprocess.run(['archivebox', 'add', 'http://localhost:8080/static/example.com.html'], capture_output=True) archived_item_path = list(tmp_path.glob('archive/**/*'))[0] assert "index.json" in [x.name for x in archived_item_path.iterdir()] @@ -34,7 +34,7 @@ def test_add_link(tmp_path, process): def test_add_link_support_stdin(tmp_path, process): os.chdir(tmp_path) stdin_process = subprocess.Popen(["archivebox", "add"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - stdin_process.communicate(input="http://example.com".encode()) + stdin_process.communicate(input="http://localhost:8080/static/example.com.html".encode()) archived_item_path = list(tmp_path.glob('archive/**/*'))[0] assert "index.json" in [x.name for x in archived_item_path.iterdir()] From 322997e229457bf43ee2281993ccdc30c8455244 Mon Sep 17 00:00:00 2001 From: Cristian Date: Mon, 13 Jul 2020 09:44:50 -0500 Subject: [PATCH 241/365] test: Refactor util tests to use local webserver --- tests/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index 1497de5a..0a076344 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,5 +1,5 @@ from archivebox import util def test_download_url_downloads_content(): - text = util.download_url("https://example.com") + text = util.download_url("http://localhost:8080/static/example.com.html") assert "Example Domain" in text \ No newline at end of file From 7cbd068c95e5a40851a40e9ed272b62c49a885e9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:22:07 -0400 Subject: [PATCH 242/365] add flake8 --- .flake8 | 6 ++++++ archivebox/.flake8 | 8 +++++--- archivebox/__main__.py | 1 + archivebox/config/__init__.py | 4 +++- archivebox/core/models.py | 1 - archivebox/index/schema.py | 1 + archivebox/main.py | 4 ++-- 7 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..01af646d --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391 +select = F,E9,W +max-line-length = 130 +max-complexity = 10 +exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv diff --git a/archivebox/.flake8 b/archivebox/.flake8 index 46da144b..dd6ba8e4 100644 --- a/archivebox/.flake8 +++ b/archivebox/.flake8 @@ -1,4 +1,6 @@ [flake8] -ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E127,E131,E241,E252,E266,E272,E701,E731,W293,W503 -select = F,E9 -exclude = migrations,util_scripts,node_modules,venv +ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391 +select = F,E9,W +max-line-length = 130 +max-complexity = 10 +exclude = migrations,tests,node_modules,vendor,static,venv,.venv,.venv2,.docker-venv diff --git a/archivebox/__main__.py b/archivebox/__main__.py index 55e94415..8afaa27a 100755 --- a/archivebox/__main__.py +++ b/archivebox/__main__.py @@ -6,5 +6,6 @@ import sys from .cli import main + if __name__ == '__main__': main(args=sys.argv[1:], stdin=sys.stdin) diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py index f06b0f3d..14b66e92 100644 --- a/archivebox/config/__init__.py +++ b/archivebox/config/__init__.py @@ -279,6 +279,8 @@ def load_config_val(key: str, config: Optional[ConfigDict]=None, env_vars: Optional[os._Environ]=None, config_file_vars: Optional[Dict[str, str]]=None) -> ConfigValue: + """parse bool, int, and str key=value pairs from env""" + config_keys_to_check = (key, *(aliases or ())) for key in config_keys_to_check: @@ -777,7 +779,7 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None: stderr() stderr(f'[!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={config["TIMEOUT"]} seconds)', color='red') stderr(' You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.') - stderr(' (Setting it to somewhere between 30 and 300 seconds is recommended)') + stderr(' (Setting it to somewhere between 30 and 3000 seconds is recommended)') stderr() stderr(' If you want to make ArchiveBox run faster, disable specific archive methods instead:') stderr(' https://github.com/pirate/ArchiveBox/wiki/Configuration#archive-method-toggles') diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 2cbfc1b1..42929e5a 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -24,7 +24,6 @@ class Snapshot(models.Model): keys = ('url', 'timestamp', 'title', 'tags', 'updated') - def __repr__(self) -> str: title = self.title or '-' return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})' diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py index 637e0589..db17c269 100644 --- a/archivebox/index/schema.py +++ b/archivebox/index/schema.py @@ -98,6 +98,7 @@ class Link: updated: Optional[datetime] = None schema: str = 'Link' + def __str__(self) -> str: return f'[{self.timestamp}] {self.base_url} "{self.title}"' diff --git a/archivebox/main.py b/archivebox/main.py index a96c4250..a6e04dd3 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -641,8 +641,8 @@ def update(resume: Optional[float]=None, out_dir: str=OUTPUT_DIR) -> List[Link]: """Import any new links from subscriptions and retry any previously failed/skipped links""" - check_dependencies() check_data_folder(out_dir=out_dir) + check_dependencies() # Step 1: Load list of links from the existing index # merge in and dedupe new links from import_path @@ -990,7 +990,7 @@ def schedule(add: bool=False, if total_runs > 60 and not quiet: stderr() stderr('{lightyellow}[!] With the current cron config, ArchiveBox is estimated to run >{} times per year.{reset}'.format(total_runs, **ANSI)) - stderr(f' Congrats on being an enthusiastic internet archiver! 👌') + stderr(' Congrats on being an enthusiastic internet archiver! 👌') stderr() stderr(' Make sure you have enough storage space available to hold all the data.') stderr(' Using a compressed/deduped filesystem like ZFS is recommended if you plan on archiving a lot.') From 96b1e4a8ec1eb64c979c185b912ef6d60b25074f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:22:58 -0400 Subject: [PATCH 243/365] accept local paths as valid link URLs when parsing --- archivebox/parsers/generic_txt.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/archivebox/parsers/generic_txt.py b/archivebox/parsers/generic_txt.py index cc3653a0..61d1973f 100644 --- a/archivebox/parsers/generic_txt.py +++ b/archivebox/parsers/generic_txt.py @@ -5,6 +5,7 @@ import re from typing import IO, Iterable from datetime import datetime +from pathlib import Path from ..index.schema import Link from ..util import ( @@ -13,14 +14,28 @@ from ..util import ( URL_REGEX ) + @enforce_types def parse_generic_txt_export(text_file: IO[str]) -> Iterable[Link]: """Parse raw links from each line in a text file""" text_file.seek(0) for line in text_file.readlines(): - urls = re.findall(URL_REGEX, line) if line.strip() else () - for url in urls: # type: ignore + if not line.strip(): + continue + + # if the line is a local file path that resolves, then we can archive it + if Path(line).exists(): + yield Link( + url=line, + timestamp=str(datetime.now().timestamp()), + title=None, + tags=None, + sources=[text_file.name], + ) + + # otherwise look for anything that looks like a URL in the line + for url in re.findall(URL_REGEX, line): yield Link( url=htmldecode(url), timestamp=str(datetime.now().timestamp()), From 16f3746712e3767ea3ab1ef0aec3cc38108b331b Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:24:36 -0400 Subject: [PATCH 244/365] check source dir at the end of checking data dir --- archivebox/config/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py index 14b66e92..3638bade 100644 --- a/archivebox/config/__init__.py +++ b/archivebox/config/__init__.py @@ -838,6 +838,10 @@ def check_data_folder(out_dir: Optional[str]=None, config: ConfigDict=CONFIG) -> stderr(' archivebox init') raise SystemExit(3) + sources_dir = os.path.join(output_dir, SOURCES_DIR_NAME) + if not os.path.exists(sources_dir): + os.makedirs(sources_dir) + def setup_django(out_dir: str=None, check_db=False, config: ConfigDict=CONFIG) -> None: From dfb83b4f2728f2f0a389650836d6164a2f80e809 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:24:49 -0400 Subject: [PATCH 245/365] add AttributeDict --- archivebox/util.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/archivebox/util.py b/archivebox/util.py index 8fdda389..0e7ebd31 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -230,6 +230,23 @@ def ansi_to_html(text): return COLOR_REGEX.sub(single_sub, text) +class AttributeDict(dict): + """Helper to allow accessing dict values via Example.key or Example['key']""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Recursively convert nested dicts to AttributeDicts (optional): + # for key, val in self.items(): + # if isinstance(val, dict) and type(val) is not AttributeDict: + # self[key] = AttributeDict(val) + + def __getattr__(self, attr: str) -> Any: + return dict.__getitem__(self, attr) + + def __setattr__(self, attr: str, value: Any) -> None: + return dict.__setitem__(self, attr, value) + + class ExtendedEncoder(pyjson.JSONEncoder): """ Extended json serializer that supports serializing several model From 354a63ccd4f021c68747c8a16d30cd54f67167b8 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:25:43 -0400 Subject: [PATCH 246/365] dont dedupe snapshots in sqlite on every run --- archivebox/index/sql.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 0ad68de0..80203980 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -26,23 +26,8 @@ def write_sql_main_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None: from core.models import Snapshot from django.db import transaction - all_urls = {link.url: link for link in links} - all_ts = {link.timestamp: link for link in links} - with transaction.atomic(): - for snapshot in Snapshot.objects.all(): - if snapshot.timestamp in all_ts: - info = {k: v for k, v in all_urls.pop(snapshot.url)._asdict().items() if k in Snapshot.keys} - snapshot.delete() - Snapshot.objects.create(**info) - elif snapshot.url in all_urls: - info = {k: v for k, v in all_urls.pop(snapshot.url)._asdict().items() if k in Snapshot.keys} - snapshot.delete() - Snapshot.objects.create(**info) - else: - snapshot.delete() - - for url, link in all_urls.items(): + for link in links: info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} Snapshot.objects.update_or_create(url=url, defaults=info) From d3bfa98a912fe4a360835b1e32258244ffa12262 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:26:30 -0400 Subject: [PATCH 247/365] fix depth flag and tweak logging --- archivebox/cli/__init__.py | 12 +++- archivebox/cli/archivebox_add.py | 24 +++---- archivebox/cli/logging.py | 61 ++++++++++++------ archivebox/extractors/__init__.py | 27 +++++++- archivebox/index/__init__.py | 29 +++++---- archivebox/main.py | 102 ++++++++++++------------------ archivebox/parsers/__init__.py | 28 ++------ 7 files changed, 156 insertions(+), 127 deletions(-) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 087f11b5..b7575c4a 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -106,8 +106,18 @@ def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, if command.help or command.subcommand is None: command.subcommand = 'help' - if command.version: + elif command.version: command.subcommand = 'version' + + if command.subcommand not in ('help', 'version', 'status'): + from ..cli.logging import log_cli_command + + log_cli_command( + subcommand=command.subcommand, + subcommand_args=command.subcommand_args, + stdin=stdin, + pwd=pwd or OUTPUT_DIR + ) run_subcommand( subcommand=command.subcommand, diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index c4c78399..55832346 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -10,7 +10,7 @@ from typing import List, Optional, IO from ..main import add, docstring from ..config import OUTPUT_DIR, ONLY_NEW -from .logging import SmartFormatter, accept_stdin +from .logging import SmartFormatter, accept_stdin, stderr @docstring(add.__doc__) @@ -33,12 +33,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional help="Add the links to the main index without archiving them", ) parser.add_argument( - 'import_path', - nargs='?', + 'urls', + nargs='*', type=str, default=None, help=( - 'URL or path to local file to start the archiving process from. e.g.:\n' + 'URLs or paths to archive e.g.:\n' ' https://getpocket.com/users/USERNAME/feed/all\n' ' https://example.com/some/rss/feed.xml\n' ' https://example.com\n' @@ -50,25 +50,21 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional "--depth", action="store", default=0, - choices=[0,1], + choices=[0, 1], type=int, help="Recursively archive all linked pages up to this many hops away" ) command = parser.parse_args(args or ()) - import_string = accept_stdin(stdin) - if import_string and command.import_path: + urls = command.urls + stdin_urls = accept_stdin(stdin) + if (stdin_urls and urls) or (not stdin and not urls): stderr( - '[X] You should pass an import path or a page url as an argument or in stdin but not both\n', + '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n', color='red', ) raise SystemExit(2) - elif import_string: - import_path = import_string - else: - import_path = command.import_path - add( - url=import_path, + urls=stdin_urls or urls, depth=command.depth, update_all=command.update_all, index_only=command.index_only, diff --git a/archivebox/cli/logging.py b/archivebox/cli/logging.py index 6de78d8f..a12c4e98 100644 --- a/archivebox/cli/logging.py +++ b/archivebox/cli/logging.py @@ -5,10 +5,12 @@ import os import sys import time import argparse +import logging +import signal +from multiprocessing import Process from datetime import datetime from dataclasses import dataclass -from multiprocessing import Process from typing import Optional, List, Dict, Union, IO from ..index.schema import Link, ArchiveResult @@ -23,11 +25,11 @@ from ..config import ( SHOW_PROGRESS, TERM_WIDTH, OUTPUT_DIR, + SOURCES_DIR_NAME, HTML_INDEX_FILENAME, stderr, ) - @dataclass class RuntimeStats: """mutable stats counter for logging archiving timing info to CLI output""" @@ -98,9 +100,9 @@ class TimedProgress: if SHOW_PROGRESS: # terminate if we havent already terminated - if self.p is not None: - self.p.terminate() - self.p = None + self.p.terminate() + self.p.join() + self.p.close() # clear whole terminal line try: @@ -145,28 +147,51 @@ def progress_bar(seconds: int, prefix: str='') -> None: seconds, )) sys.stdout.flush() - except KeyboardInterrupt: + except (KeyboardInterrupt, BrokenPipeError): print() pass +def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str): + from ..config import VERSION, ANSI + cmd = ' '.join(('archivebox', subcommand, *subcommand_args)) + stdin_hint = ' < /dev/stdin' if not stdin.isatty() else '' + print('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{stdin_hint}{reset}'.format( + now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + VERSION=VERSION, + cmd=cmd, + stdin_hint=stdin_hint, + **ANSI, + )) + print('{black} > {pwd}{reset}'.format(pwd=pwd, **ANSI)) + print() + ### Parsing Stage -def log_parsing_started(source_file: str): - start_ts = datetime.now() - _LAST_RUN_STATS.parse_start_ts = start_ts - print('\n{green}[*] [{}] Parsing new links from output/sources/{}...{reset}'.format( - start_ts.strftime('%Y-%m-%d %H:%M:%S'), - source_file.rsplit('/', 1)[-1], + +def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool): + _LAST_RUN_STATS.parse_start_ts = datetime.now() + print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format( + _LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'), + len(urls) if isinstance(urls, list) else len(urls.split('\n')), + depth, + ' (index only)' if index_only else '', **ANSI, )) +def log_source_saved(source_file: str): + print(' > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1])) -def log_parsing_finished(num_parsed: int, num_new_links: int, parser_name: str): - end_ts = datetime.now() - _LAST_RUN_STATS.parse_end_ts = end_ts - print(' > Parsed {} links as {} ({} new links added)'.format(num_parsed, parser_name, num_new_links)) +def log_parsing_finished(num_parsed: int, parser_name: str): + _LAST_RUN_STATS.parse_end_ts = datetime.now() + print(' > Parsed {} URLs from input ({})'.format(num_parsed, parser_name)) +def log_deduping_finished(num_new_links: int): + print(' > Found {} new URLs not already in index'.format(num_new_links)) + + +def log_crawl_started(new_links): + print('{lightblue}[*] Starting crawl of {} sites 1 hop out from starting point{reset}'.format(len(new_links), **ANSI)) ### Indexing Stage @@ -174,7 +199,7 @@ def log_indexing_process_started(num_links: int): start_ts = datetime.now() _LAST_RUN_STATS.index_start_ts = start_ts print() - print('{green}[*] [{}] Writing {} links to main index...{reset}'.format( + print('{black}[*] [{}] Writing {} links to main index...{reset}'.format( start_ts.strftime('%Y-%m-%d %H:%M:%S'), num_links, **ANSI, @@ -209,7 +234,7 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None): **ANSI, )) else: - print('{green}[▶] [{}] Updating content for {} matching pages in archive...{reset}'.format( + print('{green}[▶] [{}] Collecting content for {} Snapshots in archive...{reset}'.format( start_ts.strftime('%Y-%m-%d %H:%M:%S'), num_links, **ANSI, diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index c6a4f33c..c08e7c0c 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -2,7 +2,7 @@ __package__ = 'archivebox.extractors' import os -from typing import Optional +from typing import Optional, List from datetime import datetime from ..index.schema import Link @@ -13,6 +13,9 @@ from ..index import ( ) from ..util import enforce_types from ..cli.logging import ( + log_archiving_started, + log_archiving_paused, + log_archiving_finished, log_link_archiving_started, log_link_archiving_finished, log_archive_method_started, @@ -103,3 +106,25 @@ def archive_link(link: Link, overwrite: bool=False, out_dir: Optional[str]=None) raise return link + + +@enforce_types +def archive_links(links: List[Link], out_dir: Optional[str]=None) -> List[Link]: + if not links: + return [] + + log_archiving_started(len(links)) + idx: int = 0 + link: Link = links[0] + try: + for idx, link in enumerate(links): + archive_link(link, out_dir=link.link_dir) + except KeyboardInterrupt: + log_archiving_paused(len(links), idx, link.timestamp) + raise SystemExit(0) + except BaseException: + print() + raise + + log_archiving_finished(len(links)) + return links diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py index e82cfefa..7ea473d7 100644 --- a/archivebox/index/__init__.py +++ b/archivebox/index/__init__.py @@ -33,8 +33,8 @@ from ..cli.logging import ( log_indexing_process_finished, log_indexing_started, log_indexing_finished, - log_parsing_started, log_parsing_finished, + log_deduping_finished, ) from .schema import Link, ArchiveResult @@ -268,20 +268,31 @@ def load_main_index_meta(out_dir: str=OUTPUT_DIR) -> Optional[dict]: return None + @enforce_types -def import_new_links(existing_links: List[Link], - import_path: str, - out_dir: str=OUTPUT_DIR) -> Tuple[List[Link], List[Link]]: +def parse_links_from_source(source_path: str) -> Tuple[List[Link], List[Link]]: from ..parsers import parse_links new_links: List[Link] = [] # parse and validate the import file - log_parsing_started(import_path) - raw_links, parser_name = parse_links(import_path) + raw_links, parser_name = parse_links(source_path) new_links = validate_links(raw_links) + if parser_name: + num_parsed = len(raw_links) + log_parsing_finished(num_parsed, parser_name) + + return new_links + + +@enforce_types +def dedupe_links(existing_links: List[Link], + new_links: List[Link]) -> Tuple[List[Link], List[Link]]: + + from ..parsers import parse_links + # merge existing links in out_dir and new links all_links = validate_links(existing_links + new_links) all_link_urls = {link.url for link in existing_links} @@ -290,11 +301,7 @@ def import_new_links(existing_links: List[Link], link for link in new_links if link.url not in all_link_urls ] - - if parser_name: - num_parsed = len(raw_links) - num_new_links = len(all_links) - len(existing_links) - log_parsing_finished(num_parsed, num_new_links, parser_name) + log_deduping_finished(len(new_links)) return all_links, new_links diff --git a/archivebox/main.py b/archivebox/main.py index a6e04dd3..54b71acc 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -4,8 +4,7 @@ import os import sys import shutil -from typing import Dict, List, Optional, Iterable, IO - +from typing import Dict, List, Optional, Iterable, IO, Union from crontab import CronTab, CronSlices from .cli import ( @@ -17,16 +16,17 @@ from .cli import ( archive_cmds, ) from .parsers import ( - save_stdin_to_sources, - save_file_to_sources, + save_text_as_source, + save_file_as_source, ) from .index.schema import Link -from .util import enforce_types, docstring +from .util import enforce_types, docstring # type: ignore from .system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT from .index import ( links_after_timestamp, load_main_index, - import_new_links, + parse_links_from_source, + dedupe_links, write_main_index, link_matches_filter, get_indexed_folders, @@ -51,7 +51,7 @@ from .index.sql import ( apply_migrations, ) from .index.html import parse_html_main_index -from .extractors import archive_link +from .extractors import archive_links from .config import ( stderr, ConfigDict, @@ -91,9 +91,8 @@ from .config import ( from .cli.logging import ( TERM_WIDTH, TimedProgress, - log_archiving_started, - log_archiving_paused, - log_archiving_finished, + log_importing_started, + log_crawl_started, log_removal_started, log_removal_finished, log_list_started, @@ -496,59 +495,55 @@ def status(out_dir: str=OUTPUT_DIR) -> None: @enforce_types -def add(url: str, +def add(urls: Union[str, List[str]], depth: int=0, update_all: bool=not ONLY_NEW, index_only: bool=False, out_dir: str=OUTPUT_DIR) -> List[Link]: """Add a new URL or list of URLs to your archive""" + assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)' + + # Load list of links from the existing index check_data_folder(out_dir=out_dir) - - base_path = save_stdin_to_sources(url, out_dir=out_dir) - if depth == 1: - depth_path = save_file_to_sources(url, out_dir=out_dir) check_dependencies() - - # Step 1: Load list of links from the existing index - # merge in and dedupe new links from import_path all_links: List[Link] = [] new_links: List[Link] = [] all_links = load_main_index(out_dir=out_dir) - all_links, new_links = import_new_links(all_links, base_path, out_dir=out_dir) - if depth == 1: - all_links, new_links_depth = import_new_links(all_links, depth_path, out_dir=out_dir) - new_links = new_links + new_links_depth + + log_importing_started(urls=urls, depth=depth, index_only=index_only) + if isinstance(urls, str): + # save verbatim stdin to sources + write_ahead_log = save_text_as_source(urls, filename='{ts}-import.txt', out_dir=out_dir) + elif isinstance(urls, list): + # save verbatim args to sources + write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir) + + new_links += parse_links_from_source(write_ahead_log) + all_links, new_links = dedupe_links(all_links, new_links) + write_main_index(links=all_links, out_dir=out_dir, finished=not new_links) - # Step 2: Write updated index with deduped old and new links back to disk - write_main_index(links=all_links, out_dir=out_dir) + # If we're going one level deeper, download each link and look for more links + if new_links and depth == 1: + log_crawl_started(new_links) + for new_link in new_links: + downloaded_file = save_file_as_source(new_link.url, filename='{ts}-crawl-{basename}.txt', out_dir=out_dir) + new_links += parse_links_from_source(downloaded_file) + all_links, new_links = dedupe_links(all_links, new_links) + write_main_index(links=all_links, out_dir=out_dir, finished=not new_links) if index_only: return all_links - - # Step 3: Run the archive methods for each link - links = all_links if update_all else new_links - log_archiving_started(len(links)) - idx: int = 0 - link: Link = None # type: ignore - try: - for idx, link in enumerate(links): - archive_link(link, out_dir=link.link_dir) - except KeyboardInterrupt: - log_archiving_paused(len(links), idx, link.timestamp if link else '0') - raise SystemExit(0) - - except: - print() - raise - - log_archiving_finished(len(links)) + # Run the archive methods for each link + to_archive = all_links if update_all else new_links + archive_links(to_archive, out_dir=out_dir) # Step 4: Re-write links index with updated titles, icons, and resources - all_links = load_main_index(out_dir=out_dir) - write_main_index(links=list(all_links), out_dir=out_dir, finished=True) + if to_archive: + all_links = load_main_index(out_dir=out_dir) + write_main_index(links=list(all_links), out_dir=out_dir, finished=True) return all_links @enforce_types @@ -671,23 +666,8 @@ def update(resume: Optional[float]=None, return all_links # Step 3: Run the archive methods for each link - links = new_links if only_new else all_links - log_archiving_started(len(links), resume) - idx: int = 0 - link: Link = None # type: ignore - try: - for idx, link in enumerate(links_after_timestamp(links, resume)): - archive_link(link, overwrite=overwrite, out_dir=link.link_dir) - - except KeyboardInterrupt: - log_archiving_paused(len(links), idx, link.timestamp if link else '0') - raise SystemExit(0) - - except: - print() - raise - - log_archiving_finished(len(links)) + to_archive = new_links if only_new else all_links + archive_links(to_archive, out_dir=out_dir) # Step 4: Re-write links index with updated titles, icons, and resources all_links = load_main_index(out_dir=out_dir) diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 479d4e2c..eabaece2 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -29,7 +29,7 @@ from ..util import ( URL_REGEX, ) from ..index.schema import Link -from ..cli.logging import pretty_path, TimedProgress +from ..cli.logging import pretty_path, TimedProgress, log_source_saved from .pocket_html import parse_pocket_html_export from .pinboard_rss import parse_pinboard_rss_export from .shaarli_rss import parse_shaarli_rss_export @@ -83,36 +83,22 @@ def parse_links(source_file: str) -> Tuple[List[Link], str]: @enforce_types -def save_stdin_to_sources(raw_text: str, out_dir: str=OUTPUT_DIR) -> str: - check_data_folder(out_dir=out_dir) - - sources_dir = os.path.join(out_dir, SOURCES_DIR_NAME) - if not os.path.exists(sources_dir): - os.makedirs(sources_dir) - +def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: str=OUTPUT_DIR) -> str: ts = str(datetime.now().timestamp()).split('.', 1)[0] - - source_path = os.path.join(sources_dir, '{}-{}.txt'.format('stdin', ts)) + source_path = os.path.join(OUTPUT_DIR, SOURCES_DIR_NAME, filename.format(ts=ts)) atomic_write(source_path, raw_text) + log_source_saved(source_file=source_path) return source_path @enforce_types -def save_file_to_sources(path: str, timeout: int=TIMEOUT, out_dir: str=OUTPUT_DIR) -> str: +def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: str=OUTPUT_DIR) -> str: """download a given url's content into output/sources/domain-.txt""" - check_data_folder(out_dir=out_dir) - - sources_dir = os.path.join(out_dir, SOURCES_DIR_NAME) - if not os.path.exists(sources_dir): - os.makedirs(sources_dir) - ts = str(datetime.now().timestamp()).split('.', 1)[0] - - source_path = os.path.join(sources_dir, '{}-{}.txt'.format(basename(path), ts)) + source_path = os.path.join(OUTPUT_DIR, SOURCES_DIR_NAME, filename.format(basename=basename(path), ts=ts)) if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')): # Source is a URL that needs to be downloaded - source_path = os.path.join(sources_dir, '{}-{}.txt'.format(domain(path), ts)) print('{}[*] [{}] Downloading {}{}'.format( ANSI['green'], datetime.now().strftime('%Y-%m-%d %H:%M:%S'), @@ -140,7 +126,7 @@ def save_file_to_sources(path: str, timeout: int=TIMEOUT, out_dir: str=OUTPUT_DI atomic_write(source_path, raw_source_text) - print(' > {}'.format(pretty_path(source_path))) + log_source_saved(source_file=source_path) return source_path From 4c4b1e6a4bde5edb9e11942245a21437e73fe6df Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:33:35 -0400 Subject: [PATCH 248/365] fix link creation --- archivebox/index/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 80203980..b120738c 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -29,7 +29,7 @@ def write_sql_main_index(links: List[Link], out_dir: str=OUTPUT_DIR) -> None: with transaction.atomic(): for link in links: info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys} - Snapshot.objects.update_or_create(url=url, defaults=info) + Snapshot.objects.update_or_create(url=link.url, defaults=info) @enforce_types def write_sql_link_details(link: Link, out_dir: str=OUTPUT_DIR) -> None: From d159e674e1fb7005f1732f78adbd5cf5aa49436a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:41:18 -0400 Subject: [PATCH 249/365] write stderr instead of stdout for version info --- archivebox/cli/logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archivebox/cli/logging.py b/archivebox/cli/logging.py index a12c4e98..d11ffd9e 100644 --- a/archivebox/cli/logging.py +++ b/archivebox/cli/logging.py @@ -156,15 +156,15 @@ def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional from ..config import VERSION, ANSI cmd = ' '.join(('archivebox', subcommand, *subcommand_args)) stdin_hint = ' < /dev/stdin' if not stdin.isatty() else '' - print('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{stdin_hint}{reset}'.format( + stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{stdin_hint}{reset}'.format( now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), VERSION=VERSION, cmd=cmd, stdin_hint=stdin_hint, **ANSI, )) - print('{black} > {pwd}{reset}'.format(pwd=pwd, **ANSI)) - print() + stderr('{black} > {pwd}{reset}'.format(pwd=pwd, **ANSI)) + stderr() ### Parsing Stage From b4ce20cbe5b3d41676a43a337e0e12a869e53aac Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:41:27 -0400 Subject: [PATCH 250/365] write link details json before and after archiving --- archivebox/extractors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index c08e7c0c..c9685a80 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -56,6 +56,7 @@ def archive_link(link: Link, overwrite: bool=False, out_dir: Optional[str]=None) os.makedirs(out_dir) link = load_link_details(link, out_dir=out_dir) + write_link_details(link, out_dir=link.link_dir) log_link_archiving_started(link, out_dir, is_new) link = link.overwrite(updated=datetime.now()) stats = {'skipped': 0, 'succeeded': 0, 'failed': 0} From 215d5eae324d9da3ffb758bf5e47f7b31d942e9a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 11:41:37 -0400 Subject: [PATCH 251/365] normal git clone instead of mirror --- archivebox/extractors/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/extractors/git.py b/archivebox/extractors/git.py index 1534ce34..dcb1df3c 100644 --- a/archivebox/extractors/git.py +++ b/archivebox/extractors/git.py @@ -56,7 +56,7 @@ def save_git(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> A cmd = [ GIT_BINARY, 'clone', - '--mirror', + # '--mirror', '--recursive', *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']), without_query(without_fragment(link.url)), From ae208435c9c979720fad8f7782d6c74247b6c069 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 12:21:37 -0400 Subject: [PATCH 252/365] fix the add links form --- archivebox/cli/logging.py | 2 +- archivebox/core/admin.py | 2 +- archivebox/core/forms.py | 7 +++++-- archivebox/core/views.py | 4 ++-- archivebox/extractors/git.py | 1 - archivebox/themes/default/add_links.html | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/archivebox/cli/logging.py b/archivebox/cli/logging.py index d11ffd9e..f002e922 100644 --- a/archivebox/cli/logging.py +++ b/archivebox/cli/logging.py @@ -191,7 +191,7 @@ def log_deduping_finished(num_new_links: int): def log_crawl_started(new_links): - print('{lightblue}[*] Starting crawl of {} sites 1 hop out from starting point{reset}'.format(len(new_links), **ANSI)) + print('{lightred}[*] Starting crawl of {} sites 1 hop out from starting point{reset}'.format(len(new_links), **ANSI)) ### Indexing Stage diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index 7942c6c2..1b05c580 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -49,7 +49,7 @@ class SnapshotAdmin(admin.ModelAdmin): '📼 ' '📦 ' '🏛 ' - '' + '
' '{}', obj.archive_path, canon['wget_path'] or '', obj.archive_path, canon['pdf_path'], diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index 5f67e2c6..8bf0cbd0 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -1,7 +1,10 @@ from django import forms -CHOICES = (('url', 'URL'), ('feed', 'Feed')) +CHOICES = ( + ('0', 'depth=0 (archive just this url)'), + ('1', 'depth=1 (archive this url and all sites one link away)'), +) class AddLinkForm(forms.Form): url = forms.URLField() - source = forms.ChoiceField(choices=CHOICES, widget=forms.RadioSelect, initial='url') + depth = forms.ChoiceField(choices=CHOICES, widget=forms.RadioSelect, initial='0') diff --git a/archivebox/core/views.py b/archivebox/core/views.py index d9c51700..5fb43119 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -66,9 +66,9 @@ class AddLinks(View): if form.is_valid(): url = form.cleaned_data["url"] print(f'[+] Adding URL: {url}') - depth = 0 if form.cleaned_data["source"] == "url" else 1 + depth = 0 if form.cleaned_data["depth"] == "0" else 0 input_kwargs = { - "url": url, + "urls": url, "depth": depth, "update_all": False, "out_dir": OUTPUT_DIR, diff --git a/archivebox/extractors/git.py b/archivebox/extractors/git.py index dcb1df3c..c8a5eeaf 100644 --- a/archivebox/extractors/git.py +++ b/archivebox/extractors/git.py @@ -56,7 +56,6 @@ def save_git(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> A cmd = [ GIT_BINARY, 'clone', - # '--mirror', '--recursive', *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']), without_query(without_fragment(link.url)), diff --git a/archivebox/themes/default/add_links.html b/archivebox/themes/default/add_links.html index 7143c576..6e35f38c 100644 --- a/archivebox/themes/default/add_links.html +++ b/archivebox/themes/default/add_links.html @@ -212,7 +212,7 @@ - Go back to Snapshot list + Go back to Main Index From a79dd4685a2bea2f6d9b94a79215d28eb72ba722 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 Jul 2020 12:21:52 -0400 Subject: [PATCH 253/365] make snapshots unique again --- .../migrations/0004_auto_20200713_1552.py | 19 +++++++++++++++++++ archivebox/core/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 archivebox/core/migrations/0004_auto_20200713_1552.py diff --git a/archivebox/core/migrations/0004_auto_20200713_1552.py b/archivebox/core/migrations/0004_auto_20200713_1552.py new file mode 100644 index 00000000..69836623 --- /dev/null +++ b/archivebox/core/migrations/0004_auto_20200713_1552.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-07-13 15:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0003_auto_20200630_1034'), + ] + + operations = [ + migrations.AlterField( + model_name='snapshot', + name='timestamp', + field=models.CharField(db_index=True, default=None, max_length=32, unique=True), + preserve_default=False, + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 42929e5a..7ac9427b 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -13,7 +13,7 @@ class Snapshot(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) url = models.URLField(unique=True) - timestamp = models.CharField(max_length=32, null=True, default=None, db_index=True) + timestamp = models.CharField(max_length=32, unique=True, db_index=True) title = models.CharField(max_length=128, null=True, default=None, db_index=True) tags = models.CharField(max_length=256, null=True, default=None, db_index=True) From 5e2bf73f047f2a647f1497a98aedc4cf76f12832 Mon Sep 17 00:00:00 2001 From: Cristian Date: Mon, 13 Jul 2020 14:48:25 -0500 Subject: [PATCH 254/365] fix: Bugs related to add() refactor --- archivebox/index/__init__.py | 6 +++++- archivebox/main.py | 10 ++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py index 7ea473d7..cd50a185 100644 --- a/archivebox/index/__init__.py +++ b/archivebox/index/__init__.py @@ -292,7 +292,6 @@ def dedupe_links(existing_links: List[Link], new_links: List[Link]) -> Tuple[List[Link], List[Link]]: from ..parsers import parse_links - # merge existing links in out_dir and new links all_links = validate_links(existing_links + new_links) all_link_urls = {link.url for link in existing_links} @@ -301,6 +300,11 @@ def dedupe_links(existing_links: List[Link], link for link in new_links if link.url not in all_link_urls ] + + all_links_deduped = {link.url: link for link in all_links} + for i in range(len(new_links)): + if new_links[i].url in all_links_deduped.keys(): + new_links[i] = all_links_deduped[new_links[i].url] log_deduping_finished(len(new_links)) return all_links, new_links diff --git a/archivebox/main.py b/archivebox/main.py index 54b71acc..999e4650 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -520,18 +520,16 @@ def add(urls: Union[str, List[str]], write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir) new_links += parse_links_from_source(write_ahead_log) - all_links, new_links = dedupe_links(all_links, new_links) - write_main_index(links=all_links, out_dir=out_dir, finished=not new_links) - # If we're going one level deeper, download each link and look for more links + new_links_depth = [] if new_links and depth == 1: log_crawl_started(new_links) for new_link in new_links: downloaded_file = save_file_as_source(new_link.url, filename='{ts}-crawl-{basename}.txt', out_dir=out_dir) - new_links += parse_links_from_source(downloaded_file) - all_links, new_links = dedupe_links(all_links, new_links) - write_main_index(links=all_links, out_dir=out_dir, finished=not new_links) + new_links_depth += parse_links_from_source(downloaded_file) + all_links, new_links = dedupe_links(all_links, new_links + new_links_depth) + write_main_index(links=all_links, out_dir=out_dir, finished=not new_links) if index_only: return all_links From 98dda688970c8993a7a79847ea74ff5e30964b4f Mon Sep 17 00:00:00 2001 From: apkallum Date: Tue, 14 Jul 2020 10:26:33 -0400 Subject: [PATCH 255/365] fix: timestamp comparison in to_json function --- archivebox/index/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py index db17c269..eb6ef894 100644 --- a/archivebox/index/schema.py +++ b/archivebox/index/schema.py @@ -190,7 +190,10 @@ class Link: for key, val in json_info.items() if key in cls.field_names() } - info['updated'] = parse_date(info.get('updated')) + try: + info['updated'] = int(parse_date(info.get('updated'))) # Cast to int which comes with rounding down + except (ValueError, TypeError): + info['updated'] = None info['sources'] = info.get('sources') or [] json_history = info.get('history') or {} From f845224d6f60e59ee53981885c400eb83a03fb12 Mon Sep 17 00:00:00 2001 From: Cristian Date: Thu, 16 Jul 2020 09:20:33 -0500 Subject: [PATCH 256/365] fix: htmlencode titles before rendering the static html index and detail --- archivebox/index/html.py | 4 +- .../templates/title_with_html.com.html | 699 ++++++++++++++++++ tests/test_title.py | 14 + 3 files changed, 715 insertions(+), 2 deletions(-) create mode 100644 tests/mock_server/templates/title_with_html.com.html create mode 100644 tests/test_title.py diff --git a/archivebox/index/html.py b/archivebox/index/html.py index 60d41049..e21ae576 100644 --- a/archivebox/index/html.py +++ b/archivebox/index/html.py @@ -90,7 +90,7 @@ def main_index_row_template(link: Link) -> str: **link._asdict(extended=True), # before pages are finished archiving, show loading msg instead of title - 'title': ( + 'title': htmlencode( link.title or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), @@ -129,7 +129,7 @@ def link_details_template(link: Link) -> str: return render_legacy_template(LINK_DETAILS_TEMPLATE, { **link_info, **link_info['canonical'], - 'title': ( + 'title': htmlencode( link.title or (link.base_url if link.is_archived else TITLE_LOADING_MSG) ), diff --git a/tests/mock_server/templates/title_with_html.com.html b/tests/mock_server/templates/title_with_html.com.html new file mode 100644 index 00000000..e84dcaa0 --- /dev/null +++ b/tests/mock_server/templates/title_with_html.com.html @@ -0,0 +1,699 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It All Starts with a Humble <textarea> ◆ 24 ways + + +
+ Skip to content +

+ 24 ways + to impress your friends + +

+
+
+ +
+ + + + +
+ +
+ + +
+
+
+

It All Starts with a Humble <textarea>

+

+ + + + + Andy Bell + + Andy Bell + + +

+
+ +
+
    +
  • + +
    • + + +
  • Published in + UX +
    • + + +
  • + No comments +
    • +
+
+ +
+ +
+

Those that know me well know that I make + a lot + of + side projects. I most definitely make too many, but there’s one really useful thing about making lots of side projects: it allows me to experiment in a low-risk setting. +

+

Side projects also allow me to accidentally create a context where I can demonstrate a really affective, long-running methodology for building on the web: + progressive enhancement. That context is a little Progressive Web App that I’m tinkering with called + Jotter. It’s incredibly simple, but under the hood, there’s a really solid experience built on top of a + minimum viable experience + which after reading this article, you’ll hopefully apply this methodology to your own work.

+
+ The Jotter Progressive Web App presented in the Google Chrome browser. + +
+

What is a minimum viable experience?

+

The key to progressive enhancement is distilling the user experience to its lowest possible technical solution and then building on it to improve the user experience. In the context of + Jotter, that is a humble + <textarea> + element. That humble + <textarea> + is our + minimum viable experience. +

+

Let me show you how it’s built up, progressively real quick. If you disable CSS and JavaScript, you get this:

+
+ The Jotter Progressive Web App with CSS and JavaScript disabled shows a HTML only experience. + +
+

This result is great because I know that regardless of what happens, the user can do what they needed to do when the loaded Jotter in their browser: take some notes. That’s our + minimum viable experience, completed with a few lines of code that work in + every single browser—even very old browsers. Don’t you just love good ol’ HTML? +

+

Now it’s time to enhance that minimum viable experience, + progressively. It’s a good idea to do that in smaller steps rather than just provide a 0% experience or a 100% experience, which is the approach that’s often favoured by JavaScript framework enthusiasts. I think that process is counter-intuitive to the web, though, so building up from a minimum viable experience is the optimal way to go, in my opinion. +

+

Understanding how a + minimum viable experience + works can be a bit tough, admittedly, so I like to use a the following diagram to explain the process:

+
+ Minimum viable experience diagram which is described in the next paragraph. + +
+

Let me break down this diagram for both folks who can and can’t see it. On the top row, there’s four stages of a broken-up car, starting with just a wheel, all the way up to a fully functioning car. The car enhances only in a way that it is still + mostly useless + until it gets to its final form when the person is finally happy. +

+

On the second row, instead of building a car, we start with a skateboard which immediately does the job of getting the person from point A to point B. This enhances to a Micro Scooter and then to a Push Bike. Its final form is a fancy looking Motor Scooter. I choose that instead of a car deliberately because generally, when you progressively enhance a project, it turns out to be + way simpler and lighter + than a project that was built without progressive enhancement in mind.

+

Now that we know what a minimum viable experience is and how it works, let’s apply this methodology to Jotter! +

+

Add some CSS

+

The first enhancement is CSS. Jotter has a very simple design, which is mostly a full height + <textarea> + with a little sidebar. A flexbox-based, auto-stacking layout, inspired by a layout called + The Sidebar + is used and we’re good to go. +

+

Based on the diagram from earlier, we can comfortably say we’re in + Skateboard + territory now.

+

Add some JavaScript

+

We’ve got styles now, so let’s + enhance + the experience again. A user can currently load up the site and take notes. If the CSS loads, it’ll be a more pleasant experience, but if they refresh their browser, they’re going to lose all of their work.

+

We can fix that by adding some + local storage + into the mix. +

+

The functionality flow is pretty straightforward. As a user inputs content, the JavaScript listens to an + input + event and pushes the content of the + <textarea> + into + localStorage. If we then set that + localStorage + data to populate the + <textarea> + on load, that user’s experience is suddenly + enhanced + because they can’t lose their work by accidentally refreshing. +

+

The JavaScript is incredibly light, too: +

+ 
const textArea = document.querySelector('textarea');
+const storageKey = 'text';
+
+const init = () => {
+
+  textArea.value = localStorage.getItem(storageKey);
+
+  textArea.addEventListener('input', () => {
+    localStorage.setItem(storageKey, textArea.value);
+  });
+}
+
+init();
+

In around 13 lines of code (which you can see a + working demo here), we’ve been able to enhance the user’s experience + considerably, and if we think back to our diagram from earlier, we are very much in + Micro Scooter + territory now. +

+

Making it a PWA

+

We’re in really good shape now, so let’s turn Jotter into a + Motor Scooter + and make this thing work offline as an installable Progressive Web App (PWA). +

+

Making a PWA is really achievable and Google have even produced a + handy checklist + to help you get going. You can also get guidance from a + Lighthouse audit. +

+

For this little app, all we need is a + manifest + and a + Service Worker + to cache assets and serve them offline for us if needed.

+

The Service Worker is actually pretty slim, so here it is in its entirety: +

+ 
const VERSION = '0.1.3';
+const CACHE_KEYS = {
+  MAIN: `main-${VERSION}`
+};
+
+// URLS that we want to be cached when the worker is installed
+const PRE_CACHE_URLS = ['/', '/css/global.css', '/js/app.js', '/js/components/content.js'];
+
+/**
+ * Takes an array of strings and puts them in a named cache store
+ *
+ * @param {String} cacheName
+ * @param {Array} items=[]
+ */
+const addItemsToCache = function(cacheName, items = []) {
+  caches.open(cacheName).then(cache => cache.addAll(items));
+};
+
+self.addEventListener('install', evt => {
+  self.skipWaiting();
+
+  addItemsToCache(CACHE_KEYS.MAIN, PRE_CACHE_URLS);
+});
+
+self.addEventListener('activate', evt => {
+  // Look for any old caches that don't match our set and clear them out
+  evt.waitUntil(
+    caches
+      .keys()
+      .then(cacheNames => {
+        return cacheNames.filter(item => !Object.values(CACHE_KEYS).includes(item));
+      })
+      .then(itemsToDelete => {
+        return Promise.all(
+          itemsToDelete.map(item => {
+            return caches.delete(item);
+          })
+        );
+      })
+      .then(() => self.clients.claim())
+  );
+});
+
+self.addEventListener('fetch', evt => {
+  evt.respondWith(
+    caches.match(evt.request).then(cachedResponse => {
+      // Item found in cache so return
+      if (cachedResponse) {
+        return cachedResponse;
+      }
+
+      // Nothing found so load up the request from the network
+      return caches.open(CACHE_KEYS.MAIN).then(cache => {
+        return fetch(evt.request)
+          .then(response => {
+            // Put the new response in cache and return it
+            return cache.put(evt.request, response.clone()).then(() => {
+              return response;
+            });
+          })
+          .catch(ex => {
+            return;
+          });
+      });
+    })
+  );
+});
+

What the Service Worker does here is pre-cache our core assets that we define in PRE_CACHE_URLS. Then, for each fetch event which is called per request, it’ll try to fulfil the request from cache first. If it can’t do that, it’ll load the remote request for us. With this setup, we achieve two things:

+
    +
  1. We get offline support because we stick our critical assets in cache immediately so they will be accessible offline
    2. +
  2. Once those critical assets and any other requested assets are cached, the app will run faster by default
    3. +
+

Importantly now, because we have a manifest, some shortcut icons and a Service Worker that gives us offline support, we have a fully installable PWA!

+

Wrapping up

+

I hope with this simplified example you can see how approaching web design and development with a progressive enhancement approach, everyone gets an acceptable experience instead of those who are lucky enough to get every aspect of the page at the right time.

+

Jotter is very much live and in the process of being enhanced further, which you can see on its little in-app roadmap, so go ahead and play around with it.

+

Before you know it, it’ll be a car itself, but remember: it’ll always start as a humble little <textarea>.

+
+
+ +
+
+

About the author

+
+
+
+ +

Andy Bell is an independent designer and front-end developer who’s trying to make everyone’s experience on the web better with a focus on progressive enhancement and accessibility.

+

More articles by Andy

+ +
+
+
+ + + + + + + + + + +
+
+

Related articles

+
+
+
    + +
  1. + +
    +
    +

    + Be Fluid with Your Design Skills: Build Your Own Sites +

    +

    + + + Ros Horner + +

    +
    +
    +

    Ros Horner rings out a Christmas message for designers far and near of peace and goodwill to all, especially if they’re developers. With a rallying cry to take back control to see your own designs realised, young or old, merry or sober, the story is clear; as you design, so should you build.

    +
    +
    +

    + + +

    +
    +
    +
    2. + +
  2. + +
    +
    +

    + Designing Your Future +

    +

    + + + Christopher Murphy + +

    +
    +
    +

    Christopher Murphy channels the Ghost of Christmas Yet-to-Come by not just look into the future, but shaping the form it takes. By taking action now you can affect the outcome down the road, making all the difference when it comes to a big life change such as leaving full time employment.

    +
    +
    +

    + + +

    +
    +
    +
    3. + +
  3. + +
    +
    +

    + Five Ways to Animate Responsibly +

    +

    + + + Rachel Nabors + +

    +
    +
    +

    Rachel Nabors clears the snowy drift of delight from web animation to reveal the need for necessity and usefulness when we decide to animate web interactions. The box it comes in is as important as the gift.

    +
    +
    +

    + + +

    +
    +
    +
    4. + +
  4. + +
    +
    +

    + Jobs-to-Be-Done in Your UX Toolbox +

    +

    + + + Steph Troeth + +

    +
    +
    +

    Steph Troeth rallies the workshop elves around an idea for revolutionising their worksheets and giving them a new way to think about approaching each job. One thing’s for certain, as Christmas approaches there’s always plenty of jobs to be done.

    +
    +
    +

    + + +

    +
    +
    +
    5. + +
  5. + +
    +
    +

    + Levelling Up for Junior Developers +

    +

    + + + Dean Hume + +

    +
    +
    +

    Dean Hume places another log on the fire, sets the poker back on its stand, pulls up and chair and gathers the junior developers around the hearth to impart some wisdom. Whether you’re just starting out or have been in the game some time, we can all benefit from a little levelling up.

    +
    +
    +

    + + +

    +
    +
    +
    6. + +
  6. + +
    +
    +

    + Solve the Hard Problems +

    +

    + + + Drew McLellan + +

    +
    +
    +

    Drew McLellan brings our 2015 calendar to a motivational close with some encouragement for the year ahead. Year’s end is a time for reflection and finding new purpose and enthusiasm for what we do. By tackling the thorniest design and development problems, we can make the greatest impact – and have the most fun. Merry Christmas and a happy New Year!

    +
    +
    +

    + + +

    +
    +
    +
    7. + +
+
+
+ + +
+
+

Comments

+
+
+
+

No comments yet - leave yours

+
+
+
+ + + + +
+
+ diff --git a/tests/test_title.py b/tests/test_title.py new file mode 100644 index 00000000..b5090844 --- /dev/null +++ b/tests/test_title.py @@ -0,0 +1,14 @@ +from .fixtures import * + +def test_title_is_htmlencoded_in_index_html(tmp_path, process): + """ + https://github.com/pirate/ArchiveBox/issues/330 + Unencoded content should not be rendered as it facilitates xss injections + and breaks the layout. + """ + add_process = subprocess.run(['archivebox', 'add', 'http://localhost:8080/static/title_with_html.com.html'], capture_output=True) + + with open(tmp_path / "index.html", "r") as f: + output_html = f.read() + + assert "