CodeMirror/ArchiveBox
1
0
Fork 0
mirror of synced 2024-06-27 02:20:36 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into dev

Browse source
This commit is contained in:
Nick Sweeting 2019-03-27 10:36:29 -04:00
parent 0835baf435 e797a7a5f3
commit 7ae680a461
2 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archivebox/archive_methods.py
View file

@ -41,6 +41,7 @@ from config import (
GIT_VERSION,
YOUTUBEDL_VERSION,
ONLY_NEW,
WGET_AUTO_COMPRESSION,
)
from util import (
enforce_types,
@ -251,7 +252,6 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult
'--backup-converted',
'--span-hosts',
'--no-parent',
'--compression=auto',
'-e', 'robots=off',
'--restrict-file-names=unix',
'--timeout={}'.format(timeout),
@ -260,6 +260,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult
*(('--page-requisites',) if FETCH_WGET_REQUISITES else ()),
*(('--user-agent={}'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()),
*(('--load-cookies', COOKIES_FILE) if COOKIES_FILE else ()),
*(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()),
*((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', '--no-hsts'))),
link.url,
]

4
archivebox/config.py
View file

@ -213,14 +213,16 @@ try:
else:
FETCH_WGET = FETCH_WARC = False
WGET_VERSION = None
WGET_AUTO_COMPRESSION = False
if USE_WGET:
WGET_VERSION = check_version(WGET_BINARY)
WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode
WGET_USER_AGENT = WGET_USER_AGENT.format(
GIT_SHA=GIT_SHA[:9],
WGET_VERSION=WGET_VERSION or '',
)
### Make sure git is installed
GIT_VERSION = None
if FETCH_GIT: