CodeMirror/ArchiveBox
1
0
Fork 0
mirror of synced 2024-05-16 10:23:51 +12:00
Code Issues Projects Releases Wiki Activity

Fix cookies being set incorrectly

Browse source
This commit is contained in:
Ben Muthalaly 2024-03-05 01:48:35 -06:00
parent 68326a60ee
commit 4686da91e6
1 changed files with 11 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
archivebox/util.py
View file

@ -166,25 +166,25 @@ def parse_date(date: Any) -> Optional[datetime]:
def download_url(url: str, timeout: int=None) -> str: def download_url(url: str, timeout: int=None) -> str:
"""Download the contents of a remote url and return the text""" """Download the contents of a remote url and return the text"""
from .config import ( from .config import (
TIMEOUT, TIMEOUT,
CHECK_SSL_VALIDITY, CHECK_SSL_VALIDITY,
WGET_USER_AGENT, WGET_USER_AGENT,
COOKIES_FILE, COOKIES_FILE,
) )
timeout = timeout or TIMEOUT timeout = timeout or TIMEOUT
session = requests.Session()
cookie_jar = http.cookiejar.MozillaCookieJar() if COOKIES_FILE and Path(COOKIES_FILE).is_file():
if COOKIES_FILE is not None: cookie_jar = http.cookiejar.MozillaCookieJar(COOKIES_FILE)
cookie_jar.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True) cookie_jar.load(ignore_discard=True, ignore_expires=True)
else: for cookie in cookie_jar:
cookie_jar = None session.cookies.set(cookie.name, cookie.value, domain=cookie.domain, path=cookie.path)
response = requests.get( response = session.get(
url, url,
headers={'User-Agent': WGET_USER_AGENT}, headers={'User-Agent': WGET_USER_AGENT},
verify=CHECK_SSL_VALIDITY, verify=CHECK_SSL_VALIDITY,
timeout=timeout, timeout=timeout,
cookies=cookie_jar,
) )
content_type = response.headers.get('Content-Type', '') content_type = response.headers.get('Content-Type', '')