add DOM archiving with chrome headless
This commit is contained in:
Nick Sweeting
parent
2e19050aba
commit
f78838ef40
|
|
@ -17,6 +17,7 @@ from config import (
|
|||
FETCH_WGET_REQUISITES,
|
||||
FETCH_PDF,
|
||||
FETCH_SCREENSHOT,
|
||||
FETCH_DOM,
|
||||
RESOLUTION,
|
||||
CHECK_SSL_VALIDITY,
|
||||
SUBMIT_ARCHIVE_DOT_ORG,
|
||||
|
|
@ -93,6 +94,9 @@ def archive_link(link_dir, link, overwrite=True):
|
|||
if FETCH_SCREENSHOT:
|
||||
link = fetch_screenshot(link_dir, link, overwrite=overwrite)
|
||||
|
||||
if FETCH_DOM:
|
||||
link = fetch_dom(link_dir, link, overwrite=overwrite)
|
||||
|
||||
if SUBMIT_ARCHIVE_DOT_ORG:
|
||||
link = archive_dot_org(link_dir, link, overwrite=overwrite)
|
||||
|
||||
|
|
@ -252,7 +256,6 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
|
|||
'output': output,
|
||||
}
|
||||
|
||||
|
||||
@attach_result_to_link('screenshot')
|
||||
def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR, resolution=RESOLUTION):
|
||||
"""take screenshot of site using chrome --headless"""
|
||||
|
|
@ -289,6 +292,43 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
|
|||
'output': output,
|
||||
}
|
||||
|
||||
@attach_result_to_link('dom')
|
||||
def fetch_dom(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR):
|
||||
"""print HTML of site to file using chrome --dump-html"""
|
||||
|
||||
if link['type'] in ('PDF', 'image'):
|
||||
return {'output': wget_output_path(link)}
|
||||
|
||||
output_path = os.path.join(link_dir, 'output.html')
|
||||
|
||||
if os.path.exists(output_path):
|
||||
return {'output': 'output.html', 'status': 'skipped'}
|
||||
|
||||
CMD = [
|
||||
*chrome_headless(user_data_dir=user_data_dir),
|
||||
'--dump-dom',
|
||||
link['url']
|
||||
]
|
||||
end = progress(timeout, prefix=' ')
|
||||
try:
|
||||
with open(output_path, 'w+') as f:
|
||||
result = run(CMD, stdout=f, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # output.html
|
||||
end()
|
||||
if result.returncode:
|
||||
print(' ', (result.stderr).decode())
|
||||
raise Exception('Failed to fetch DOM')
|
||||
chmod_file('output.html', cwd=link_dir)
|
||||
output = 'output.html'
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
'cmd': CMD,
|
||||
'output': output,
|
||||
}
|
||||
|
||||
@attach_result_to_link('archive_org')
|
||||
def archive_dot_org(link_dir, link, timeout=TIMEOUT):
|
||||
|
|
@ -445,7 +485,7 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
|
|||
|
||||
|
||||
def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR):
|
||||
args = [binary, '--headless', '--disable-gpu']
|
||||
args = [binary, '--headless'] # '--disable-gpu'
|
||||
default_profile = os.path.expanduser('~/Library/Application Support/Google/Chrome/Default')
|
||||
if user_data_dir:
|
||||
args.append('--user-data-dir={}'.format(user_data_dir))
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ FETCH_AUDIO = os.getenv('FETCH_AUDIO', 'False'
|
|||
FETCH_VIDEO = os.getenv('FETCH_VIDEO', 'False' ).lower() == 'true'
|
||||
FETCH_PDF = os.getenv('FETCH_PDF', 'True' ).lower() == 'true'
|
||||
FETCH_SCREENSHOT = os.getenv('FETCH_SCREENSHOT', 'True' ).lower() == 'true'
|
||||
FETCH_DOM = os.getenv('FETCH_DOM', 'True' ).lower() == 'true'
|
||||
FETCH_FAVICON = os.getenv('FETCH_FAVICON', 'True' ).lower() == 'true'
|
||||
SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True' ).lower() == 'true'
|
||||
RESOLUTION = os.getenv('RESOLUTION', '1440,1200' )
|
||||
|
|
|
|||
3
links.py
3
links.py
|
|
@ -68,6 +68,9 @@ def validate_links(links):
|
|||
if not link['latest'].get('screenshot'):
|
||||
link['latest']['screenshot'] = None
|
||||
|
||||
if not link['latest'].get('dom'):
|
||||
link['latest']['dom'] = None
|
||||
|
||||
return list(links)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@
|
|||
<th style="width: 2vw;">Status</th>
|
||||
<th style="width: 43vw;">Saved Articles ($num_links)</th>
|
||||
<th style="width: 50px">Index</th>
|
||||
<th style="width: 50px">HTML</th>
|
||||
<th style="width: 50px">PDF</th>
|
||||
<th style="width: 60px;font-size:0.8em;">Screenshot</th>
|
||||
<th style="width: 50px">A.org</th>
|
||||
|
|
|
|||
|
|
@ -5,12 +5,13 @@
|
|||
<img src="$favicon_url" onerror="this.src='static/spinner.gif'" class="link-favicon">
|
||||
</a>
|
||||
</td>
|
||||
<td><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
|
||||
<td style="text-align: left"><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
|
||||
$title <small style="background-color: #eee;border-radius:4px; float:right">$tags</small>
|
||||
</td>
|
||||
<td style="text-align:center"><a href="$files_url" title="Files">📂</a></td>
|
||||
<td style="text-align:center"><a href="$pdf_link" title="PDF">📄</a></td>
|
||||
<td style="text-align:center"><a href="$screenshot_link" title="Screenshot">🖼</a></td>
|
||||
<td style="text-align:center"><a href="$archive_org_url" title="Archive.org">🏛</a></td>
|
||||
<td>🔗 <img src="$google_favicon_url" height="16px"> <a href="$url">$url</a></td>
|
||||
<td><a href="$files_url" title="Files">📂</a></td>
|
||||
<td><a href="$dom_link" title="HTML">📄</a></td>
|
||||
<td><a href="$pdf_link" title="PDF">📜</a></td>
|
||||
<td><a href="$screenshot_link" title="Screenshot">🖼</a></td>
|
||||
<td><a href="$archive_org_url" title="Archive.org">🏛</a></td>
|
||||
<td style="text-align: left"><!--🔗 <img src="$google_favicon_url" height="16px">--> <a href="$url">$url</a></td>
|
||||
</tr>
|
||||
|
|
|
|||
7
util.py
7
util.py
|
|
@ -23,6 +23,7 @@ from config import (
|
|||
FETCH_WGET,
|
||||
FETCH_PDF,
|
||||
FETCH_SCREENSHOT,
|
||||
FETCH_DOM,
|
||||
FETCH_FAVICON,
|
||||
FETCH_AUDIO,
|
||||
FETCH_VIDEO,
|
||||
|
|
@ -49,7 +50,7 @@ def check_dependencies():
|
|||
print(' See https://github.com/pirate/bookmark-archiver#troubleshooting for help upgrading your Python installation.')
|
||||
raise SystemExit(1)
|
||||
|
||||
if FETCH_PDF or FETCH_SCREENSHOT:
|
||||
if FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM:
|
||||
if run(['which', CHROME_BINARY], stdout=DEVNULL).returncode:
|
||||
print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset']))
|
||||
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
|
||||
|
|
@ -64,7 +65,7 @@ def check_dependencies():
|
|||
version = [l for l in version_lines if l.isdigit()][-1]
|
||||
if int(version) < 59:
|
||||
print(version_lines)
|
||||
print('{red}[X] Chrome version must be 59 or greater for headless PDF and screenshot saving{reset}'.format(**ANSI))
|
||||
print('{red}[X] Chrome version must be 59 or greater for headless PDF, screenshot, and DOM saving{reset}'.format(**ANSI))
|
||||
print(' See https://github.com/pirate/bookmark-archiver for help.')
|
||||
raise SystemExit(1)
|
||||
except (IndexError, TypeError, OSError):
|
||||
|
|
@ -459,6 +460,7 @@ def derived_link_info(link):
|
|||
'archive_url': 'archive/{}/{}'.format(link['timestamp'], wget_output_path(link)),
|
||||
'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
|
||||
'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
|
||||
'dom_link': 'archive/{timestamp}/output.html'.format(**link),
|
||||
'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
|
||||
}
|
||||
|
||||
|
|
@ -469,6 +471,7 @@ def derived_link_info(link):
|
|||
'archive_url': 'archive/{timestamp}/{base_url}'.format(**link),
|
||||
'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
|
||||
'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
|
||||
'dom_link': 'archive/{timestamp}/{base_url}'.format(**link),
|
||||
'title': '{title} ({type})'.format(**link),
|
||||
})
|
||||
return link_info
|
||||
|
|
|
|||
Loading…
Reference in a new issue