1
0
Fork 0
mirror of synced 2024-06-28 11:00:35 +12:00

fix: Update command was failing

This commit is contained in:
Cristian 2021-01-19 11:24:41 -05:00
parent ab311d86e1
commit 13c3650637
2 changed files with 17 additions and 17 deletions

View file

@ -114,7 +114,7 @@ from .logging_util import (
printable_dependency_version, printable_dependency_version,
) )
from .search import flush_search_index, index_links from .search import flush_search_index, index_snapshots
ALLOWED_IN_OUTPUT_DIR = { ALLOWED_IN_OUTPUT_DIR = {
'lost+found', 'lost+found',
@ -700,7 +700,7 @@ def update(resume: Optional[float]=None,
check_data_folder(out_dir=out_dir) check_data_folder(out_dir=out_dir)
check_dependencies() check_dependencies()
new_links: List[Snapshot] = [] # TODO: Remove input argument: only_new new_snapshots: List[Snapshot] = [] # TODO: Remove input argument: only_new
extractors = extractors.split(",") if extractors else [] extractors = extractors.split(",") if extractors else []
@ -717,25 +717,25 @@ def update(resume: Optional[float]=None,
status=status, status=status,
out_dir=out_dir, out_dir=out_dir,
) )
all_links = [link for link in matching_folders.values() if link] all_snapshots = [snapshot for snapshot in matching_folders.values()]
if index_only: if index_only:
for snapshot in all_snapshots: for snapshot in all_snapshots:
write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=True) write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=True)
index_links(all_links, out_dir=out_dir) index_snapshots(all_snapshots, out_dir=out_dir)
return all_links return all_snapshots
# Step 2: Run the archive methods for each link # Step 2: Run the archive methods for each link
to_archive = new_links if only_new else all_links to_archive = new_snapshots if only_new else all_snapshots
if resume: if resume:
to_archive = [ to_archive = [
link for link in to_archive snapshot for snapshot in to_archive
if link.timestamp >= str(resume) if snapshot.timestamp >= str(resume)
] ]
if not to_archive: if not to_archive:
stderr('') stderr('')
stderr(f'[√] Nothing found to resume after {resume}', color='green') stderr(f'[√] Nothing found to resume after {resume}', color='green')
return all_links return all_snapshots
archive_kwargs = { archive_kwargs = {
"out_dir": out_dir, "out_dir": out_dir,
@ -746,8 +746,8 @@ def update(resume: Optional[float]=None,
archive_snapshots(to_archive, overwrite=overwrite, **archive_kwargs) archive_snapshots(to_archive, overwrite=overwrite, **archive_kwargs)
# Step 4: Re-write links index with updated titles, icons, and resources # Step 4: Re-write links index with updated titles, icons, and resources
all_links = load_main_index(out_dir=out_dir) all_snapshots = load_main_index(out_dir=out_dir)
return all_links return all_snapshots
@enforce_types @enforce_types
def list_all(filter_patterns_str: Optional[str]=None, def list_all(filter_patterns_str: Optional[str]=None,

View file

@ -83,17 +83,17 @@ def flush_search_index(snapshots: QuerySet):
) )
@enforce_types @enforce_types
def index_links(links: Union[List[Link],None], out_dir: Path=OUTPUT_DIR): def index_snapshots(snapshots: Union[List[Model],None], out_dir: Path=OUTPUT_DIR):
if not links: if not snapshots:
return return
from core.models import Snapshot, ArchiveResult from core.models import Snapshot, ArchiveResult
for link in links: for snapshot in snapshots:
snap = Snapshot.objects.filter(url=link.url).first() snap = Snapshot.objects.filter(url=snapshot.url).first()
if snap: if snap:
results = ArchiveResult.objects.indexable().filter(snapshot=snap) results = ArchiveResult.objects.indexable().filter(snapshot=snap)
log_index_started(link.url) log_index_started(snapshot.url)
try: try:
texts = get_indexable_content(results) texts = get_indexable_content(results)
except Exception as err: except Exception as err:
@ -103,4 +103,4 @@ def index_links(links: Union[List[Link],None], out_dir: Path=OUTPUT_DIR):
color='red', color='red',
) )
else: else:
write_search_index(link, texts, out_dir=out_dir) write_search_index(snapshot, texts, out_dir=out_dir)