diff --git a/archivebox/cli/archivebox_list.py b/archivebox/cli/archivebox_list.py index dd4b62f8..91b740d7 100644 --- a/archivebox/cli/archivebox_list.py +++ b/archivebox/cli/archivebox_list.py @@ -7,11 +7,21 @@ __description__ = 'List all the URLs currently in the archive.' import sys import argparse - -from ..legacy.util import reject_stdin, to_json, to_csv -from ..legacy.config import check_data_folder -from ..legacy.main import list_archive_data - +from ..legacy.util import SmartFormatter, reject_stdin, to_json, to_csv +from ..legacy.config import check_data_folder, OUTPUT_DIR +from ..legacy.main import ( + list_archive_data, + get_indexed_folders, + get_archived_folders, + get_unarchived_folders, + get_present_folders, + get_valid_folders, + get_invalid_folders, + get_duplicate_folders, + get_orphaned_folders, + get_corrupted_folders, + get_unrecognized_folders, +) def main(args=None): check_data_folder() @@ -22,6 +32,7 @@ def main(args=None): prog=__command__, description=__description__, add_help=True, + formatter_class=SmartFormatter, ) group = parser.add_mutually_exclusive_group() group.add_argument( @@ -44,15 +55,36 @@ def main(args=None): parser.add_argument( '--before', #'-b', type=float, - help="List only URLs bookmarked before the given timestamp.", + help="List only links bookmarked before the given timestamp.", default=None, ) parser.add_argument( '--after', #'-a', type=float, - help="List only URLs bookmarked after the given timestamp.", + help="List only links bookmarked after the given timestamp.", default=None, ) + parser.add_argument( + '--status', + type=str, + choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'), + default='indexed', + help=( + 'List only links or data directories that have the given status\n' + f' indexed {get_indexed_folders.__doc__} (the default)\n' + f' archived {get_archived_folders.__doc__}\n' + f' unarchived {get_unarchived_folders.__doc__}\n' + '\n' + f' present {get_present_folders.__doc__}\n' + f' valid {get_valid_folders.__doc__}\n' + f' invalid {get_invalid_folders.__doc__}\n' + '\n' + f' duplicate {get_duplicate_folders.__doc__}\n' + f' orphaned {get_orphaned_folders.__doc__}\n' + f' corrupted {get_corrupted_folders.__doc__}\n' + f' unrecognized {get_unrecognized_folders.__doc__}\n' + ) + ) parser.add_argument( '--filter-type', type=str, @@ -76,17 +108,40 @@ def main(args=None): before=command.before, after=command.after, ) + if command.sort: links = sorted(links, key=lambda link: getattr(link, command.sort)) + if command.status == 'indexed': + folders = get_indexed_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'archived': + folders = get_archived_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'unarchived': + folders = get_unarchived_folders(links, out_dir=OUTPUT_DIR) + + elif command.status == 'present': + folders = get_present_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'valid': + folders = get_valid_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'invalid': + folders = get_invalid_folders(links, out_dir=OUTPUT_DIR) + + elif command.status == 'duplicate': + folders = get_duplicate_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'orphaned': + folders = get_orphaned_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'corrupted': + folders = get_corrupted_folders(links, out_dir=OUTPUT_DIR) + elif command.status == 'unrecognized': + folders = get_unrecognized_folders(links, out_dir=OUTPUT_DIR) if command.csv: - print(to_csv(links, csv_cols=command.csv.split(','), header=True)) + print(to_csv(folders.values(), csv_cols=command.csv.split(','), header=True)) elif command.json: - print(to_json(list(links), indent=4, sort_keys=True)) + print(to_json(folders.values(), indent=4, sort_keys=True)) else: - print('\n'.join(link.url for link in links)) - + print('\n'.join(f'{folder} {link}' for folder, link in folders.items())) + raise SystemExit(not folders) if __name__ == '__main__': main() diff --git a/archivebox/legacy/main.py b/archivebox/legacy/main.py index 1921ff7c..b3f15a79 100644 --- a/archivebox/legacy/main.py +++ b/archivebox/legacy/main.py @@ -2,7 +2,8 @@ import os import re import shutil -from typing import List, Optional, Iterable +from typing import Dict, List, Optional, Iterable +from itertools import chain from .schema import Link from .util import ( @@ -17,8 +18,13 @@ from .index import ( import_new_links, write_main_index, ) -from .storage.json import parse_json_main_index, parse_json_links_details -from .storage.sql import parse_sql_main_index +from .storage.json import ( + parse_json_main_index, + parse_json_link_details, + parse_json_links_details, +) +from .storage.sql import parse_sql_main_index, get_admins +from .storage.html import parse_html_main_index from .archive_methods import archive_link from .config import ( stderr, @@ -164,11 +170,39 @@ def init(): orphaned_data_dir_links = { link.url: link for link in parse_json_links_details(OUTPUT_DIR) - if link.url not in all_links } - if orphaned_data_dir_links: - all_links.update(orphaned_data_dir_links) - print(' {lightyellow}√ Added {} orphaned links from existing archive directories...{reset}'.format(len(orphaned_data_dir_links), **ANSI)) + orphan_new_links = { + url: link + for url, link in orphaned_data_dir_links.items() + if url not in all_links + } + orphan_duplicates = { + url: link + for url, link in orphaned_data_dir_links.items() + if url in all_links + } + if orphan_new_links: + all_links.update(orphan_new_links) + print(' {lightyellow}√ Added {} orphaned links from existing archive directories...{reset}'.format(len(orphan_new_links), **ANSI)) + if orphan_duplicates: + print(' {lightyellow}! Skipped adding {} orphaned link data directories that would have overwritten existing data.{reset}'.format(len(orphan_duplicates), **ANSI)) + + orphaned_data_dirs = {folder for folder in orphan_duplicates.keys()} + invalid_folders = { + folder: link + for folder, link in get_invalid_folders(all_links.values(), out_dir=OUTPUT_DIR).items() + if folder not in orphaned_data_dirs + } + if invalid_folders: + print(' {lightyellow}! Skipped adding {} corrupted/unrecognized link data directories that could not be read.{reset}'.format(len(orphan_duplicates), **ANSI)) + + if orphan_duplicates or invalid_folders: + print(' For more information about the link data directories that were skipped, run:') + print(' archivebox info') + print(' archivebox list --status=invalid') + print(' archivebox list --status=orphaned') + print(' archivebox list --status=duplicate') + write_main_index(list(all_links.values()), out_dir=OUTPUT_DIR) @@ -190,55 +224,88 @@ def init(): @enforce_types def info(): - all_links = load_main_index(out_dir=OUTPUT_DIR) - print('{green}[*] Scanning archive collection main index with {} links:{reset}'.format(len(all_links), **ANSI)) - print(f' {OUTPUT_DIR}') - num_bytes, num_dirs, num_files = get_dir_size(OUTPUT_DIR, recursive=False) + print('{green}[*] Scanning archive collection main index...{reset}'.format(**ANSI)) + print(f' {OUTPUT_DIR}/*') + num_bytes, num_dirs, num_files = get_dir_size(OUTPUT_DIR, recursive=False, pattern='index.') size = human_readable_size(num_bytes) - print(f' > Index Size: {size} across {num_files} files') + print(f' Size: {size} across {num_files} files') print() - setup_django() - from django.contrib.auth.models import User - from core.models import Page + links = load_main_index(out_dir=OUTPUT_DIR) + num_json_links = len(links) + num_sql_links = sum(1 for link in parse_sql_main_index(out_dir=OUTPUT_DIR)) + num_html_links = sum(1 for url in parse_html_main_index(out_dir=OUTPUT_DIR)) + num_link_details = sum(1 for link in parse_json_links_details(out_dir=OUTPUT_DIR)) + users = get_admins().values_list('username', flat=True) + print(f' > JSON Main Index: {num_json_links} links'.ljust(36), f'(found in {JSON_INDEX_FILENAME})') + print(f' > SQL Main Index: {num_sql_links} links'.ljust(36), f'(found in {SQL_INDEX_FILENAME})') + print(f' > HTML Main Index: {num_html_links} links'.ljust(36), f'(found in {HTML_INDEX_FILENAME})') + print(f' > JSON Link Details: {num_link_details} links'.ljust(36), f'(found in {ARCHIVE_DIR_NAME}/*/index.json)') - users = User.objects.all() - num_pages = Page.objects.count() + print(f' > Admin: {len(users)} users {", ".join(users)}'.ljust(36), f'(found in {SQL_INDEX_FILENAME})') - print(f' > {len(users)} admin users:', ', '.join(u.username for u in users)) - print(f' > {num_pages} pages in SQL database {SQL_INDEX_FILENAME}') - print(f' > {len(all_links)} pages in JSON database {JSON_INDEX_FILENAME}') - print() + if num_html_links != len(links) or num_sql_links != len(links): + print() + print(' {lightred}Hint:{reset} You can fix index count differences automatically by running:'.format(**ANSI)) + print(' archivebox init') + + if not users: + print() + print(' {lightred}Hint:{reset} You can create an admin user by running:'.format(**ANSI)) + print(' archivebox manage createsuperuser') - print('{green}[*] Scanning archive collection data directory with {} entries:{reset}'.format(len(all_links), **ANSI)) - print(f' {ARCHIVE_DIR}') + print() + print('{green}[*] Scanning archive collection link data directories...{reset}'.format(**ANSI)) + print(f' {ARCHIVE_DIR}/*') num_bytes, num_dirs, num_files = get_dir_size(ARCHIVE_DIR) size = human_readable_size(num_bytes) - print(f' > Total Size: {size} across {num_files} files in {num_dirs} directories') + print(f' Size: {size} across {num_files} files in {num_dirs} directories') print() - link_data_dirs = {link.link_dir for link in all_links} - valid_archive_dirs = set() - num_invalid = 0 - for entry in os.scandir(ARCHIVE_DIR): - if entry.is_dir(follow_symlinks=True): - if os.path.exists(os.path.join(entry.path, 'index.json')): - valid_archive_dirs.add(entry.path) - else: - num_invalid += 1 - - print(f' > {len(valid_archive_dirs)} valid archive data directories (valid directories matched to links in the index)') - - num_unarchived = sum(1 for link in all_links if link.link_dir not in valid_archive_dirs) - print(f' > {num_unarchived} missing data directories (directories missing for links in the index)') - - print(f' > {num_invalid} invalid data directories (directories present that don\'t contain an index file)') - - num_orphaned = sum(1 for data_dir in valid_archive_dirs if data_dir not in link_data_dirs) - print(f' > {num_orphaned} orphaned data directories (directories present for links that don\'t exist in the index)') + num_indexed = len(get_indexed_folders(links, out_dir=OUTPUT_DIR)) + num_archived = len(get_archived_folders(links, out_dir=OUTPUT_DIR)) + num_unarchived = len(get_unarchived_folders(links, out_dir=OUTPUT_DIR)) + print(f' > indexed: {num_indexed}'.ljust(36), f'({get_indexed_folders.__doc__})') + print(f' > archived: {num_archived}'.ljust(36), f'({get_archived_folders.__doc__})') + print(f' > unarchived: {num_unarchived}'.ljust(36), f'({get_unarchived_folders.__doc__})') + num_present = len(get_present_folders(links, out_dir=OUTPUT_DIR)) + num_valid = len(get_valid_folders(links, out_dir=OUTPUT_DIR)) + print() + print(f' > present: {num_present}'.ljust(36), f'({get_present_folders.__doc__})') + print(f' > valid: {num_valid}'.ljust(36), f'({get_valid_folders.__doc__})') + + duplicate = get_duplicate_folders(links, out_dir=OUTPUT_DIR) + orphaned = get_orphaned_folders(links, out_dir=OUTPUT_DIR) + corrupted = get_corrupted_folders(links, out_dir=OUTPUT_DIR) + unrecognized = get_unrecognized_folders(links, out_dir=OUTPUT_DIR) + num_invalid = len({**duplicate, **orphaned, **corrupted, **unrecognized}) + print(f' > invalid: {num_invalid}'.ljust(36), f'({get_invalid_folders.__doc__})') + print(f' > duplicate: {len(duplicate)}'.ljust(36), f'({get_duplicate_folders.__doc__})') + print(f' > orphaned: {len(orphaned)}'.ljust(36), f'({get_orphaned_folders.__doc__})') + print(f' > corrupted: {len(corrupted)}'.ljust(36), f'({get_corrupted_folders.__doc__})') + print(f' > unrecognized: {len(unrecognized)}'.ljust(36), f'({get_unrecognized_folders.__doc__})') + + if num_indexed: + print() + print(' {lightred}Hint:{reset} You can list link data directories by status like so:'.format(**ANSI)) + print(' archivebox list --status= (e.g. indexed, corrupted, archived, etc.)') + + if orphaned: + print() + print(' {lightred}Hint:{reset} To automatically import orphaned data directories into the main index, run:'.format(**ANSI)) + print(' archivebox init') + + if num_invalid: + print() + print(' {lightred}Hint:{reset} You may need to manually remove or fix some invalid data directories, afterwards make sure to run:'.format(**ANSI)) + print(' archivebox init') + + print() + + @enforce_types def update_archive_data(import_path: Optional[str]=None, resume: Optional[float]=None, only_new: bool=False) -> List[Link]: @@ -367,3 +434,182 @@ def remove_archive_links(filter_patterns: List[str], filter_type: str='exact', log_removal_finished(len(all_links), len(to_keep)) return to_keep + + + +def get_indexed_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """indexed links without checking archive status or data directory validity""" + return { + link.link_dir: link + for link in links + } + +def get_archived_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """indexed links that are archived with a valid data directory""" + return { + link.link_dir: link + for link in filter(is_archived, links) + } + +def get_unarchived_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """indexed links that are unarchived with no data directory or an empty data directory""" + return { + link.link_dir: link + for link in filter(is_unarchived, links) + } + +def get_present_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs that are expected to exist based on the main index""" + all_folders = {} + + for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)): + if entry.is_dir(follow_symlinks=True): + link = None + try: + link = parse_json_link_details(entry.path) + except Exception: + pass + + all_folders[entry.path] = link + + return all_folders + +def get_valid_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs with a valid index matched to the main index and archived content""" + return { + link.link_dir: link + for link in filter(is_valid, links) + } + +def get_invalid_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs that are invalid for any reason: corrupted/duplicate/orphaned/unrecognized""" + duplicate = get_duplicate_folders(links, out_dir=OUTPUT_DIR) + orphaned = get_orphaned_folders(links, out_dir=OUTPUT_DIR) + corrupted = get_corrupted_folders(links, out_dir=OUTPUT_DIR) + unrecognized = get_unrecognized_folders(links, out_dir=OUTPUT_DIR) + return {**duplicate, **orphaned, **corrupted, **unrecognized} + + +def get_duplicate_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs that conflict with other directories that have the same link URL or timestamp""" + links = list(links) + by_url = {link.url: 0 for link in links} + by_timestamp = {link.timestamp: 0 for link in links} + + duplicate_folders = {} + + indexed_folders = {link.link_dir for link in links} + data_folders = ( + entry.path + for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)) + if entry.is_dir(follow_symlinks=True) and entry.path not in indexed_folders + ) + + for path in chain(sorted(indexed_folders), sorted(data_folders)): + link = None + try: + link = parse_json_link_details(path) + except Exception: + pass + + if link: + # link folder has same timestamp as different link folder + by_timestamp[link.timestamp] = by_timestamp.get(link.timestamp, 0) + 1 + if by_timestamp[link.timestamp] > 1: + duplicate_folders[path] = link + + # link folder has same url as different link folder + by_url[link.url] = by_url.get(link.url, 0) + 1 + if by_url[link.url] > 1: + duplicate_folders[path] = link + + return duplicate_folders + +def get_orphaned_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs that contain a valid index but aren't listed in the main index""" + links = list(links) + indexed_folders = {link.link_dir: link for link in links} + orphaned_folders = {} + + for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)): + if entry.is_dir(follow_symlinks=True): + index_exists = os.path.exists(os.path.join(entry.path, 'index.json')) + link = None + try: + link = parse_json_link_details(entry.path) + except Exception: + pass + + if index_exists and entry.path not in indexed_folders: + # folder is a valid link data dir with index details, but it's not in the main index + orphaned_folders[entry.path] = link + + return orphaned_folders + +def get_corrupted_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs that don't contain a valid index and aren't listed in the main index""" + return { + link.link_dir: link + for link in filter(is_corrupt, links) + } + +def get_unrecognized_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Optional[Link]]: + """dirs that don't contain recognizable archive data and aren't listed in the main index""" + by_timestamp = {link.timestamp: 0 for link in links} + unrecognized_folders: Dict[str, Optional[Link]] = {} + + for entry in os.scandir(os.path.join(out_dir, ARCHIVE_DIR_NAME)): + if entry.is_dir(follow_symlinks=True): + index_exists = os.path.exists(os.path.join(entry.path, 'index.json')) + link = None + try: + link = parse_json_link_details(entry.path) + except Exception: + pass + + if index_exists and link is None: + # index exists but it's corrupted or unparseable + unrecognized_folders[entry.path] = link + + elif not index_exists: + # link details index doesn't exist and the folder isn't in the main index + timestamp = entry.path.rsplit('/', 1)[-1] + if timestamp not in by_timestamp: + unrecognized_folders[entry.path] = link + + return unrecognized_folders + + +def is_valid(link: Link) -> bool: + dir_exists = os.path.exists(link.link_dir) + index_exists = os.path.exists(os.path.join(link.link_dir, 'index.json')) + if not dir_exists: + # unarchived links are not included in the valid list + return False + if dir_exists and not index_exists: + return False + if dir_exists and index_exists: + try: + parsed_link = parse_json_link_details(link.link_dir) + return link.url == parsed_link.url + except Exception: + pass + return False + +def is_corrupt(link: Link) -> bool: + if not os.path.exists(link.link_dir): + # unarchived links are not considered corrupt + return False + + if is_valid(link): + return False + + return True + +def is_archived(link: Link) -> bool: + return is_valid(link) and link.is_archived + +def is_unarchived(link: Link) -> bool: + if not os.path.exists(link.link_dir): + return True + return not link.is_archived