From 6006b4f93b5fb51d7b2fa4074c6720d46ec65b91 Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 24 Jul 2020 12:25:25 -0500 Subject: [PATCH] refactor: Organize code to remove flake8 issues --- archivebox/cli/archivebox_add.py | 3 ++- archivebox/cli/archivebox_config.py | 3 ++- archivebox/cli/archivebox_help.py | 3 ++- archivebox/cli/archivebox_init.py | 3 ++- archivebox/cli/archivebox_list.py | 3 ++- archivebox/cli/archivebox_manage.py | 3 ++- archivebox/cli/archivebox_remove.py | 3 ++- archivebox/cli/archivebox_schedule.py | 3 ++- archivebox/cli/archivebox_server.py | 3 ++- archivebox/cli/archivebox_shell.py | 3 ++- archivebox/cli/archivebox_status.py | 3 ++- archivebox/cli/archivebox_update.py | 3 ++- archivebox/cli/archivebox_version.py | 3 ++- archivebox/cli/tests.py | 2 +- archivebox/config/__init__.py | 2 +- archivebox/core/settings.py | 1 - archivebox/core/tests.py | 2 +- archivebox/core/views.py | 3 +-- archivebox/core/welcome_message.py | 1 - archivebox/index/__init__.py | 5 ++--- archivebox/logging_util.py | 10 ++++------ archivebox/main.py | 7 +++---- archivebox/parsers/__init__.py | 4 +--- archivebox/util.py | 6 +++--- 24 files changed, 43 insertions(+), 39 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index de6c3dbc..b9c06a55 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -8,7 +8,8 @@ import argparse from typing import List, Optional, IO -from ..main import add, docstring +from ..main import add +from ..util import docstring from ..config import OUTPUT_DIR, ONLY_NEW from ..logging_util import SmartFormatter, accept_stdin, stderr diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py index 7dffbfad..f81286c6 100644 --- a/archivebox/cli/archivebox_config.py +++ b/archivebox/cli/archivebox_config.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import config, docstring +from ..main import config +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, accept_stdin diff --git a/archivebox/cli/archivebox_help.py b/archivebox/cli/archivebox_help.py index 0b245a03..46f17cbc 100755 --- a/archivebox/cli/archivebox_help.py +++ b/archivebox/cli/archivebox_help.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import help, docstring +from ..main import help +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py index e27b26d9..6255ef26 100755 --- a/archivebox/cli/archivebox_init.py +++ b/archivebox/cli/archivebox_init.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import init, docstring +from ..main import init +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/archivebox_list.py b/archivebox/cli/archivebox_list.py index 2f2180b0..95c5cc4e 100644 --- a/archivebox/cli/archivebox_list.py +++ b/archivebox/cli/archivebox_list.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import list_all, docstring +from ..main import list_all +from ..util import docstring from ..config import OUTPUT_DIR from ..index import ( get_indexed_folders, diff --git a/archivebox/cli/archivebox_manage.py b/archivebox/cli/archivebox_manage.py index cab5d565..f05604e1 100644 --- a/archivebox/cli/archivebox_manage.py +++ b/archivebox/cli/archivebox_manage.py @@ -7,7 +7,8 @@ import sys from typing import Optional, List, IO -from ..main import manage, docstring +from ..main import manage +from ..util import docstring from ..config import OUTPUT_DIR diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index 36779e87..8fe717fb 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import remove, docstring +from ..main import remove +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, accept_stdin diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index 4dfa2769..ffd09f22 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import schedule, docstring +from ..main import schedule +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 0d529a40..a5c168cc 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import server, docstring +from ..main import server +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/archivebox_shell.py b/archivebox/cli/archivebox_shell.py index 5d5498cc..bcd5fdd6 100644 --- a/archivebox/cli/archivebox_shell.py +++ b/archivebox/cli/archivebox_shell.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import shell, docstring +from ..main import shell +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py index c0bed3fe..2bef19c7 100644 --- a/archivebox/cli/archivebox_status.py +++ b/archivebox/cli/archivebox_status.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import status, docstring +from ..main import status +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 663e03f3..9d483362 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -8,7 +8,8 @@ import argparse from typing import List, Optional, IO -from ..main import update, docstring +from ..main import update +from ..util import docstring from ..config import OUTPUT_DIR from ..index import ( get_indexed_folders, diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py index c56ab43f..e7922f37 100755 --- a/archivebox/cli/archivebox_version.py +++ b/archivebox/cli/archivebox_version.py @@ -8,7 +8,8 @@ import argparse from typing import Optional, List, IO -from ..main import version, docstring +from ..main import version +from ..util import docstring from ..config import OUTPUT_DIR from ..logging_util import SmartFormatter, reject_stdin diff --git a/archivebox/cli/tests.py b/archivebox/cli/tests.py index 14d0e4c6..1f44784d 100755 --- a/archivebox/cli/tests.py +++ b/archivebox/cli/tests.py @@ -198,7 +198,7 @@ class TestRemove(unittest.TestCase): def test_remove_regex(self): with output_hidden(): - archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', 'http(s)?:\/\/(.+\.)?(example\d\.com)']) + archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)']) all_links = load_main_index(out_dir=OUTPUT_DIR) assert len(all_links) == 4 diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py index 87bedbd6..12579dbe 100644 --- a/archivebox/config/__init__.py +++ b/archivebox/config/__init__.py @@ -868,4 +868,4 @@ def setup_django(out_dir: str=None, check_db=False, config: ConfigDict=CONFIG) - except KeyboardInterrupt: raise SystemExit(2) -os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8)) +os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8)) # noqa: F821 diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 9a28b4f6..e9cc0dc8 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -8,7 +8,6 @@ from django.utils.crypto import get_random_string from ..config import ( OUTPUT_DIR, SECRET_KEY, - DEBUG, ALLOWED_HOSTS, PYTHON_DIR, ACTIVE_THEME, diff --git a/archivebox/core/tests.py b/archivebox/core/tests.py index 7ce503c2..4d66077c 100644 --- a/archivebox/core/tests.py +++ b/archivebox/core/tests.py @@ -1,3 +1,3 @@ -from django.test import TestCase +#from django.test import TestCase # Create your tests here. diff --git a/archivebox/core/views.py b/archivebox/core/views.py index ce6e8f58..c411e98b 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -4,7 +4,6 @@ from django.shortcuts import render, redirect from django.http import HttpResponse from django.views import View, static -from django.conf import settings from core.models import Snapshot @@ -75,7 +74,7 @@ class AddLinks(View): } add_stdout = StringIO() with redirect_stdout(add_stdout): - extracted_links = add(**input_kwargs) + add(**input_kwargs) print(add_stdout.getvalue()) context = { diff --git a/archivebox/core/welcome_message.py b/archivebox/core/welcome_message.py index a948c78e..ed5d2d77 100644 --- a/archivebox/core/welcome_message.py +++ b/archivebox/core/welcome_message.py @@ -2,5 +2,4 @@ from archivebox.logging_util import log_shell_welcome_msg if __name__ == '__main__': - from main import * log_shell_welcome_msg() diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py index b4e918b8..09c4d8a3 100644 --- a/archivebox/index/__init__.py +++ b/archivebox/index/__init__.py @@ -294,7 +294,6 @@ def parse_links_from_source(source_path: str) -> Tuple[List[Link], List[Link]]: def dedupe_links(existing_links: List[Link], new_links: List[Link]) -> Tuple[List[Link], List[Link]]: - from ..parsers import parse_links # merge existing links in out_dir and new links all_links = validate_links(existing_links + new_links) all_link_urls = {link.url for link in existing_links} @@ -537,7 +536,7 @@ def get_unrecognized_folders(links, out_dir: str=OUTPUT_DIR) -> Dict[str, Option link_guessed = parse_json_link_details(entry.path, guess=True) write_json_link_details(link_guessed, out_dir=entry.path) link = parse_json_link_details(entry.path) - except Exception as e: + except Exception: pass if index_exists and link is None: @@ -565,7 +564,7 @@ def is_valid(link: Link) -> bool: try: parsed_link = parse_json_link_details(link.link_dir, guess=True) return link.url == parsed_link.url - except Exception as e: + except Exception: pass return False diff --git a/archivebox/logging_util.py b/archivebox/logging_util.py index 361a1aec..d60cf5fe 100644 --- a/archivebox/logging_util.py +++ b/archivebox/logging_util.py @@ -5,8 +5,6 @@ import os import sys import time import argparse -import logging -import signal from multiprocessing import Process from datetime import datetime @@ -263,9 +261,9 @@ def log_archiving_finished(num_links: int): assert _LAST_RUN_STATS.archiving_start_ts is not None seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp() if seconds > 60: - duration = '{0:.2f} min'.format(seconds / 60, 2) + duration = '{0:.2f} min'.format(seconds / 60) else: - duration = '{0:.2f} sec'.format(seconds, 2) + duration = '{0:.2f} sec'.format(seconds) print() print('{}[√] [{}] Update of {} pages complete ({}){}'.format( @@ -385,8 +383,8 @@ def log_removal_started(links: List["Link"], yes: bool, delete: bool): ) else: print( - f' Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n' - f' (Pass --delete if you also want to permanently delete the data folders)' + ' Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n' + ' (Pass --delete if you also want to permanently delete the data folders)' ) if not yes: diff --git a/archivebox/main.py b/archivebox/main.py index 1c7068b3..1cb34b30 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -20,10 +20,9 @@ from .parsers import ( save_file_as_source, ) from .index.schema import Link -from .util import enforce_types, docstring # type: ignore +from .util import enforce_types # type: ignore from .system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT from .index import ( - links_after_timestamp, load_main_index, parse_links_from_source, dedupe_links, @@ -291,7 +290,6 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None: print('\n{green}[+] Building main SQL index and running migrations...{reset}'.format(**ANSI)) setup_django(out_dir, check_db=False) - from django.conf import settings DATABASE_FILE = os.path.join(out_dir, SQL_INDEX_FILENAME) print(f' √ {DATABASE_FILE}') print() @@ -469,7 +467,8 @@ def status(out_dir: str=OUTPUT_DIR) -> None: users = get_admins().values_list('username', flat=True) print(f' UI users {len(users)}: {", ".join(users)}') last_login = User.objects.order_by('last_login').last() - print(f' Last UI login: {last_login.username} @ {str(last_login.last_login)[:16]}') + if last_login: + print(f' Last UI login: {last_login.username} @ {str(last_login.last_login)[:16]}') last_updated = Snapshot.objects.order_by('updated').last() print(f' Last changes: {str(last_updated.updated)[:16]}') diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 40899b06..20c8ef52 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -19,17 +19,15 @@ from ..config import ( OUTPUT_DIR, SOURCES_DIR_NAME, TIMEOUT, - check_data_folder, ) from ..util import ( basename, - domain, download_url, enforce_types, URL_REGEX, ) from ..index.schema import Link -from ..logging_util import pretty_path, TimedProgress, log_source_saved +from ..logging_util import TimedProgress, log_source_saved from .pocket_html import parse_pocket_html_export from .pinboard_rss import parse_pinboard_rss_export from .shaarli_rss import parse_shaarli_rss_export diff --git a/archivebox/util.py b/archivebox/util.py index c43585c0..ca940e30 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -226,11 +226,11 @@ def ansi_to_html(text): argsdict = match.groupdict() if argsdict['arg_3'] is None: if argsdict['arg_2'] is None: - bold, color = 0, argsdict['arg_1'] + _, color = 0, argsdict['arg_1'] else: - bold, color = argsdict['arg_1'], argsdict['arg_2'] + _, color = argsdict['arg_1'], argsdict['arg_2'] else: - bold, color = argsdict['arg_3'], argsdict['arg_2'] + _, color = argsdict['arg_3'], argsdict['arg_2'] return TEMPLATE.format(COLOR_DICT[color][0])