diff --git a/archivebox/config.py b/archivebox/config.py index 62600bf5..da2d1b04 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -281,6 +281,7 @@ TEMPLATES_DIR_NAME = 'templates' ARCHIVE_DIR_NAME = 'archive' SOURCES_DIR_NAME = 'sources' LOGS_DIR_NAME = 'logs' +CACHE_DIR_NAME = 'cache' PERSONAS_DIR_NAME = 'personas' CRONTABS_DIR_NAME = 'crontabs' SQL_INDEX_FILENAME = 'index.sqlite3' @@ -360,6 +361,7 @@ ALLOWED_IN_OUTPUT_DIR = { ARCHIVE_DIR_NAME, SOURCES_DIR_NAME, LOGS_DIR_NAME, + CACHE_DIR_NAME, PERSONAS_DIR_NAME, SQL_INDEX_FILENAME, f'{SQL_INDEX_FILENAME}-wal', @@ -511,6 +513,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = { 'ARCHIVE_DIR': {'default': lambda c: c['OUTPUT_DIR'] / ARCHIVE_DIR_NAME}, 'SOURCES_DIR': {'default': lambda c: c['OUTPUT_DIR'] / SOURCES_DIR_NAME}, 'LOGS_DIR': {'default': lambda c: c['OUTPUT_DIR'] / LOGS_DIR_NAME}, + 'CACHE_DIR': {'default': lambda c: c['OUTPUT_DIR'] / CACHE_DIR_NAME}, 'PERSONAS_DIR': {'default': lambda c: c['OUTPUT_DIR'] / PERSONAS_DIR_NAME}, 'CONFIG_FILE': {'default': lambda c: Path(c['CONFIG_FILE']).resolve() if c['CONFIG_FILE'] else c['OUTPUT_DIR'] / CONFIG_FILENAME}, 'COOKIES_FILE': {'default': lambda c: c['COOKIES_FILE'] and Path(c['COOKIES_FILE']).resolve()}, @@ -1038,6 +1041,11 @@ def get_data_locations(config: ConfigDict) -> ConfigValue: 'enabled': True, 'is_valid': config['LOGS_DIR'].exists(), }, + 'CACHE_DIR': { + 'path': config['CACHE_DIR'].resolve(), + 'enabled': True, + 'is_valid': config['CACHE_DIR'].exists(), + }, 'CUSTOM_TEMPLATES_DIR': { 'path': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).resolve(), 'enabled': bool(config['CUSTOM_TEMPLATES_DIR']), @@ -1388,6 +1396,7 @@ def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CO (Path(output_dir) / SOURCES_DIR_NAME).mkdir(exist_ok=True) (Path(output_dir) / LOGS_DIR_NAME).mkdir(exist_ok=True) + (Path(output_dir) / CACHE_DIR_NAME).mkdir(exist_ok=True) (Path(output_dir) / PERSONAS_DIR_NAME).mkdir(exist_ok=True) (Path(output_dir) / PERSONAS_DIR_NAME / 'Default').mkdir(exist_ok=True) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index d322f711..0c1efbd4 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -20,6 +20,7 @@ from ..config import ( OUTPUT_DIR, ARCHIVE_DIR, LOGS_DIR, + CACHE_DIR, TIMEZONE, LDAP, @@ -227,6 +228,11 @@ TEMPLATES = [ ### External Service Settings ################################################################################ + +CACHE_DB_FILENAME = 'cache.sqlite3' +CACHE_DB_PATH = CACHE_DIR / CACHE_DB_FILENAME +CACHE_DB_TABLE = 'django_cache' + DATABASE_FILE = Path(OUTPUT_DIR) / SQL_INDEX_FILENAME DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(DATABASE_FILE)) @@ -240,18 +246,24 @@ DATABASES = { }, 'TIME_ZONE': TIMEZONE, # DB setup is sometimes modified at runtime by setup_django() in config.py - } + }, + 'cache': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': CACHE_DB_PATH, + 'OPTIONS': { + 'timeout': 60, + 'check_same_thread': False, + }, + 'TIME_ZONE': TIMEZONE, + }, } -CACHE_BACKEND = 'django.core.cache.backends.locmem.LocMemCache' -# CACHE_BACKEND = 'django.core.cache.backends.db.DatabaseCache' -# CACHE_BACKEND = 'django.core.cache.backends.dummy.DummyCache' CACHES = { - 'default': { - 'BACKEND': CACHE_BACKEND, - 'LOCATION': 'django_cache_default', - } + 'default': {'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache'}, + 'dummy': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}, + 'locmem': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}, + # 'filebased': {"BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "LOCATION": CACHE_DIR / 'cache_filebased'}, } EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'