feat: Update extractors and add command to use sql index as source of truth

archivebox/extractors/__init__.py

@@ -113,8 +113,6 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
             pass
 
         write_link_details(link, out_dir=out_dir, skip_sql_index=skip_index)
-        if not skip_index:
-            patch_main_index(link)
 
         # # If any changes were made, update the main links index json and html
         # was_changed = stats['succeeded'] or stats['failed']

archivebox/index/__init__.py

@@ -236,7 +236,7 @@ def timed_index_update(out_path: str):
 
 
 @enforce_types
-def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False, write_static: bool=False) -> None:
+def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=False) -> None:
     """create index.html file for a given list of links"""
 
     log_indexing_process_started(len(links))
@@ -246,7 +246,7 @@ def write_main_index(links: List[Link], out_dir: str=OUTPUT_DIR, finished: bool=
             write_sql_main_index(links, out_dir=out_dir)
             os.chmod(os.path.join(out_dir, SQL_INDEX_FILENAME), int(OUTPUT_PERMISSIONS, base=8)) # set here because we don't write it with atomic writes
 
-        if write_static:
+        if finished:
             with timed_index_update(os.path.join(out_dir, JSON_INDEX_FILENAME)):
                 write_json_main_index(links, out_dir=out_dir)
 

archivebox/main.py

@@ -368,7 +368,7 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None:
         print('        archivebox list --status=invalid')
 
 
-    write_main_index(list(all_links.values()), out_dir=out_dir, write_static=True)
+    write_main_index(list(all_links.values()), out_dir=out_dir)
 
     print('\n{green}------------------------------------------------------------------{reset}'.format(**ANSI))
     if existing_index: