CodeMirror/ArchiveBox
1
0
Fork 0
mirror of synced 2024-06-22 04:10:30 +12:00
Code Issues Projects Releases Wiki Activity

fix readability indexing process and implement a max total character length on indexed content

Browse source
This commit is contained in:
Nick Sweeting 2021-04-06 02:01:38 -04:00
parent b3a89172ab
commit f67a5a215a
2 changed files with 12 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
archivebox/search/backends/sonic.py
View file

@ -5,13 +5,19 @@ from sonic import IngestClient, SearchClient
from archivebox.util import enforce_types
from archivebox.config import SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD, SONIC_BUCKET, SONIC_COLLECTION
MAX_SONIC_TEXT_LENGTH = 2000
MAX_SONIC_TEXT_TOTAL_LENGTH = 100000000 # dont index more than 100 million characters per text
MAX_SONIC_TEXT_CHUNK_LENGTH = 2000 # dont index more than 2000 characters per chunk
@enforce_types
def index(snapshot_id: str, texts: List[str]):
with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
for text in texts:
chunks = [text[i:i+MAX_SONIC_TEXT_LENGTH] for i in range(0, len(text), MAX_SONIC_TEXT_LENGTH)]
max_length = 1000000
chunks = (
text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
for i in range(0, min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH), MAX_SONIC_TEXT_CHUNK_LENGTH)
)
for chunk in chunks:
ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))

8
archivebox/search/utils.py
View file

@ -36,10 +36,10 @@ def get_indexable_content(results: QuerySet):
# TODO: banish this duplication and get these from the extractor file
if method == 'readability':
return get_file_result_content(res, 'content.txt')
return get_file_result_content(res, 'content.txt', use_pwd=True)
elif method == 'singlefile':
return get_file_result_content(res,'',use_pwd=True)
return get_file_result_content(res, '', use_pwd=True)
elif method == 'dom':
return get_file_result_content(res,'',use_pwd=True)
return get_file_result_content(res, '', use_pwd=True)
elif method == 'wget':
return get_file_result_content(res,'',use_pwd=True)
return get_file_result_content(res, '', use_pwd=True)