2019-04-17 19:49:18 +12:00
|
|
|
__package__ = 'archivebox.core'
|
|
|
|
|
2021-04-10 20:13:43 +12:00
|
|
|
|
2019-04-17 19:49:18 +12:00
|
|
|
import uuid
|
2024-05-12 23:45:34 +12:00
|
|
|
import ulid
|
2021-04-10 20:13:43 +12:00
|
|
|
import json
|
2024-05-12 23:45:34 +12:00
|
|
|
import hashlib
|
|
|
|
from typeid import TypeID
|
2021-04-10 20:13:43 +12:00
|
|
|
|
|
|
|
from pathlib import Path
|
2024-05-12 23:45:34 +12:00
|
|
|
from typing import Optional, List, NamedTuple
|
2024-05-12 17:28:59 +12:00
|
|
|
from importlib import import_module
|
2019-04-17 19:49:18 +12:00
|
|
|
|
2021-03-27 18:50:26 +13:00
|
|
|
from django.db import models
|
2020-06-30 22:41:48 +12:00
|
|
|
from django.utils.functional import cached_property
|
2020-10-13 07:47:03 +13:00
|
|
|
from django.utils.text import slugify
|
2021-02-17 09:49:29 +13:00
|
|
|
from django.core.cache import cache
|
2021-04-10 20:13:43 +12:00
|
|
|
from django.urls import reverse
|
2020-11-24 07:04:38 +13:00
|
|
|
from django.db.models import Case, When, Value, IntegerField
|
2021-04-06 13:14:22 +12:00
|
|
|
from django.contrib.auth.models import User # noqa
|
2020-09-22 04:50:26 +12:00
|
|
|
|
2021-02-17 09:49:29 +13:00
|
|
|
from ..config import ARCHIVE_DIR, ARCHIVE_DIR_NAME
|
|
|
|
from ..system import get_dir_size
|
2024-05-13 00:21:58 +12:00
|
|
|
from ..util import parse_date, base_url, hashurl, domain
|
2019-04-28 09:26:24 +12:00
|
|
|
from ..index.schema import Link
|
2021-04-01 19:22:15 +13:00
|
|
|
from ..index.html import snapshot_icons
|
2024-05-12 17:28:59 +12:00
|
|
|
from ..extractors import get_default_archive_methods, ARCHIVE_METHODS_INDEXING_PRECEDENCE, EXTRACTORS
|
2020-11-05 01:28:02 +13:00
|
|
|
|
2024-05-12 17:28:59 +12:00
|
|
|
EXTRACTOR_CHOICES = [(extractor_name, extractor_name) for extractor_name in EXTRACTORS.keys()]
|
2020-11-24 12:28:43 +13:00
|
|
|
STATUS_CHOICES = [
|
|
|
|
("succeeded", "succeeded"),
|
|
|
|
("failed", "failed"),
|
|
|
|
("skipped", "skipped")
|
|
|
|
]
|
2019-04-23 05:20:19 +12:00
|
|
|
|
2020-12-12 07:45:44 +13:00
|
|
|
try:
|
|
|
|
JSONField = models.JSONField
|
|
|
|
except AttributeError:
|
|
|
|
import jsonfield
|
|
|
|
JSONField = jsonfield.JSONField
|
|
|
|
|
2019-04-17 19:49:18 +12:00
|
|
|
|
2024-05-12 23:45:34 +12:00
|
|
|
class ULIDParts(NamedTuple):
|
|
|
|
timestamp: str
|
|
|
|
url: str
|
|
|
|
subtype: str
|
|
|
|
randomness: str
|
|
|
|
|
|
|
|
|
2020-10-13 07:47:03 +13:00
|
|
|
class Tag(models.Model):
|
|
|
|
"""
|
|
|
|
Based on django-taggit model
|
|
|
|
"""
|
2021-04-10 22:22:07 +12:00
|
|
|
id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
|
2021-04-10 23:22:20 +12:00
|
|
|
|
2021-03-01 19:39:06 +13:00
|
|
|
name = models.CharField(unique=True, blank=False, max_length=100)
|
2021-03-27 20:57:05 +13:00
|
|
|
|
|
|
|
# slug is autoset on save from name, never set it manually
|
2021-03-01 19:39:06 +13:00
|
|
|
slug = models.SlugField(unique=True, blank=True, max_length=100)
|
2020-09-22 04:50:26 +12:00
|
|
|
|
2021-03-27 20:57:05 +13:00
|
|
|
|
2020-09-22 04:50:26 +12:00
|
|
|
class Meta:
|
|
|
|
verbose_name = "Tag"
|
|
|
|
verbose_name_plural = "Tags"
|
|
|
|
|
2020-10-13 07:47:03 +13:00
|
|
|
def __str__(self):
|
|
|
|
return self.name
|
|
|
|
|
|
|
|
def slugify(self, tag, i=None):
|
|
|
|
slug = slugify(tag)
|
|
|
|
if i is not None:
|
|
|
|
slug += "_%d" % i
|
|
|
|
return slug
|
|
|
|
|
|
|
|
def save(self, *args, **kwargs):
|
|
|
|
if self._state.adding and not self.slug:
|
|
|
|
self.slug = self.slugify(self.name)
|
|
|
|
|
2021-03-01 16:54:04 +13:00
|
|
|
# if name is different but slug conficts with another tags slug, append a counter
|
|
|
|
# with transaction.atomic():
|
|
|
|
slugs = set(
|
|
|
|
type(self)
|
|
|
|
._default_manager.filter(slug__startswith=self.slug)
|
|
|
|
.values_list("slug", flat=True)
|
|
|
|
)
|
|
|
|
|
|
|
|
i = None
|
|
|
|
while True:
|
|
|
|
slug = self.slugify(self.name, i)
|
|
|
|
if slug not in slugs:
|
|
|
|
self.slug = slug
|
|
|
|
return super().save(*args, **kwargs)
|
|
|
|
i = 1 if i is None else i+1
|
2020-10-13 07:47:03 +13:00
|
|
|
else:
|
|
|
|
return super().save(*args, **kwargs)
|
|
|
|
|
2020-12-07 06:23:02 +13:00
|
|
|
|
2019-05-01 15:44:51 +12:00
|
|
|
class Snapshot(models.Model):
|
2019-04-17 19:49:18 +12:00
|
|
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
|
|
|
|
2021-04-01 20:31:20 +13:00
|
|
|
url = models.URLField(unique=True, db_index=True)
|
2020-07-14 04:21:52 +12:00
|
|
|
timestamp = models.CharField(max_length=32, unique=True, db_index=True)
|
2019-04-17 19:49:18 +12:00
|
|
|
|
2021-02-18 20:33:08 +13:00
|
|
|
title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
|
2019-04-17 19:49:18 +12:00
|
|
|
|
2020-06-26 15:32:01 +12:00
|
|
|
added = models.DateTimeField(auto_now_add=True, db_index=True)
|
2021-02-17 09:54:27 +13:00
|
|
|
updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
|
2021-02-19 02:04:14 +13:00
|
|
|
tags = models.ManyToManyField(Tag, blank=True)
|
2019-04-17 19:49:18 +12:00
|
|
|
|
2019-04-17 19:50:41 +12:00
|
|
|
keys = ('url', 'timestamp', 'title', 'tags', 'updated')
|
2019-04-17 19:49:18 +12:00
|
|
|
|
2024-05-12 23:45:34 +12:00
|
|
|
@property
|
|
|
|
def ulid_from_timestamp(self):
|
|
|
|
return str(ulid.from_timestamp(self.added))[:10]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_urlhash(self):
|
|
|
|
return str(ulid.from_randomness(self.url_hash))[10:18]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_type(self):
|
|
|
|
return '00'
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_randomness(self):
|
|
|
|
return str(ulid.from_uuid(self.id))[20:]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_tuple(self) -> ULIDParts:
|
|
|
|
return ULIDParts(self.ulid_from_timestamp, self.ulid_from_urlhash, self.ulid_from_type, self.ulid_from_randomness)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid(self):
|
|
|
|
return ulid.parse(''.join(self.ulid_tuple))
|
|
|
|
|
|
|
|
@property
|
|
|
|
def uuid(self):
|
|
|
|
return self.ulid.uuid
|
|
|
|
|
|
|
|
@property
|
|
|
|
def typeid(self):
|
|
|
|
return TypeID.from_uuid(prefix='snapshot', suffix=self.ulid.uuid)
|
|
|
|
|
2019-04-23 05:20:19 +12:00
|
|
|
def __repr__(self) -> str:
|
2020-06-30 21:55:34 +12:00
|
|
|
title = self.title or '-'
|
|
|
|
return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
|
2019-04-23 05:20:19 +12:00
|
|
|
|
|
|
|
def __str__(self) -> str:
|
2020-06-30 21:55:34 +12:00
|
|
|
title = self.title or '-'
|
|
|
|
return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
|
2019-04-23 05:20:19 +12:00
|
|
|
|
2019-04-17 19:49:18 +12:00
|
|
|
@classmethod
|
|
|
|
def from_json(cls, info: dict):
|
2019-04-17 19:50:41 +12:00
|
|
|
info = {k: v for k, v in info.items() if k in cls.keys}
|
2019-04-17 19:49:18 +12:00
|
|
|
return cls(**info)
|
|
|
|
|
|
|
|
def as_json(self, *args) -> dict:
|
2019-04-17 19:50:41 +12:00
|
|
|
args = args or self.keys
|
2019-04-17 19:49:18 +12:00
|
|
|
return {
|
|
|
|
key: getattr(self, key)
|
2020-11-01 00:55:27 +13:00
|
|
|
if key != 'tags' else self.tags_str()
|
2020-11-24 07:04:38 +13:00
|
|
|
for key in args
|
2019-04-17 19:49:18 +12:00
|
|
|
}
|
2019-04-23 05:20:19 +12:00
|
|
|
|
|
|
|
def as_link(self) -> Link:
|
|
|
|
return Link.from_json(self.as_json())
|
|
|
|
|
2020-08-29 04:08:03 +12:00
|
|
|
def as_link_with_details(self) -> Link:
|
|
|
|
from ..index import load_link_details
|
|
|
|
return load_link_details(self.as_link())
|
2020-11-24 07:04:38 +13:00
|
|
|
|
2021-03-27 21:30:15 +13:00
|
|
|
def tags_str(self, nocache=True) -> str:
|
2021-02-17 09:49:29 +13:00
|
|
|
cache_key = f'{self.id}-{(self.updated or self.added).timestamp()}-tags'
|
|
|
|
calc_tags_str = lambda: ','.join(self.tags.order_by('name').values_list('name', flat=True))
|
2021-03-27 21:30:15 +13:00
|
|
|
if nocache:
|
|
|
|
tags_str = calc_tags_str()
|
|
|
|
cache.set(cache_key, tags_str)
|
|
|
|
return tags_str
|
2021-02-17 09:49:29 +13:00
|
|
|
return cache.get_or_set(cache_key, calc_tags_str)
|
2020-08-29 04:08:03 +12:00
|
|
|
|
2021-04-01 19:22:15 +13:00
|
|
|
def icons(self) -> str:
|
|
|
|
return snapshot_icons(self)
|
|
|
|
|
2021-04-10 20:13:43 +12:00
|
|
|
@cached_property
|
|
|
|
def extension(self) -> str:
|
|
|
|
from ..util import extension
|
|
|
|
return extension(self.url)
|
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2019-04-23 13:40:42 +12:00
|
|
|
def bookmarked(self):
|
|
|
|
return parse_date(self.timestamp)
|
|
|
|
|
2021-02-16 14:44:08 +13:00
|
|
|
@cached_property
|
|
|
|
def bookmarked_date(self):
|
|
|
|
# TODO: remove this
|
|
|
|
return self.bookmarked
|
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2019-04-23 05:20:19 +12:00
|
|
|
def is_archived(self):
|
|
|
|
return self.as_link().is_archived
|
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2019-04-23 05:20:19 +12:00
|
|
|
def num_outputs(self):
|
2020-11-28 19:05:53 +13:00
|
|
|
return self.archiveresult_set.filter(status='succeeded').count()
|
2019-04-23 05:20:19 +12:00
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2019-04-23 05:20:19 +12:00
|
|
|
def url_hash(self):
|
2024-05-12 23:45:34 +12:00
|
|
|
# return hashurl(self.url)
|
2024-05-13 00:21:58 +12:00
|
|
|
return hashlib.sha256(self.url.encode('utf-8')).hexdigest()[:16].upper()
|
2019-04-23 05:20:19 +12:00
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2019-04-23 05:20:19 +12:00
|
|
|
def base_url(self):
|
2021-02-16 14:44:08 +13:00
|
|
|
return base_url(self.url)
|
2019-05-01 15:13:04 +12:00
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2019-05-01 15:13:04 +12:00
|
|
|
def link_dir(self):
|
2021-02-16 14:44:08 +13:00
|
|
|
return str(ARCHIVE_DIR / self.timestamp)
|
2020-06-26 15:32:01 +12:00
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2020-06-26 15:32:01 +12:00
|
|
|
def archive_path(self):
|
2021-02-17 00:16:02 +13:00
|
|
|
return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp)
|
2020-06-26 15:32:01 +12:00
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
@cached_property
|
2020-06-26 15:32:01 +12:00
|
|
|
def archive_size(self):
|
2021-02-17 09:49:29 +13:00
|
|
|
cache_key = f'{str(self.id)[:12]}-{(self.updated or self.added).timestamp()}-size'
|
|
|
|
|
|
|
|
def calc_dir_size():
|
|
|
|
try:
|
|
|
|
return get_dir_size(self.link_dir)[0]
|
|
|
|
except Exception:
|
|
|
|
return 0
|
|
|
|
|
|
|
|
return cache.get_or_set(cache_key, calc_dir_size)
|
2020-06-30 22:41:48 +12:00
|
|
|
|
|
|
|
@cached_property
|
2021-04-10 20:13:43 +12:00
|
|
|
def thumbnail_url(self) -> Optional[str]:
|
|
|
|
result = self.archiveresult_set.filter(
|
|
|
|
extractor='screenshot',
|
|
|
|
status='succeeded'
|
|
|
|
).only('output').last()
|
|
|
|
if result:
|
|
|
|
return reverse('Snapshot', args=[f'{str(self.timestamp)}/{result.output}'])
|
|
|
|
return None
|
|
|
|
|
|
|
|
@cached_property
|
|
|
|
def headers(self) -> Optional[dict]:
|
|
|
|
try:
|
2021-04-10 23:22:32 +12:00
|
|
|
return json.loads((Path(self.link_dir) / 'headers.json').read_text(encoding='utf-8').strip())
|
2021-04-10 20:13:43 +12:00
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
return None
|
|
|
|
|
|
|
|
@cached_property
|
|
|
|
def status_code(self) -> Optional[str]:
|
|
|
|
return self.headers and self.headers.get('Status-Code')
|
|
|
|
|
|
|
|
@cached_property
|
|
|
|
def history(self) -> dict:
|
2020-11-28 19:05:53 +13:00
|
|
|
# TODO: use ArchiveResult for this instead of json
|
2020-11-28 19:14:45 +13:00
|
|
|
return self.as_link_with_details().history
|
2020-06-30 22:41:48 +12:00
|
|
|
|
|
|
|
@cached_property
|
2021-04-10 20:13:43 +12:00
|
|
|
def latest_title(self) -> Optional[str]:
|
2021-02-17 09:49:53 +13:00
|
|
|
if self.title:
|
2021-02-18 20:33:28 +13:00
|
|
|
return self.title # whoopdedoo that was easy
|
2021-02-17 09:49:53 +13:00
|
|
|
|
|
|
|
try:
|
2021-02-18 20:33:28 +13:00
|
|
|
# take longest successful title from ArchiveResult db history
|
|
|
|
return sorted(
|
|
|
|
self.archiveresult_set\
|
|
|
|
.filter(extractor='title', status='succeeded', output__isnull=False)\
|
|
|
|
.values_list('output', flat=True),
|
|
|
|
key=lambda r: len(r),
|
|
|
|
)[-1]
|
|
|
|
except IndexError:
|
2021-02-17 09:49:53 +13:00
|
|
|
pass
|
|
|
|
|
2021-02-18 20:33:28 +13:00
|
|
|
try:
|
|
|
|
# take longest successful title from Link json index file history
|
|
|
|
return sorted(
|
|
|
|
(
|
|
|
|
result.output.strip()
|
|
|
|
for result in self.history['title']
|
|
|
|
if result.status == 'succeeded' and result.output.strip()
|
|
|
|
),
|
|
|
|
key=lambda r: len(r),
|
|
|
|
)[-1]
|
|
|
|
except (KeyError, IndexError):
|
|
|
|
pass
|
2021-02-17 09:49:53 +13:00
|
|
|
|
2020-06-30 22:41:48 +12:00
|
|
|
return None
|
2020-10-13 07:47:03 +13:00
|
|
|
|
2021-04-10 20:13:43 +12:00
|
|
|
def save_tags(self, tags: List[str]=()) -> None:
|
2020-10-13 07:47:03 +13:00
|
|
|
tags_id = []
|
|
|
|
for tag in tags:
|
2021-02-18 20:33:42 +13:00
|
|
|
if tag.strip():
|
|
|
|
tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
|
2020-10-13 07:47:03 +13:00
|
|
|
self.tags.clear()
|
|
|
|
self.tags.add(*tags_id)
|
2020-11-04 03:54:02 +13:00
|
|
|
|
2020-12-07 06:23:02 +13:00
|
|
|
|
2024-05-13 00:21:58 +12:00
|
|
|
def get_storage_dir(self, create=True, symlink=True) -> Path:
|
|
|
|
date_str = self.added.strftime('%Y%m%d')
|
|
|
|
domain_str = domain(self.url)
|
|
|
|
abs_storage_dir = Path(ARCHIVE_DIR) / 'snapshots' / date_str / domain_str / str(self.ulid)
|
|
|
|
|
|
|
|
if create and not abs_storage_dir.is_dir():
|
|
|
|
abs_storage_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
|
if symlink:
|
|
|
|
LINK_PATHS = [
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'all_by_id' / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'snapshots_by_id' / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'snapshots_by_domain' / domain_str / date_str / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'snapshots_by_date' / date_str / domain_str / str(self.ulid),
|
|
|
|
]
|
|
|
|
for link_path in LINK_PATHS:
|
|
|
|
link_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
|
|
link_path.symlink_to(abs_storage_dir)
|
|
|
|
except FileExistsError:
|
|
|
|
link_path.unlink()
|
|
|
|
link_path.symlink_to(abs_storage_dir)
|
|
|
|
|
|
|
|
return abs_storage_dir
|
|
|
|
|
2020-11-24 07:04:38 +13:00
|
|
|
class ArchiveResultManager(models.Manager):
|
|
|
|
def indexable(self, sorted: bool = True):
|
|
|
|
INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
|
|
|
|
qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS,status='succeeded')
|
2020-11-04 03:54:02 +13:00
|
|
|
|
2020-11-24 07:04:38 +13:00
|
|
|
if sorted:
|
|
|
|
precedence = [ When(extractor=method, then=Value(precedence)) for method, precedence in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
|
|
|
|
qs = qs.annotate(indexing_precedence=Case(*precedence, default=Value(1000),output_field=IntegerField())).order_by('indexing_precedence')
|
|
|
|
return qs
|
2020-12-07 06:23:02 +13:00
|
|
|
|
|
|
|
|
2020-11-04 03:54:02 +13:00
|
|
|
class ArchiveResult(models.Model):
|
2024-05-12 17:28:59 +12:00
|
|
|
EXTRACTOR_CHOICES = EXTRACTOR_CHOICES
|
|
|
|
|
2021-04-10 22:22:07 +12:00
|
|
|
id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
|
2024-05-12 23:45:34 +12:00
|
|
|
uuid = models.UUIDField(default=uuid.uuid4, editable=True)
|
2021-02-17 09:54:27 +13:00
|
|
|
|
2020-11-04 03:54:02 +13:00
|
|
|
snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
|
2024-05-12 17:28:59 +12:00
|
|
|
extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32)
|
2020-12-12 07:45:44 +13:00
|
|
|
cmd = JSONField()
|
2020-11-27 18:01:34 +13:00
|
|
|
pwd = models.CharField(max_length=256)
|
2021-02-17 09:54:27 +13:00
|
|
|
cmd_version = models.CharField(max_length=128, default=None, null=True, blank=True)
|
|
|
|
output = models.CharField(max_length=1024)
|
|
|
|
start_ts = models.DateTimeField(db_index=True)
|
2020-11-05 01:28:02 +13:00
|
|
|
end_ts = models.DateTimeField()
|
2020-11-24 12:28:43 +13:00
|
|
|
status = models.CharField(max_length=16, choices=STATUS_CHOICES)
|
2020-11-05 04:31:20 +13:00
|
|
|
|
2020-11-24 07:04:38 +13:00
|
|
|
objects = ArchiveResultManager()
|
|
|
|
|
2020-11-05 04:31:20 +13:00
|
|
|
def __str__(self):
|
|
|
|
return self.extractor
|
2024-05-12 17:28:59 +12:00
|
|
|
|
|
|
|
@cached_property
|
|
|
|
def snapshot_dir(self):
|
|
|
|
return Path(self.snapshot.link_dir)
|
|
|
|
|
2024-05-12 23:45:34 +12:00
|
|
|
@property
|
|
|
|
def ulid_from_timestamp(self):
|
|
|
|
return self.snapshot.ulid_from_timestamp
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_urlhash(self):
|
|
|
|
return self.snapshot.ulid_from_urlhash
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_snapshot(self):
|
|
|
|
return str(self.snapshot.ulid)[:18]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_type(self):
|
|
|
|
return hashlib.sha256(self.extractor.encode('utf-8')).hexdigest()[:2]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_from_randomness(self):
|
|
|
|
return str(ulid.from_uuid(self.uuid))[20:]
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid_tuple(self) -> ULIDParts:
|
|
|
|
return ULIDParts(self.ulid_from_timestamp, self.ulid_from_urlhash, self.ulid_from_type, self.ulid_from_randomness)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def ulid(self):
|
|
|
|
final_ulid = ulid.parse(''.join(self.ulid_tuple))
|
|
|
|
# TODO: migrate self.uuid to match this new uuid
|
|
|
|
# self.uuid = final_ulid.uuid
|
|
|
|
return final_ulid
|
|
|
|
|
|
|
|
@property
|
|
|
|
def typeid(self):
|
|
|
|
return TypeID.from_uuid(prefix='result', suffix=self.ulid.uuid)
|
2024-05-12 17:28:59 +12:00
|
|
|
|
|
|
|
@property
|
|
|
|
def extractor_module(self):
|
|
|
|
return EXTRACTORS[self.extractor]
|
|
|
|
|
|
|
|
def output_path(self) -> str:
|
|
|
|
"""return the canonical output filename or directory name within the snapshot dir"""
|
|
|
|
return self.extractor_module.get_output_path()
|
|
|
|
|
|
|
|
def embed_path(self) -> str:
|
|
|
|
"""
|
|
|
|
return the actual runtime-calculated path to the file on-disk that
|
|
|
|
should be used for user-facing iframe embeds of this result
|
|
|
|
"""
|
|
|
|
|
|
|
|
if hasattr(self.extractor_module, 'get_embed_path'):
|
|
|
|
return self.extractor_module.get_embed_path(self)
|
|
|
|
|
|
|
|
return self.extractor_module.get_output_path()
|
|
|
|
|
|
|
|
def legacy_output_path(self):
|
|
|
|
link = self.snapshot.as_link()
|
|
|
|
return link.canonical_outputs().get(f'{self.extractor}_path')
|
|
|
|
|
|
|
|
def output_exists(self) -> bool:
|
|
|
|
return Path(self.output_path()).exists()
|
2024-05-13 00:21:58 +12:00
|
|
|
|
|
|
|
|
|
|
|
def get_storage_dir(self, create=True, symlink=True):
|
|
|
|
date_str = self.snapshot.added.strftime('%Y%m%d')
|
|
|
|
domain_str = domain(self.snapshot.url)
|
|
|
|
abs_storage_dir = Path(ARCHIVE_DIR) / 'results' / date_str / domain_str / str(self.ulid)
|
|
|
|
|
|
|
|
if create and not abs_storage_dir.is_dir():
|
|
|
|
abs_storage_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
|
if symlink:
|
|
|
|
LINK_PATHS = [
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'all_by_id' / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'results_by_id' / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'results_by_date' / date_str / domain_str / self.extractor / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'results_by_domain' / domain_str / date_str / self.extractor / str(self.ulid),
|
|
|
|
Path(ARCHIVE_DIR).parent / 'index' / 'results_by_type' / self.extractor / date_str / domain_str / str(self.ulid),
|
|
|
|
]
|
|
|
|
for link_path in LINK_PATHS:
|
|
|
|
link_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
|
|
link_path.symlink_to(abs_storage_dir)
|
|
|
|
except FileExistsError:
|
|
|
|
link_path.unlink()
|
|
|
|
link_path.symlink_to(abs_storage_dir)
|
|
|
|
|
|
|
|
return abs_storage_dir
|
|
|
|
|
|
|
|
def symlink_index(self, create=True):
|
|
|
|
abs_result_dir = self.get_storage_dir(create=create)
|