From 241a7c6ab2980a37185c0cc1618779df1a0d2ee2 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 13 May 2024 07:50:07 -0700 Subject: [PATCH] add created, modified, updated, created_by and update django admin --- archivebox/abid_utils/models.py | 51 +++++++++++----- archivebox/api/models.py | 12 +++- archivebox/core/admin.py | 105 +++++++++++++++++++++----------- archivebox/core/models.py | 12 ++-- archivebox/index/sql.py | 2 +- 5 files changed, 124 insertions(+), 58 deletions(-) diff --git a/archivebox/abid_utils/models.py b/archivebox/abid_utils/models.py index 917b5283..0645a32f 100644 --- a/archivebox/abid_utils/models.py +++ b/archivebox/abid_utils/models.py @@ -1,14 +1,16 @@ -from typing import Any, Dict, Union, List, Set, cast +from typing import Any, Dict, Union, List, Set, NamedTuple, cast -import ulid -from uuid import UUID +from ulid import ULID +from uuid import uuid4, UUID from typeid import TypeID # type: ignore[import-untyped] from datetime import datetime from functools import partial from charidfield import CharIDField # type: ignore[import-untyped] +from django.conf import settings from django.db import models from django.db.utils import OperationalError +from django.contrib.auth import get_user_model from django_stubs_ext.db.models import TypedModelMeta @@ -37,6 +39,19 @@ ABIDField = partial( unique=True, ) +def get_or_create_system_user_pk(username='system'): + """Get or create a system user with is_superuser=True to be the default owner for new DB rows""" + + User = get_user_model() + + # if only one user exists total, return that user + if User.objects.filter(is_superuser=True).count() == 1: + return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0] + + # otherwise, create a dedicated "system" user + user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''}) + return user.pk + class ABIDModel(models.Model): abid_prefix: str = DEFAULT_ABID_PREFIX # e.g. 'tag_' @@ -45,11 +60,13 @@ class ABIDModel(models.Model): abid_subtype_src = 'None' # e.g. 'self.extractor' abid_rand_src = 'None' # e.g. 'self.uuid' or 'self.id' - # abid = ABIDField(prefix=abid_prefix, db_index=True, unique=True, null=True, blank=True, editable=True) + id = models.UUIDField(primary_key=True, default=uuid4, editable=True) + uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True) + abid = ABIDField(prefix=abid_prefix) - # created = models.DateTimeField(auto_now_add=True, blank=True, null=True, db_index=True) - # modified = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True) - # created_by = models.ForeignKeyField(get_user_model(), blank=True, null=True, db_index=True) + created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk) + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) class Meta(TypedModelMeta): abstract = True @@ -64,15 +81,21 @@ class ABIDModel(models.Model): super().save(*args, **kwargs) - def calculate_abid(self) -> ABID: + @property + def abid_values(self) -> Dict[str, Any]: + return { + 'prefix': self.abid_prefix, + 'ts': eval(self.abid_ts_src), + 'uri': eval(self.abid_uri_src), + 'subtype': eval(self.abid_subtype_src), + 'rand': eval(self.abid_rand_src), + } + + def get_abid(self) -> ABID: """ Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src). """ - prefix = self.abid_prefix - ts = eval(self.abid_ts_src) - uri = eval(self.abid_uri_src) - subtype = eval(self.abid_subtype_src) - rand = eval(self.abid_rand_src) + prefix, ts, uri, subtype, rand = self.abid_values.values() if (not prefix) or prefix == DEFAULT_ABID_PREFIX: suggested_abid = self.__class__.__name__[:3].lower() @@ -112,7 +135,7 @@ class ABIDModel(models.Model): return ABID.parse(self.abid) if getattr(self, 'abid', None) else self.calculate_abid() @property - def ULID(self) -> ulid.ULID: + def ULID(self) -> ULID: """ Get a ulid.ULID representation of the object's ABID. """ diff --git a/archivebox/api/models.py b/archivebox/api/models.py index 87593bea..8d286a8b 100644 --- a/archivebox/api/models.py +++ b/archivebox/api/models.py @@ -21,7 +21,11 @@ def generate_secret_token() -> str: class APIToken(ABIDModel): - abid_prefix = 'apt' + """ + A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox. + """ + # ABID: apt____ + abid_prefix = 'apt_' abid_ts_src = 'self.created' abid_uri_src = 'self.token' abid_subtype_src = 'self.user_id' @@ -31,11 +35,12 @@ class APIToken(ABIDModel): uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True) abid = ABIDField(prefix=abid_prefix) - user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) token = models.CharField(max_length=32, default=generate_secret_token, unique=True) created = models.DateTimeField(auto_now_add=True) expires = models.DateTimeField(null=True, blank=True) + class Meta(TypedModelMeta): verbose_name = "API Key" @@ -86,12 +91,13 @@ class OutboundWebhook(ABIDModel, WebhookBase): Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using: settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook' """ - abid_prefix = 'whk' + abid_prefix = 'whk_' abid_ts_src = 'self.created' abid_uri_src = 'self.endpoint' abid_subtype_src = 'self.ref' abid_rand_src = 'self.id' + id = models.UUIDField(blank=True, null=True, unique=True, editable=True) uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True) abid = ABIDField(prefix=abid_prefix) diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index 4f84ebcf..15822478 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -160,14 +160,41 @@ class SnapshotActionForm(ActionForm): # ) +def get_abid_info(self, obj): + return format_html( + # URL Hash: {}
+ ''' +     ABID:  {}
+     TS:                  {} ({})
+     URI:                 {} ({})
+     SUBTYPE:       {} ({})
+     RAND:              {} ({})

+     ABID AS UUID:  {}    

+ +     .uuid:                 {}    
+     .id:                      {}    
+     .pk:                     {}    

+ ''', + obj.abid, + obj.ABID.ts, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'], + obj.ABID.uri, str(obj.abid_values['uri']), + obj.ABID.subtype, str(obj.abid_values['subtype']), + obj.ABID.rand, str(obj.abid_values['rand'])[-7:], + obj.ABID.uuid, + obj.uuid, + obj.id, + obj.pk, + ) + + @admin.register(Snapshot, site=archivebox_admin) class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin): list_display = ('added', 'title_str', 'files', 'size', 'url_str') sort_fields = ('title_str', 'url_str', 'added', 'files') - readonly_fields = ('info', 'pk', 'uuid', 'abid', 'calculate_abid', 'bookmarked', 'added', 'updated') + readonly_fields = ('admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'identifiers') search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name') - fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields) - list_filter = ('added', 'updated', 'tags', 'archiveresult__status') + fields = ('url', 'timestamp', 'created_by', 'tags', 'title', *readonly_fields) + list_filter = ('added', 'updated', 'tags', 'archiveresult__status', 'created_by') ordering = ['-added'] actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots'] autocomplete_fields = ['tags'] @@ -216,29 +243,30 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin): # obj.pk, # ) - def info(self, obj): + def admin_actions(self, obj): return format_html( + # URL Hash: {}
+ ''' + Summary page ➡️     + Result files 📑     + Admin actions ⚙️ + ''', + obj.timestamp, + obj.timestamp, + obj.pk, + ) + + def status_info(self, obj): + return format_html( + # URL Hash: {}
''' - PK: {}     - ABID: {}     - UUID: {}     - Timestamp: {}     - URL Hash: {}
Archived: {} ({} files {})     Favicon:     - Status code: {}     + Status code: {}    
Server: {}     Content type: {}     Extension: {}     -

- View Snapshot index ➡️     - View actions ⚙️ ''', - obj.pk, - obj.ABID, - obj.uuid, - obj.timestamp, - obj.url_hash, '✅' if obj.is_archived else '❌', obj.num_outputs, self.size(obj), @@ -247,10 +275,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin): obj.headers and obj.headers.get('Server') or '?', obj.headers and obj.headers.get('Content-Type') or '?', obj.extension or '?', - obj.timestamp, - obj.uuid, ) + def identifiers(self, obj): + return get_abid_info(self, obj) + @admin.display( description='Title', ordering='title', @@ -310,7 +339,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin): return format_html( '{}', obj.url, - obj.url, + obj.url[:128], ) def grid_view(self, request, extra_context=None): @@ -413,14 +442,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin): @admin.register(Tag, site=archivebox_admin) class TagAdmin(admin.ModelAdmin): - list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id') - sort_fields = ('id', 'name', 'slug') - readonly_fields = ('id', 'pk', 'abid', 'calculate_abid', 'num_snapshots', 'snapshots') - search_fields = ('id', 'name', 'slug') - fields = (*readonly_fields, 'name', 'slug') + list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'abid') + sort_fields = ('id', 'name', 'slug', 'abid') + readonly_fields = ('created', 'modified', 'identifiers', 'num_snapshots', 'snapshots') + search_fields = ('id', 'abid', 'uuid', 'name', 'slug') + fields = ('name', 'slug', 'created_by', *readonly_fields, ) actions = ['delete_selected'] ordering = ['-id'] + def identifiers(self, obj): + return get_abid_info(self, obj) + def num_snapshots(self, tag): return format_html( '{} total', @@ -444,11 +476,11 @@ class TagAdmin(admin.ModelAdmin): @admin.register(ArchiveResult, site=archivebox_admin) class ArchiveResultAdmin(admin.ModelAdmin): - list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str') + list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str') sort_fields = ('start_ts', 'extractor', 'status') - readonly_fields = ('id', 'ABID', 'snapshot_str', 'tags_str') + readonly_fields = ('snapshot_info', 'tags_str', 'created_by', 'created', 'modified', 'identifiers') search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp') - fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version') + fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd', 'start_ts', 'end_ts', 'cmd_version', *readonly_fields) autocomplete_fields = ['snapshot'] list_filter = ('status', 'extractor', 'start_ts', 'cmd_version') @@ -456,19 +488,22 @@ class ArchiveResultAdmin(admin.ModelAdmin): list_per_page = SNAPSHOTS_PER_PAGE @admin.display( - description='snapshot' + description='Snapshot Info' ) - def snapshot_str(self, result): + def snapshot_info(self, result): return format_html( - '[{}]
' - '{}', - result.snapshot.timestamp, + '[{}]   {}   {}
', result.snapshot.timestamp, + result.snapshot.abid, + result.snapshot.added.strftime('%Y-%m-%d %H:%M'), result.snapshot.url[:128], ) + def identifiers(self, obj): + return get_abid_info(self, obj) + @admin.display( - description='tags' + description='Snapshot Tags' ) def tags_str(self, result): return result.snapshot.tags_str() diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 8fced67d..0761985f 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -53,19 +53,20 @@ class Tag(ABIDModel): Based on django-taggit model """ abid_prefix = 'tag_' - abid_ts_src = 'None' # TODO: add created/modified time + abid_ts_src = 'self.created' # TODO: add created/modified time abid_uri_src = 'self.name' abid_subtype_src = '"03"' abid_rand_src = 'self.id' + # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True) id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID') + uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True) abid = ABIDField(prefix=abid_prefix) - # no uuid on Tags + name = models.CharField(unique=True, blank=False, max_length=100) - - # slug is autoset on save from name, never set it manually slug = models.SlugField(unique=True, blank=True, max_length=100) + # slug is autoset on save from name, never set it manually class Meta(TypedModelMeta): @@ -325,8 +326,9 @@ class ArchiveResult(ABIDModel): abid_rand_src = 'self.uuid' EXTRACTOR_CHOICES = EXTRACTOR_CHOICES + # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID') # legacy pk - uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) # legacy uuid + uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True) abid = ABIDField(prefix=abid_prefix) snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index 3c4c2a96..8a67f109 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -143,7 +143,7 @@ def list_migrations(out_dir: Path=OUTPUT_DIR) -> List[Tuple[bool, str]]: def apply_migrations(out_dir: Path=OUTPUT_DIR) -> List[str]: from django.core.management import call_command null, out = StringIO(), StringIO() - call_command("makemigrations", interactive=False, stdout=null) + # call_command("makemigrations", interactive=False, stdout=null) call_command("migrate", interactive=False, stdout=out) out.seek(0)