66 lines
2.6 KiB
Python
66 lines
2.6 KiB
Python
# Generated by Django 3.0.8 on 2020-11-04 12:25
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from django.db import migrations, models
|
|
import django.db.models.deletion
|
|
|
|
from config import CONFIG
|
|
|
|
|
|
def forwards_func(apps, schema_editor):
|
|
from core.models import EXTRACTORS
|
|
|
|
Snapshot = apps.get_model("core", "Snapshot")
|
|
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
|
|
|
snapshots = Snapshot.objects.all()
|
|
for snapshot in snapshots:
|
|
out_dir = Path(CONFIG['ARCHIVE_DIR']) / snapshot.timestamp
|
|
|
|
try:
|
|
with open(out_dir / "index.json", "r") as f:
|
|
fs_index = json.load(f)
|
|
except Exception as e:
|
|
continue
|
|
|
|
history = fs_index["history"]
|
|
|
|
for extractor in history:
|
|
for result in history[extractor]:
|
|
ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=json.dumps(result["cmd"]), cmd_version=result["cmd_version"],
|
|
start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"])
|
|
|
|
|
|
|
|
def reverse_func(apps, schema_editor):
|
|
ArchiveResult = apps.get_model("core", "ArchiveResult")
|
|
ArchiveResult.objects.all().delete()
|
|
|
|
|
|
class Migration(migrations.Migration):
|
|
|
|
dependencies = [
|
|
('core', '0006_auto_20201012_1520'),
|
|
]
|
|
|
|
operations = [
|
|
migrations.CreateModel(
|
|
name='ArchiveResult',
|
|
fields=[
|
|
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
('cmd', models.CharField(default='', max_length=500)),
|
|
('pwd', models.CharField(default='', max_length=200)),
|
|
('cmd_version', models.CharField(default='', max_length=20)),
|
|
('status', models.CharField(max_length=10)),
|
|
('output', models.CharField(default='', max_length=500)),
|
|
('start_ts', models.DateTimeField()),
|
|
('end_ts', models.DateTimeField()),
|
|
('extractor', models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('wget', 'wget'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('headers', 'headers'), ('archive_org', 'archive_org')], max_length=20)),
|
|
('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Snapshot')),
|
|
],
|
|
),
|
|
migrations.RunPython(forwards_func, reverse_func),
|
|
]
|