Merge pull request #636 from ArchiveBox/dev
This commit is contained in:
Nick Sweeting
GitHub
commit
ea15a71012
12
.github/CONTRIBUTING.md
vendored
12
.github/CONTRIBUTING.md
vendored
|
|
@ -9,15 +9,15 @@
|
|||
|
||||
**Useful links:**
|
||||
|
||||
- https://github.com/pirate/ArchiveBox/issues
|
||||
- https://github.com/pirate/ArchiveBox/pulls
|
||||
- https://github.com/pirate/ArchiveBox/wiki/Roadmap
|
||||
- https://github.com/pirate/ArchiveBox/wiki/Install#manual-setup
|
||||
- https://github.com/ArchiveBox/ArchiveBox/issues
|
||||
- https://github.com/ArchiveBox/ArchiveBox/pulls
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
|
||||
|
||||
### Development Setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/pirate/ArchiveBox
|
||||
git clone https://github.com/ArchiveBox/ArchiveBox
|
||||
cd ArchiveBox
|
||||
# Ideally do this in a virtualenv
|
||||
pip install -e '.[dev]' # or use: pipenv install --dev
|
||||
|
|
@ -31,6 +31,8 @@ pip install -e '.[dev]' # or use: pipenv install --dev
|
|||
./bin/build.sh
|
||||
```
|
||||
|
||||
For more common tasks see the `Development` section at the bottom of the README.
|
||||
|
||||
### Getting Help
|
||||
|
||||
Open issues on Github or message me https://sweeting.me/#contact.
|
||||
|
|
|
|||
15
.github/ISSUE_TEMPLATE/bug_report.md
vendored
15
.github/ISSUE_TEMPLATE/bug_report.md
vendored
|
|
@ -10,7 +10,8 @@ assignees: ''
|
|||
<!--
|
||||
Please fill out the following information,
|
||||
feel free to delete sections if they're not applicable
|
||||
or if long issue templates annoy you :)
|
||||
or if long issue templates annoy you.
|
||||
(the only required section is the version information)
|
||||
-->
|
||||
|
||||
#### Describe the bug
|
||||
|
|
@ -35,9 +36,11 @@ If applicable, post any relevant screenshots or copy/pasted terminal output from
|
|||
If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
|
||||
-->
|
||||
|
||||
#### Software versions
|
||||
#### ArchiveBox version
|
||||
|
||||
- OS: ([e.g. macOS 10.14] the operating system you're running ArchiveBox on)
|
||||
- ArchiveBox version: (`git rev-parse HEAD | head -c7` [e.g. d798117] commit ID of the version you're running)
|
||||
- Python version: (`python3 --version` [e.g. 3.7.0])
|
||||
- Chrome version: (`chromium-browser --version` [e.g. 73.1.2.3] if relevant to bug)
|
||||
<!-- Run the `archivebox version` command locally then copy paste the result here: -->
|
||||
```logs
|
||||
replace this line with the *full*, unshortened output of running `archivebox version`
|
||||
```
|
||||
<!-- Tickets without full version info will closed until it is provided,
|
||||
we need the full output here to help you solve your issue -->
|
||||
|
|
|
|||
2
.github/ISSUE_TEMPLATE/feature_request.md
vendored
2
.github/ISSUE_TEMPLATE/feature_request.md
vendored
|
|
@ -45,6 +45,6 @@ workarounds, or other software you've considered using to fix the problem.
|
|||
|
||||
---
|
||||
|
||||
- [ ] I'm willing to contribute dev time / money to fix this issue
|
||||
- [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
|
||||
- [ ] I like ArchiveBox so far / would recommend it to a friend
|
||||
- [ ] I've had a lot of difficulty getting ArchiveBox set up
|
||||
|
|
|
|||
32
.github/workflows/codeql-analysis.yml
vendored
Normal file
32
.github/workflows/codeql-analysis.yml
vendored
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ dev ]
|
||||
pull_request:
|
||||
branches: [ dev ]
|
||||
schedule:
|
||||
- cron: '43 1 * * 2'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'python' ]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-extended
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
39
.github/workflows/docker.yml
vendored
39
.github/workflows/docker.yml
vendored
|
|
@ -1,9 +1,7 @@
|
|||
name: Build Docker image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
on: workflow_dispatch
|
||||
release:
|
||||
types:
|
||||
- created
|
||||
|
|
@ -16,12 +14,6 @@ jobs:
|
|||
buildx:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Docker Login
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
|
|
@ -51,6 +43,23 @@ jobs:
|
|||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-
|
||||
|
||||
- name: Docker Login
|
||||
uses: docker/login-action@v1
|
||||
if: github.event_name != 'pull_request'
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Collect Docker tags
|
||||
id: docker_meta
|
||||
uses: crazy-max/ghaction-docker-meta@v1
|
||||
with:
|
||||
images: archivebox/archivebox,nikisweeting/archivebox
|
||||
tag-sha: true
|
||||
tag-semver: |
|
||||
{{version}}
|
||||
{{major}}.{{minor}}
|
||||
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
|
|
@ -59,15 +68,11 @@ jobs:
|
|||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USERNAME }}/archivebox:latest
|
||||
${{ secrets.DOCKER_USERNAME }}/archivebox:${{ github.sha }}
|
||||
archivebox/archivebox:latest
|
||||
archivebox/archivebox:${{ github.sha }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
cache-from: type=local,src=/tmp/.buildx-cache
|
||||
cache-to: type=local,dest=/tmp/.buildx-cache
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7
|
||||
|
||||
platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
|
||||
- name: Image digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
|
|
|||
2
.github/workflows/pip.yml
vendored
2
.github/workflows/pip.yml
vendored
|
|
@ -1,4 +1,4 @@
|
|||
name: Build pip package
|
||||
name: Build Pip package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
|
|
|||
424
README.md
424
README.md
|
|
@ -1,6 +1,6 @@
|
|||
<div align="center">
|
||||
<em><img src="https://i.imgur.com/5B48E3N.png" height="90px"></em>
|
||||
<h1>ArchiveBox<br/><sub>The open-source self-hosted web archive.</sub></h1>
|
||||
<h1>ArchiveBox<br/><sub>Open-source self-hosted web archiving.</sub></h1>
|
||||
|
||||
▶️ <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> |
|
||||
<a href="https://archivebox.zervice.io/">Demo</a> |
|
||||
|
|
@ -17,96 +17,145 @@
|
|||
<!--<a href="http://webchat.freenode.net?channels=ArchiveBox&uio=d4"><img src="https://img.shields.io/badge/Community_chat-IRC-%2328A745.svg"/></a>-->
|
||||
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a>
|
||||
<a href="https://test.pypi.org/project/archivebox/"><img src="https://img.shields.io/badge/Python-%3E%3D3.7-yellow.svg?logo=python&logoColor=yellow"/></a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies"><img src="https://img.shields.io/badge/Chromium-%3E%3D59-orange.svg?logo=Google+Chrome&logoColor=orange"/></a>
|
||||
<a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/badge/Docker-all%20platforms-lightblue.svg?logo=docker&logoColor=lightblue"/></a>
|
||||
<a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/badge/Docker-all%20platforms-lightblue.svg?logo=docker&logoColor=lightblue"/></a><br/>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=active"/></a>
|
||||
<a href="https://lgtm.com/projects/g/ArchiveBox/ArchiveBox/context:python"><img alt="Language grade: Python" src="https://img.shields.io/lgtm/grade/python/g/ArchiveBox/ArchiveBox.svg?logo=lgtm&logoWidth=18"/></a>
|
||||
<a href="https://lgtm.com/projects/g/ArchiveBox/ArchiveBox/context:javascript"><img alt="Language grade: JavaScript" src="https://img.shields.io/lgtm/grade/javascript/g/ArchiveBox/ArchiveBox.svg?logo=lgtm&logoWidth=18"/></a>
|
||||
<a href="https://lgtm.com/projects/g/ArchiveBox/ArchiveBox/alerts/"><img alt="Total alerts" src="https://img.shields.io/lgtm/alerts/g/ArchiveBox/ArchiveBox.svg?logo=lgtm&logoWidth=18"/></a>
|
||||
|
||||
|
||||
<hr/>
|
||||
</div>
|
||||
|
||||
ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
|
||||
ArchiveBox is a powerful self-hosted internet archiving solution written in Python. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on setup and content within.
|
||||
|
||||
Your archive can be managed through the command line with commands like `archivebox add`, through the built-in Web UI `archivebox server`, or via the Python library API (beta). It can ingest bookmarks from a browser or service like Pocket/Pinboard, your entire browsing history, RSS feeds, or URLs one at a time. You can also schedule regular/realtime imports with `archivebox schedule`.
|
||||
**🔢 Run ArchiveBox via [Docker Compose (recommended)](#Quickstart), Docker, Apt, Brew, or Pip ([see below](#Quickstart)).**
|
||||
|
||||
```bash
|
||||
apt/brew/pip3 install archivebox
|
||||
|
||||
archivebox init # run this in an empty folder
|
||||
archivebox add 'https://example.com' # start adding URLs to archive
|
||||
curl https://example.com/rss.xml | archivebox add # or add via stdin
|
||||
archivebox schedule --every=day https://example.com/rss.xml
|
||||
```
|
||||
|
||||
For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).
|
||||
|
||||
```bash
|
||||
archivebox server --createsuperuser 0.0.0.0:8000 # use the interactive web UI
|
||||
archivebox list 'https://example.com' # use the CLI commands (--help for more)
|
||||
ls ./archive/*/index.json # or browse directly via the filesystem
|
||||
```
|
||||
|
||||
You can then manage your snapshots via the [filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout), [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [Web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [SQLite DB](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py) (`./index.sqlite3`), [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha).
|
||||
|
||||
At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
|
||||
|
||||
<div align="center">
|
||||
<br/><br/>
|
||||
<img src="https://i.imgur.com/PAzXZE8.png" height="70px" alt="bookshelf graphic"> <img src="https://i.imgur.com/asPNk8n.png" height="75px" alt="logo" align="top"/> <img src="https://i.imgur.com/PAzXZE8.png" height="70px" alt="bookshelf graphic">
|
||||
<br/><br/>
|
||||
</div>
|
||||
|
||||
#### ⚡️ CLI Usage
|
||||
|
||||
```bash
|
||||
# archivebox [subcommand] [--args]
|
||||
archivebox --version
|
||||
archivebox help
|
||||
```
|
||||
|
||||
- `archivebox init/version/status/config/manage` to administer your collection
|
||||
- `archivebox add/remove/update/list` to manage Snapshots in the archive
|
||||
- `archivebox schedule` to pull in fresh URLs in regularly from [boorkmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats)
|
||||
- `archivebox oneshot` archive single URLs without starting a whole collection
|
||||
- `archivebox shell/manage dbshell` open a REPL to use the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or SQL API
|
||||
|
||||
<div align="center">
|
||||
<br/>
|
||||
<sup><a href="https://archivebox.zervice.io/">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></sup>
|
||||
<br/>
|
||||
<sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
|
||||
<br/><br/>
|
||||
<img src="https://i.imgur.com/njxgSbl.png" width="22%" alt="cli init screenshot" align="top">
|
||||
<img src="https://i.imgur.com/p6wK6KM.png" width="22%" alt="server snapshot admin screenshot" align="top">
|
||||
<img src="https://i.imgur.com/RefWsXB.jpg" width="28.6%" alt="server snapshot details page screenshot" align="top"/>
|
||||
<br/>
|
||||
<br/>
|
||||
<img src="https://i.imgur.com/T2UAGUD.png" width="49%" alt="grass"/><img src="https://i.imgur.com/T2UAGUD.png" width="49%" alt="grass"/>
|
||||
</div>
|
||||
|
||||
The main index is a self-contained `index.sqlite3` file, and each snapshot is stored as a folder `data/archive/<timestamp>/`, with an easy-to-read `index.html` and `index.json` within. For each page, ArchiveBox auto-extracts many types of assets/media and saves them in standard formats, with out-of-the-box support for: several types of HTML snapshots (wget, Chrome headless, singlefile), PDF snapshotting, screenshotting, WARC archiving, git repositories, images, audio, video, subtitles, article text, and more. The snapshots are browseable and managable offline through the filesystem, the built-in webserver, or the Python library API.
|
||||
|
||||
### Quickstart
|
||||
|
||||
It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`).
|
||||
**🖥 Supported OSs:** Linux/BSD, macOS, Windows **🎮 CPU Architectures:** x86, amd64, arm7, arm8 (raspi >=3)
|
||||
**📦 Distributions:** `docker`/`apt`/`brew`/`pip3`/`npm` (in order of completeness)
|
||||
|
||||
```bash
|
||||
pip3 install archivebox
|
||||
archivebox --version
|
||||
# install extras as-needed, or use one of full setup methods below to get everything out-of-the-box
|
||||
|
||||
mkdir ~/archivebox && cd ~/archivebox # this can be anywhere
|
||||
archivebox init
|
||||
|
||||
archivebox add 'https://example.com'
|
||||
archivebox add --depth=1 'https://example.com'
|
||||
archivebox schedule --every=day https://getpocket.com/users/USERNAME/feed/all
|
||||
archivebox oneshot --extract=title,favicon,media https://www.youtube.com/watch?v=dQw4w9WgXcQ
|
||||
archivebox help # to see more options
|
||||
```
|
||||
|
||||
*(click to expand the sections below for full setup instructions)*
|
||||
*(click to expand your preferred **► `distribution`** below for full setup instructions)*
|
||||
|
||||
<details>
|
||||
<summary><b>Get ArchiveBox with <code>docker-compose</code> on any platform (recommended, everything included out-of-the-box)</b></summary>
|
||||
|
||||
First make sure you have Docker installed: https://docs.docker.com/get-docker/
|
||||
<br/><br/>
|
||||
This is the recommended way to run ArchiveBox because it includes *all* the extractors like chrome, wget, youtube-dl, git, etc., as well as full-text search with sonic, and many other great features.
|
||||
<i>First make sure you have Docker installed: https://docs.docker.com/get-docker/</i>
|
||||
|
||||
```bash
|
||||
<pre lang="bash"><code>
|
||||
# create a new empty directory and initalize your collection (can be anywhere)
|
||||
mkdir ~/archivebox && cd ~/archivebox
|
||||
curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml
|
||||
curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml'
|
||||
docker-compose run archivebox init
|
||||
docker-compose run archivebox --version
|
||||
|
||||
# start the webserver and open the UI (optional)
|
||||
docker-compose run archivebox manage createsuperuser
|
||||
docker-compose up -d
|
||||
open http://127.0.0.1:8000
|
||||
open 'http://127.0.0.1:8000'
|
||||
|
||||
# you can also add links and manage your archive via the CLI:
|
||||
docker-compose run archivebox add 'https://example.com'
|
||||
docker-compose run archivebox status
|
||||
docker-compose run archivebox help # to see more options
|
||||
```
|
||||
</code></pre>
|
||||
|
||||
This is the recommended way to run ArchiveBox because it includes <i>all</i> the extractors like:<br/>
|
||||
chrome, wget, youtube-dl, git, etc., full-text search w/ sonic, and many other great features.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Get ArchiveBox with <code>docker</code> on any platform</b></summary>
|
||||
|
||||
First make sure you have Docker installed: https://docs.docker.com/get-docker/<br/>
|
||||
```bash
|
||||
<i>First make sure you have Docker installed: https://docs.docker.com/get-docker/</i>
|
||||
|
||||
<pre lang="bash"><code>
|
||||
# create a new empty directory and initalize your collection (can be anywhere)
|
||||
mkdir ~/archivebox && cd ~/archivebox
|
||||
docker run -v $PWD:/data -it archivebox/archivebox init
|
||||
docker run -v $PWD:/data -it archivebox/archivebox --version
|
||||
|
||||
# start the webserver and open the UI (optional)
|
||||
docker run -v $PWD:/data -it archivebox/archivebox manage createsuperuser
|
||||
docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox server 0.0.0.0:8000
|
||||
docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox server --createsuperuser 0.0.0.0:8000
|
||||
open http://127.0.0.1:8000
|
||||
|
||||
# you can also add links and manage your archive via the CLI:
|
||||
docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
|
||||
docker run -v $PWD:/data -it archivebox/archivebox status
|
||||
docker run -v $PWD:/data -it archivebox/archivebox help # to see more options
|
||||
```
|
||||
</code></pre>
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Get ArchiveBox with <code>apt</code> on Ubuntu >=20.04</b></summary>
|
||||
|
||||
```bash
|
||||
<i>First make sure you're on Ubuntu >= 20.04, or scroll down for older/non-Ubuntu instructions.</i>
|
||||
|
||||
<pre lang="bash"><code>
|
||||
# add the repo to your sources and install the archivebox package using apt
|
||||
sudo apt install software-properties-common
|
||||
sudo add-apt-repository -u ppa:archivebox/archivebox
|
||||
sudo apt install archivebox
|
||||
|
||||
|
|
@ -117,8 +166,7 @@ archivebox init
|
|||
archivebox --version
|
||||
|
||||
# start the webserver and open the web UI (optional)
|
||||
archivebox manage createsuperuser
|
||||
archivebox server 0.0.0.0:8000
|
||||
archivebox server --createsuperuser 0.0.0.0:8000
|
||||
open http://127.0.0.1:8000
|
||||
|
||||
# you can also add URLs and manage the archive via the CLI and filesystem:
|
||||
|
|
@ -127,13 +175,17 @@ archivebox status
|
|||
archivebox list --html --with-headers > index.html
|
||||
archivebox list --json --with-headers > index.json
|
||||
archivebox help # to see more options
|
||||
```
|
||||
</code></pre>
|
||||
|
||||
For other Debian-based systems or older Ubuntu systems you can add these sources to `/etc/apt/sources.list`:
|
||||
```bash
|
||||
|
||||
<pre lang="bash"><code>
|
||||
deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
|
||||
deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
|
||||
```
|
||||
</code></pre>
|
||||
|
||||
Then run `apt update; apt install archivebox; archivebox --version`.
|
||||
|
||||
(you may need to install some other dependencies manually however)
|
||||
|
||||
</details>
|
||||
|
|
@ -141,7 +193,10 @@ deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
|
|||
<details>
|
||||
<summary><b>Get ArchiveBox with <code>brew</code> on macOS >=10.13</b></summary>
|
||||
|
||||
```bash
|
||||
<i>First make sure you have Homebrew installed: https://brew.sh/#install</i>
|
||||
|
||||
<pre lang="bash"><code>
|
||||
# install the archivebox package using homebrew
|
||||
brew install archivebox/archivebox/archivebox
|
||||
|
||||
# create a new empty directory and initalize your collection (can be anywhere)
|
||||
|
|
@ -151,8 +206,7 @@ archivebox init
|
|||
archivebox --version
|
||||
|
||||
# start the webserver and open the web UI (optional)
|
||||
archivebox manage createsuperuser
|
||||
archivebox server 0.0.0.0:8000
|
||||
archivebox server --createsuperuser 0.0.0.0:8000
|
||||
open http://127.0.0.1:8000
|
||||
|
||||
# you can also add URLs and manage the archive via the CLI and filesystem:
|
||||
|
|
@ -161,14 +215,17 @@ archivebox status
|
|||
archivebox list --html --with-headers > index.html
|
||||
archivebox list --json --with-headers > index.json
|
||||
archivebox help # to see more options
|
||||
```
|
||||
</code></pre>
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Get ArchiveBox with <code>pip</code> on any platform</b></summary>
|
||||
|
||||
```bash
|
||||
<i>First make sure you have Python >= 3.7 installed: https://realpython.com/installing-python/</i>
|
||||
|
||||
<pre lang="bash"><code>
|
||||
# install the archivebox package using pip3
|
||||
pip3 install archivebox
|
||||
|
||||
# create a new empty directory and initalize your collection (can be anywhere)
|
||||
|
|
@ -179,8 +236,7 @@ archivebox --version
|
|||
# Install any missing extras like wget/git/chrome/etc. manually as needed
|
||||
|
||||
# start the webserver and open the web UI (optional)
|
||||
archivebox manage createsuperuser
|
||||
archivebox server 0.0.0.0:8000
|
||||
archivebox server --createsuperuser 0.0.0.0:8000
|
||||
open http://127.0.0.1:8000
|
||||
|
||||
# you can also add URLs and manage the archive via the CLI and filesystem:
|
||||
|
|
@ -189,56 +245,58 @@ archivebox status
|
|||
archivebox list --html --with-headers > index.html
|
||||
archivebox list --json --with-headers > index.json
|
||||
archivebox help # to see more options
|
||||
```
|
||||
</code></pre>
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/lUuicew.png" width="400px">
|
||||
<br/>
|
||||
|
||||
<a href="https://archivebox.zervice.io">DEMO: archivebox.zervice.io/</a>
|
||||
For more information, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">full Quickstart guide</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a>, and <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> docs.
|
||||
No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
|
||||
|
||||
<small>
|
||||
|
||||
1. Install ArchiveBox: `apt/brew/pip3 install archivebox`
|
||||
2. Start a collection: `archivebox init`
|
||||
3. Start archiving: `archivebox add 'https://example.com'`
|
||||
|
||||
</small>
|
||||
|
||||
<br/>
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/6AmOGJT.png" width="49%" alt="grass"/><img src="https://i.imgur.com/6AmOGJT.png" width="49%" alt="grass"/>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
|
||||
# Overview
|
||||
|
||||
ArchiveBox is a command line tool, self-hostable web-archiving server, and Python library all-in-one. It can be installed on Docker, macOS, and Linux/BSD, and Windows. You can download and install it as a Debian/Ubuntu package, Homebrew package, Python3 package, or a Docker image. No matter which install method you choose, they all provide the same CLI, Web UI, and on-disk data format.
|
||||
|
||||
To use ArchiveBox you start by creating a folder for your data to live in (it can be anywhere on your system), and running `archivebox init` inside of it. That will create a sqlite3 index and an `ArchiveBox.conf` file. After that, you can continue to add/export/manage/etc using the CLI `archivebox help`, or you can run the Web UI (recommended). If you only want to archive a single site, you can run `archivebox oneshot` to avoid having to create a whole collection.
|
||||
|
||||
The CLI is considered "stable", the ArchiveBox Python API and REST APIs are "beta", and the [desktop app](https://github.com/ArchiveBox/desktop) is "alpha".
|
||||
|
||||
At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer). You can also self-host your archivebox server on a public domain to provide archive.org-style public access to your site snapshots.
|
||||
<br/>
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/3tBL7PU.png" width="22%" alt="CLI Screenshot" align="top">
|
||||
<img src="https://i.imgur.com/viklZNG.png" width="22%" alt="Desktop index screenshot" align="top">
|
||||
<img src="https://i.imgur.com/RefWsXB.jpg" width="22%" alt="Desktop details page Screenshot"/>
|
||||
<img src="https://i.imgur.com/M6HhzVx.png" width="22%" alt="Desktop details page Screenshot"/><br/>
|
||||
<sup><a href="https://archive.sweeting.me/">Demo</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></sup>
|
||||
<br/>
|
||||
<img src="https://i.imgur.com/lUuicew.png" width="22.4%" align="top">
|
||||
<img src="https://i.imgur.com/p6wK6KM.png" width="35.9%" align="top">
|
||||
<img src="https://i.imgur.com/pzq4uXq.png" width="29.7%" align="top">
|
||||
<br/><br/>
|
||||
<sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
|
||||
</div><br/>
|
||||
|
||||
<br/><br/>
|
||||
<a href="https://archivebox.zervice.io">DEMO: <code>https://archivebox.zervice.io</code></a><br/>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a>
|
||||
<br/>
|
||||
</div>
|
||||
|
||||
## Key Features
|
||||
|
||||
- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally
|
||||
- [**Few dependencies**](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies) and [simple command line interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage)
|
||||
- [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies)
|
||||
- [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
|
||||
- Easy to set up **[scheduled importing](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from multiple sources**
|
||||
- Uses common, **durable, [long-term formats](#saves-lots-of-useful-stuff-for-each-imported-link)** like HTML, JSON, PDF, PNG, and WARC
|
||||
- ~~**Suitable for paywalled / [authenticated content](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir)** (can use your cookies)~~ (do not do this until v0.5 is released with some security fixes)
|
||||
- **Doesn't require a constantly-running daemon**, proxy, or native app
|
||||
- Provides a CLI, Python API, self-hosted web UI, and REST API (WIP)
|
||||
- Architected to be able to run [**many varieties of scripts during archiving**](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to extract media, summarize articles, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), expand comment threads, etc.
|
||||
- Can also [**mirror content to 3rd-party archiving services**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) automatically for redundancy
|
||||
- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (youtube-dl), articles (readability), code (git), etc.](#output-formats)
|
||||
- [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
|
||||
- [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC
|
||||
- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
|
||||
- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
|
||||
- Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released)
|
||||
- Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hiding](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expander](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
|
||||
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/OUmgdlH.png" width="96%" alt="lego">
|
||||
</div>
|
||||
|
||||
## Input formats
|
||||
|
||||
|
|
@ -253,9 +311,10 @@ archivebox add --depth=1 'https://example.com/some/downloads.html'
|
|||
archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
|
||||
```
|
||||
|
||||
- <img src="https://nicksweeting.com/images/bookmarks.png" height="22px"/> Browser history or bookmarks exports (Chrome, Firefox, Safari, IE, Opera, and more)
|
||||
- <img src="https://nicksweeting.com/images/rss.svg" height="22px"/> RSS, XML, JSON, CSV, SQL, HTML, Markdown, TXT, or any other text-based format
|
||||
- <img src="https://getpocket.com/favicon.ico" height="22px"/> Pocket, Pinboard, Instapaper, Shaarli, Delicious, Reddit Saved Posts, Wallabag, Unmark.it, OneTab, and more
|
||||
|
||||
- <img src="https://nicksweeting.com/images/rss.svg" height="22px"/> TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file)
|
||||
- <img src="https://nicksweeting.com/images/bookmarks.png" height="22px"/> [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive))
|
||||
- <img src="https://getpocket.com/favicon.ico" height="22px"/> [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
|
||||
|
||||
See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
|
||||
|
||||
|
|
@ -272,34 +331,51 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
|
|||
```
|
||||
|
||||
- **Index:** `index.html` & `index.json` HTML and JSON index files containing metadata and details
|
||||
- **Title:** `title` title of the site
|
||||
- **Favicon:** `favicon.ico` favicon of the site
|
||||
- **Headers:** `headers.json` Any HTTP headers the site returns are saved in a json file
|
||||
- **SingleFile:** `singlefile.html` HTML snapshot rendered with headless Chrome using SingleFile
|
||||
- **WGET Clone:** `example.com/page-name.html` wget clone of the site, with .html appended if not present
|
||||
- **WARC:** `warc/<timestamp>.gz` gzipped WARC of all the resources fetched while archiving
|
||||
- **PDF:** `output.pdf` Printed PDF of site using headless chrome
|
||||
- **Screenshot:** `screenshot.png` 1440x900 screenshot of site using headless chrome
|
||||
- **DOM Dump:** `output.html` DOM Dump of the HTML after rendering using headless chrome
|
||||
- **Readability:** `article.html/json` Article text extraction using Readability
|
||||
- **URL to Archive.org:** `archive.org.txt` A link to the saved site on archive.org
|
||||
- **Title**, **Favicon**, **Headers** Response headers, site favicon, and parsed site title
|
||||
- **Wget Clone:** `example.com/page-name.html` wget clone of the site with `warc/<timestamp>.gz`
|
||||
- Chrome Headless
|
||||
- **SingleFile:** `singlefile.html` HTML snapshot rendered with headless Chrome using SingleFile
|
||||
- **PDF:** `output.pdf` Printed PDF of site using headless chrome
|
||||
- **Screenshot:** `screenshot.png` 1440x900 screenshot of site using headless chrome
|
||||
- **DOM Dump:** `output.html` DOM Dump of the HTML after rendering using headless chrome
|
||||
- **Readability:** `article.html/json` Article text extraction using Readability
|
||||
- **Archive.org Permalink:** `archive.org.txt` A link to the saved site on archive.org
|
||||
- **Audio & Video:** `media/` all audio/video files + playlists, including subtitles & metadata with youtube-dl
|
||||
- **Source Code:** `git/` clone of any repository found on github, bitbucket, or gitlab links
|
||||
- _More coming soon! See the [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap)..._
|
||||
|
||||
It does everything out-of-the-box by default, but you can disable or tweak [individual archive methods](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) via environment variables or config file.
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/ucyimDX.png" width="96%" alt="lego graphic">
|
||||
</div>
|
||||
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<br/>
|
||||
|
||||
## Dependencies
|
||||
|
||||
You don't need to install all the dependencies, ArchiveBox will automatically enable the relevant modules based on whatever you have available, but it's recommended to use the official [Docker image](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) with everything preinstalled.
|
||||
|
||||
If you so choose, you can also install ArchiveBox and its dependencies directly on any Linux or macOS systems using the [automated setup script](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart) or the [system package manager](https://github.com/ArchiveBox/ArchiveBox/wiki/Install).
|
||||
If you so choose, you can also install ArchiveBox and its dependencies directly on any Linux or macOS systems using the [system package manager](https://github.com/ArchiveBox/ArchiveBox/wiki/Install) or by running the [automated setup script](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart).
|
||||
|
||||
ArchiveBox is written in Python 3 so it requires `python3` and `pip3` available on your system. It also uses a set of optional, but highly recommended external dependencies for archiving sites: `wget` (for plain HTML, static files, and WARC saving), `chromium` (for screenshots, PDFs, JS execution, and more), `youtube-dl` (for audio and video), `git` (for cloning git repos), and `nodejs` (for readability and singlefile), and more.
|
||||
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<img src="https://docs.monadical.com/uploads/upload_b6900afc422ae699bfefa2dcda3306f3.png" width="100%" alt="security graphic"/>
|
||||
</div>
|
||||
|
||||
## Caveats
|
||||
|
||||
If you're importing URLs containing secret slugs or pages with private content (e.g Google Docs, CodiMD notepads, etc), you may want to disable some of the extractor modules to avoid leaking private URLs to 3rd party APIs during the archiving process.
|
||||
|
||||
```bash
|
||||
# don't do this:
|
||||
archivebox add 'https://docs.google.com/document/d/12345somelongsecrethere'
|
||||
|
|
@ -312,6 +388,7 @@ archivebox config --set CHROME_BINARY=chromium # optional: switch to chromium t
|
|||
```
|
||||
|
||||
Be aware that malicious archived JS can also read the contents of other pages in your archive due to snapshot CSRF and XSS protections being imperfect. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) page for more details.
|
||||
|
||||
```bash
|
||||
# visiting an archived page with malicious JS:
|
||||
https://127.0.0.1:8000/archive/1602401954/example.com/index.html
|
||||
|
|
@ -323,20 +400,67 @@ https://127.0.0.1:8000/archive/*
|
|||
```
|
||||
|
||||
Support for saving multiple snapshots of each site over time will be [added soon](https://github.com/ArchiveBox/ArchiveBox/issues/179) (along with the ability to view diffs of the changes between runs). For now ArchiveBox is designed to only archive each URL with each extractor type once. A workaround to take multiple snapshots of the same URL is to make them slightly different by adding a hash:
|
||||
|
||||
```bash
|
||||
archivebox add 'https://example.com#2020-10-24'
|
||||
...
|
||||
archivebox add 'https://example.com#2020-10-25'
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<br/>
|
||||
|
||||
## Screenshots
|
||||
|
||||
<div align="center" width="80%">
|
||||
<img src="https://i.imgur.com/PVO88AZ.png" width="80%"/>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/npareKG.png" alt="brew install archivebox"><br/>
|
||||
<img src="https://i.imgur.com/5vSBO2R.png" alt="archivebox version">
|
||||
</td>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/JXXxFzB.png" alt="archivebox init"><br/>
|
||||
</td>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/wNYtV3v.jpg" alt="archivebox add">
|
||||
</td>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/uZcIOn9.png" alt="archivebox data dir">
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/H08eaia.png" alt="archivebox server">
|
||||
</td>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/zM4z1aU.png" alt="archivebox server add">
|
||||
</td>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/p6wK6KM.png" alt="archivebox server list">
|
||||
</td>
|
||||
<td>
|
||||
<img src="https://i.imgur.com/pzq4uXq.png" alt="archivebox server detail">
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<br/>
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/PVO88AZ.png" width="80%"/>
|
||||
<img src="https://i.imgur.com/ZSUm9mr.png" width="100%" alt="paisley graphic">
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
# Background & Motivation
|
||||
|
||||
Vast treasure troves of knowledge are lost every day on the internet to link rot. As a society, we have an imperative to preserve some important parts of that treasure, just like we preserve our books, paintings, and music in physical libraries long after the originals go out of print or fade into obscurity.
|
||||
|
|
@ -376,6 +500,11 @@ Unlike crawler software that starts from a seed URL and works outwards, or publi
|
|||
|
||||
Because ArchiveBox is designed to ingest a firehose of browser history and bookmark feeds to a local disk, it can be much more disk-space intensive than a centralized service like the Internet Archive or Archive.today. However, as storage space gets cheaper and compression improves, you should be able to use it continuously over the years without having to delete anything. In my experience, ArchiveBox uses about 5gb per 1000 articles, but your milage may vary depending on which options you have enabled and what types of sites you're archiving. By default, it archives everything in as many formats as possible, meaning it takes more space than a using a single method, but more content is accurately replayable over extended periods of time. Storage requirements can be reduced by using a compressed/deduplicated filesystem like ZFS/BTRFS, or by setting `SAVE_MEDIA=False` to skip audio & video files.
|
||||
|
||||
<div align="center">
|
||||
<br/>
|
||||
<img src="https://i.imgur.com/q0Oe36M.png" width="100%" alt="dependencies graphic">
|
||||
</div>
|
||||
|
||||
## Learn more
|
||||
|
||||
Whether you want to learn which organizations are the big players in the web archiving space, want to find a specific open-source tool for your web archiving need, or just want to see where archivists hang out online, our Community Wiki page serves as an index of the broader web archiving community. Check it out to learn about some of the coolest web archiving projects and communities on the web!
|
||||
|
|
@ -383,20 +512,26 @@ Whether you want to learn which organizations are the big players in the web arc
|
|||
<img src="https://i.imgur.com/0ZOmOvN.png" width="14%" align="right"/>
|
||||
|
||||
- [Community Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
|
||||
- [The Master Lists](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#The-Master-Lists)
|
||||
- [The Master Lists](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#the-master-lists)
|
||||
_Community-maintained indexes of archiving tools and institutions._
|
||||
- [Web Archiving Software](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)
|
||||
- [Web Archiving Software](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)
|
||||
_Open source tools and projects in the internet archiving space._
|
||||
- [Reading List](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Reading-List)
|
||||
- [Reading List](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#reading-list)
|
||||
_Articles, posts, and blogs relevant to ArchiveBox and web archiving in general._
|
||||
- [Communities](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Communities)
|
||||
- [Communities](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#communities)
|
||||
_A collection of the most active internet archiving communities and initiatives._
|
||||
- Check out the ArchiveBox [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) and [Changelog](https://github.com/ArchiveBox/ArchiveBox/wiki/Changelog)
|
||||
- Learn why archiving the internet is important by reading the "[On the Importance of Web Archiving](https://parameters.ssrc.org/2018/09/on-the-importance-of-web-archiving/)" blog post.
|
||||
- Or reach out to me for questions and comments via [@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) or [@theSquashSH](https://twitter.com/thesquashSH) on Twitter.
|
||||
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/SMkGW0L.png" width="100%" alt="documentation graphic">
|
||||
</div>
|
||||
|
||||
# Documentation
|
||||
|
||||
<img src="https://read-the-docs-guidelines.readthedocs-hosted.com/_images/logo-dark.png" width="13%" align="right"/>
|
||||
|
|
@ -422,8 +557,8 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
|
|||
- [Chromium Install](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install)
|
||||
- [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview)
|
||||
- [Troubleshooting](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
|
||||
- [Python API](https://docs.archivebox.io/en/latest/modules.html)
|
||||
- REST API (coming soon...)
|
||||
- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
|
||||
- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)
|
||||
|
||||
## More Info
|
||||
|
||||
|
|
@ -434,37 +569,58 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
|
|||
- [Background & Motivation](https://github.com/ArchiveBox/ArchiveBox#background--motivation)
|
||||
- [Web Archiving Community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
|
||||
|
||||
<br/>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<img src="https://i.imgur.com/EGWjbD4.png" width="100%" alt="development">
|
||||
</div>
|
||||
|
||||
# ArchiveBox Development
|
||||
|
||||
All contributions to ArchiveBox are welcomed! Check our [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) for things to work on, and please open an issue to discuss your proposed implementation before working on things! Otherwise we may have to close your PR if it doesn't align with our roadmap.
|
||||
|
||||
Low hanging fruit / easy first tickets:<br/>
|
||||
<a href="https://lgtm.com/projects/g/ArchiveBox/ArchiveBox/alerts/"><img alt="Total alerts" src="https://img.shields.io/lgtm/alerts/g/ArchiveBox/ArchiveBox.svg?logo=lgtm&logoWidth=18"/></a>
|
||||
|
||||
### Setup the dev environment
|
||||
|
||||
First, install the system dependencies from the "Bare Metal" section above.
|
||||
Then you can clone the ArchiveBox repo and install
|
||||
```python3
|
||||
git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
|
||||
git checkout master # or the branch you want to test
|
||||
#### 1. Clone the main code repo (making sure to pull the submodules as well)
|
||||
|
||||
```bash
|
||||
git clone --recurse-submodules https://github.com/ArchiveBox/ArchiveBox
|
||||
cd ArchiveBox
|
||||
git checkout dev # or the branch you want to test
|
||||
git submodule update --init --recursive
|
||||
git pull --recurse-submodules
|
||||
```
|
||||
|
||||
#### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
|
||||
|
||||
```bash
|
||||
# Install ArchiveBox + python dependencies
|
||||
python3 -m venv .venv && source .venv/bin/activate && pip install -e .[dev]
|
||||
# or with pipenv: pipenv install --dev && pipenv shell
|
||||
python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]'
|
||||
# or: pipenv install --dev && pipenv shell
|
||||
|
||||
# Install node dependencies
|
||||
npm install
|
||||
|
||||
# Optional: install extractor dependencies manually or with helper script
|
||||
# Check to see if anything is missing
|
||||
archivebox --version
|
||||
# install any missing dependencies manually, or use the helper script:
|
||||
./bin/setup.sh
|
||||
```
|
||||
|
||||
#### 2. Option B: Build the docker container and use that for development instead
|
||||
|
||||
```bash
|
||||
# Optional: develop via docker by mounting the code dir into the container
|
||||
# if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
|
||||
# inside the container will reload and pick up your changes
|
||||
docker build . -t archivebox
|
||||
docker run -it -p 8000:8000 \
|
||||
docker run -it --rm archivebox version
|
||||
docker run -it --rm -p 8000:8000 \
|
||||
-v $PWD/data:/data \
|
||||
-v $PWD/archivebox:/app/archivebox \
|
||||
archivebox server 0.0.0.0:8000 --debug --reload
|
||||
|
|
@ -475,6 +631,21 @@ docker run -it -p 8000:8000 \
|
|||
See the `./bin/` folder and read the source of the bash scripts within.
|
||||
You can also run all these in Docker. For more examples see the Github Actions CI/CD tests that are run: `.github/workflows/*.yaml`.
|
||||
|
||||
#### Run in DEBUG mode
|
||||
|
||||
```bash
|
||||
archivebox config --set DEBUG=True
|
||||
# or
|
||||
archivebox server --debug ...
|
||||
```
|
||||
|
||||
#### Build and run a Github branch
|
||||
|
||||
```bash
|
||||
docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
|
||||
docker run -it -v $PWD:/data archivebox:dev ...
|
||||
```
|
||||
|
||||
#### Run the linters
|
||||
|
||||
```bash
|
||||
|
|
@ -491,17 +662,20 @@ You can also run all these in Docker. For more examples see the Github Actions C
|
|||
|
||||
#### Make migrations or enter a django shell
|
||||
|
||||
Make sure to run this whenever you change things in `models.py`.
|
||||
```bash
|
||||
cd archivebox/
|
||||
./manage.py makemigrations
|
||||
|
||||
cd data/
|
||||
cd path/to/test/data/
|
||||
archivebox shell
|
||||
archivebox manage dbshell
|
||||
```
|
||||
(uses `pytest -s`)
|
||||
|
||||
#### Build the docs, pip package, and docker image
|
||||
|
||||
(Normally CI takes care of this, but these scripts can be run to do it manually)
|
||||
```bash
|
||||
./bin/build.sh
|
||||
|
||||
|
|
@ -515,11 +689,17 @@ archivebox shell
|
|||
|
||||
#### Roll a release
|
||||
|
||||
(Normally CI takes care of this, but these scripts can be run to do it manually)
|
||||
```bash
|
||||
./bin/release.sh
|
||||
```
|
||||
(bumps the version, builds, and pushes a release to PyPI, Docker Hub, and Github Packages)
|
||||
|
||||
# or individually:
|
||||
./bin/release_docs.sh
|
||||
./bin/release_pip.sh
|
||||
./bin/release_deb.sh
|
||||
./bin/release_brew.sh
|
||||
./bin/release_docker.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
|
|||
parser.add_argument(
|
||||
'--depth', # '-d',
|
||||
type=int,
|
||||
choices=[0, 1],
|
||||
default=0,
|
||||
help='Depth to archive to [0] or 1, see "add" command help for more info.',
|
||||
)
|
||||
|
|
|
|||
|
|
@ -43,6 +43,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
|
|||
action='store_true',
|
||||
help='Run archivebox init before starting the server',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--createsuperuser',
|
||||
action='store_true',
|
||||
help='Run archivebox manage createsuperuser before starting the server',
|
||||
)
|
||||
command = parser.parse_args(args or ())
|
||||
reject_stdin(__command__, stdin)
|
||||
|
||||
|
|
@ -51,6 +56,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
|
|||
reload=command.reload,
|
||||
debug=command.debug,
|
||||
init=command.init,
|
||||
createsuperuser=command.createsuperuser,
|
||||
out_dir=pwd or OUTPUT_DIR,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import re
|
|||
import sys
|
||||
import json
|
||||
import getpass
|
||||
import platform
|
||||
import shutil
|
||||
import django
|
||||
|
||||
|
|
@ -51,7 +52,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
|
|||
'SHELL_CONFIG': {
|
||||
'IS_TTY': {'type': bool, 'default': lambda _: sys.stdout.isatty()},
|
||||
'USE_COLOR': {'type': bool, 'default': lambda c: c['IS_TTY']},
|
||||
'SHOW_PROGRESS': {'type': bool, 'default': lambda c: c['IS_TTY']},
|
||||
'SHOW_PROGRESS': {'type': bool, 'default': lambda c: (c['IS_TTY'] and platform.system() != 'Darwin')}, # progress bars are buggy on mac, disable for now
|
||||
'IN_DOCKER': {'type': bool, 'default': False},
|
||||
# TODO: 'SHOW_HINTS': {'type: bool, 'default': True},
|
||||
},
|
||||
|
|
@ -76,7 +77,6 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
|
|||
'PUBLIC_SNAPSHOTS': {'type': bool, 'default': True},
|
||||
'PUBLIC_ADD_VIEW': {'type': bool, 'default': False},
|
||||
'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'},
|
||||
'ACTIVE_THEME': {'type': str, 'default': 'default'},
|
||||
},
|
||||
|
||||
'ARCHIVE_METHOD_TOGGLES': {
|
||||
|
|
@ -116,16 +116,15 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
|
|||
'--write-annotations',
|
||||
'--write-thumbnail',
|
||||
'--no-call-home',
|
||||
'--user-agent',
|
||||
'--all-subs',
|
||||
'--extract-audio',
|
||||
'--keep-video',
|
||||
'--yes-playlist',
|
||||
'--continue',
|
||||
'--ignore-errors',
|
||||
'--geo-bypass',
|
||||
'--audio-format', 'mp3',
|
||||
'--audio-quality', '320K',
|
||||
'--embed-thumbnail',
|
||||
'--add-metadata']},
|
||||
'--add-metadata',
|
||||
'--max-filesize=750m',
|
||||
]},
|
||||
|
||||
|
||||
'WGET_ARGS': {'type': list, 'default': ['--no-verbose',
|
||||
'--adjust-extension',
|
||||
|
|
@ -205,12 +204,11 @@ def get_real_name(key: str) -> str:
|
|||
################################ Constants #####################################
|
||||
|
||||
PACKAGE_DIR_NAME = 'archivebox'
|
||||
TEMPLATES_DIR_NAME = 'themes'
|
||||
TEMPLATES_DIR_NAME = 'templates'
|
||||
|
||||
ARCHIVE_DIR_NAME = 'archive'
|
||||
SOURCES_DIR_NAME = 'sources'
|
||||
LOGS_DIR_NAME = 'logs'
|
||||
STATIC_DIR_NAME = 'static'
|
||||
SQL_INDEX_FILENAME = 'index.sqlite3'
|
||||
JSON_INDEX_FILENAME = 'index.json'
|
||||
HTML_INDEX_FILENAME = 'index.html'
|
||||
|
|
@ -703,7 +701,7 @@ def get_code_locations(config: ConfigDict) -> SimpleConfigValueDict:
|
|||
'TEMPLATES_DIR': {
|
||||
'path': (config['TEMPLATES_DIR']).resolve(),
|
||||
'enabled': True,
|
||||
'is_valid': (config['TEMPLATES_DIR'] / config['ACTIVE_THEME'] / 'static').exists(),
|
||||
'is_valid': (config['TEMPLATES_DIR'] / 'static').exists(),
|
||||
},
|
||||
# 'NODE_MODULES_DIR': {
|
||||
# 'path': ,
|
||||
|
|
@ -775,7 +773,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
|
|||
'version': config['PYTHON_VERSION'],
|
||||
'hash': bin_hash(config['PYTHON_BINARY']),
|
||||
'enabled': True,
|
||||
'is_valid': bool(config['DJANGO_VERSION']),
|
||||
'is_valid': bool(config['PYTHON_VERSION']),
|
||||
},
|
||||
'DJANGO_BINARY': {
|
||||
'path': bin_path(config['DJANGO_BINARY']),
|
||||
|
|
@ -787,7 +785,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
|
|||
'CURL_BINARY': {
|
||||
'path': bin_path(config['CURL_BINARY']),
|
||||
'version': config['CURL_VERSION'],
|
||||
'hash': bin_hash(config['PYTHON_BINARY']),
|
||||
'hash': bin_hash(config['CURL_BINARY']),
|
||||
'enabled': config['USE_CURL'],
|
||||
'is_valid': bool(config['CURL_VERSION']),
|
||||
},
|
||||
|
|
@ -803,7 +801,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
|
|||
'version': config['NODE_VERSION'],
|
||||
'hash': bin_hash(config['NODE_BINARY']),
|
||||
'enabled': config['USE_NODE'],
|
||||
'is_valid': bool(config['SINGLEFILE_VERSION']),
|
||||
'is_valid': bool(config['NODE_VERSION']),
|
||||
},
|
||||
'SINGLEFILE_BINARY': {
|
||||
'path': bin_path(config['SINGLEFILE_BINARY']),
|
||||
|
|
@ -917,7 +915,12 @@ os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8)) # noqa: F821
|
|||
NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))
|
||||
sys.path.append(NODE_BIN_PATH)
|
||||
|
||||
|
||||
# disable stderr "you really shouldnt disable ssl" warnings with library config
|
||||
if not CONFIG['CHECK_SSL_VALIDITY']:
|
||||
import urllib3
|
||||
import requests
|
||||
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
########################### Config Validity Checkers ###########################
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ class ConfigDict(BaseConfig, total=False):
|
|||
PUBLIC_INDEX: bool
|
||||
PUBLIC_SNAPSHOTS: bool
|
||||
FOOTER_INFO: str
|
||||
ACTIVE_THEME: str
|
||||
|
||||
SAVE_TITLE: bool
|
||||
SAVE_FAVICON: bool
|
||||
|
|
|
|||
|
|
@ -11,18 +11,29 @@ from django.shortcuts import render, redirect
|
|||
from django.contrib.auth import get_user_model
|
||||
from django import forms
|
||||
|
||||
from ..util import htmldecode, urldecode, ansi_to_html
|
||||
|
||||
from core.models import Snapshot, Tag
|
||||
from core.forms import AddLinkForm, TagField
|
||||
|
||||
from core.mixins import SearchResultsAdminMixin
|
||||
|
||||
from index.html import snapshot_icons
|
||||
from util import htmldecode, urldecode, ansi_to_html
|
||||
from logging_util import printable_filesize
|
||||
from main import add, remove
|
||||
from config import OUTPUT_DIR
|
||||
from extractors import archive_links
|
||||
|
||||
# Admin URLs
|
||||
# /admin/
|
||||
# /admin/login/
|
||||
# /admin/core/
|
||||
# /admin/core/snapshot/
|
||||
# /admin/core/snapshot/:uuid/
|
||||
# /admin/core/tag/
|
||||
# /admin/core/tag/:uuid/
|
||||
|
||||
|
||||
# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
|
||||
|
||||
def update_snapshots(modeladmin, request, queryset):
|
||||
|
|
@ -88,13 +99,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
|||
list_display = ('added', 'title_str', 'url_str', 'files', 'size')
|
||||
sort_fields = ('title_str', 'url_str', 'added')
|
||||
readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
|
||||
search_fields = ['url', 'timestamp', 'title', 'tags__name']
|
||||
search_fields = ['url__icontains', 'timestamp', 'title', 'tags__name']
|
||||
fields = (*readonly_fields, 'title', 'tags')
|
||||
list_filter = ('added', 'updated', 'tags')
|
||||
ordering = ['-added']
|
||||
actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots]
|
||||
actions_template = 'admin/actions_as_select.html'
|
||||
form = SnapshotAdminForm
|
||||
list_per_page = 40
|
||||
|
||||
def get_urls(self):
|
||||
urls = super().get_urls()
|
||||
|
|
@ -170,7 +182,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
|
|||
saved_list_max_show_all = self.list_max_show_all
|
||||
|
||||
# Monkey patch here plus core_tags.py
|
||||
self.change_list_template = 'admin/grid_change_list.html'
|
||||
self.change_list_template = 'private_index_grid.html'
|
||||
self.list_per_page = 20
|
||||
self.list_max_show_all = self.list_per_page
|
||||
|
||||
|
|
@ -248,7 +260,7 @@ class ArchiveBoxAdmin(admin.AdminSite):
|
|||
else:
|
||||
context["form"] = form
|
||||
|
||||
return render(template_name='add_links.html', request=request, context=context)
|
||||
return render(template_name='add.html', request=request, context=context)
|
||||
|
||||
admin.site = ArchiveBoxAdmin()
|
||||
admin.site.register(get_user_model())
|
||||
|
|
|
|||
|
|
@ -22,10 +22,32 @@ class AddLinkForm(forms.Form):
|
|||
url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
|
||||
depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
|
||||
archive_methods = forms.MultipleChoiceField(
|
||||
label="Archive methods (select at least 1, otherwise all will be used by default)",
|
||||
required=False,
|
||||
widget=forms.SelectMultiple,
|
||||
choices=ARCHIVE_METHODS,
|
||||
)
|
||||
# TODO: hook these up to the view and put them
|
||||
# in a collapsible UI section labeled "Advanced"
|
||||
#
|
||||
# exclude_patterns = forms.CharField(
|
||||
# label="Exclude patterns",
|
||||
# min_length='1',
|
||||
# required=False,
|
||||
# initial=URL_BLACKLIST,
|
||||
# )
|
||||
# timeout = forms.IntegerField(
|
||||
# initial=TIMEOUT,
|
||||
# )
|
||||
# overwrite = forms.BooleanField(
|
||||
# label="Overwrite any existing Snapshots",
|
||||
# initial=False,
|
||||
# )
|
||||
# index_only = forms.BooleanField(
|
||||
# label="Add URLs to index without Snapshotting",
|
||||
# initial=False,
|
||||
# )
|
||||
|
||||
class TagWidgetMixin:
|
||||
def format_value(self, value):
|
||||
if value is not None and not isinstance(value, str):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from ..config import (
|
|||
SECRET_KEY,
|
||||
ALLOWED_HOSTS,
|
||||
PACKAGE_DIR,
|
||||
ACTIVE_THEME,
|
||||
TEMPLATES_DIR_NAME,
|
||||
SQL_INDEX_FILENAME,
|
||||
OUTPUT_DIR,
|
||||
|
|
@ -34,6 +33,8 @@ LOGOUT_REDIRECT_URL = '/'
|
|||
PASSWORD_RESET_URL = '/accounts/password_reset/'
|
||||
APPEND_SLASH = True
|
||||
|
||||
DEBUG = DEBUG or ('--debug' in sys.argv)
|
||||
|
||||
INSTALLED_APPS = [
|
||||
'django.contrib.auth',
|
||||
'django.contrib.contenttypes',
|
||||
|
|
@ -69,13 +70,12 @@ AUTHENTICATION_BACKENDS = [
|
|||
STATIC_URL = '/static/'
|
||||
|
||||
STATICFILES_DIRS = [
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / ACTIVE_THEME / 'static'),
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'default' / 'static'),
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
|
||||
]
|
||||
|
||||
TEMPLATE_DIRS = [
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / ACTIVE_THEME),
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'default'),
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
|
||||
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
|
||||
]
|
||||
|
||||
|
|
@ -101,7 +101,7 @@ TEMPLATES = [
|
|||
################################################################################
|
||||
|
||||
DATABASE_FILE = Path(OUTPUT_DIR) / SQL_INDEX_FILENAME
|
||||
DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", DATABASE_FILE)
|
||||
DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(DATABASE_FILE))
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ register = template.Library()
|
|||
def snapshot_image(snapshot):
|
||||
result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first()
|
||||
if result:
|
||||
return reverse('LinkAssets', args=[f'{str(snapshot.timestamp)}/{result.output}'])
|
||||
return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
|
||||
|
||||
return static('archive.png')
|
||||
|
||||
|
|
|
|||
|
|
@ -5,22 +5,24 @@ from django.views import static
|
|||
from django.conf import settings
|
||||
from django.views.generic.base import RedirectView
|
||||
|
||||
from core.views import MainIndex, LinkDetails, PublicArchiveView, AddView
|
||||
from core.views import HomepageView, SnapshotView, PublicIndexView, AddView
|
||||
|
||||
|
||||
# print('DEBUG', settings.DEBUG)
|
||||
|
||||
urlpatterns = [
|
||||
path('public/', PublicIndexView.as_view(), name='public-index'),
|
||||
|
||||
path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
|
||||
path('favicon.ico', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'favicon.ico'}),
|
||||
|
||||
path('docs/', RedirectView.as_view(url='https://github.com/ArchiveBox/ArchiveBox/wiki'), name='Docs'),
|
||||
|
||||
path('archive/', RedirectView.as_view(url='/')),
|
||||
path('archive/<path:path>', LinkDetails.as_view(), name='LinkAssets'),
|
||||
path('archive/<path:path>', SnapshotView.as_view(), name='Snapshot'),
|
||||
|
||||
path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
|
||||
path('add/', AddView.as_view()),
|
||||
path('add/', AddView.as_view(), name='add'),
|
||||
|
||||
path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
|
||||
path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
|
||||
|
|
@ -31,6 +33,37 @@ urlpatterns = [
|
|||
|
||||
path('index.html', RedirectView.as_view(url='/')),
|
||||
path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),
|
||||
path('', MainIndex.as_view(), name='Home'),
|
||||
path('public/', PublicArchiveView.as_view(), name='public-index'),
|
||||
path('', HomepageView.as_view(), name='Home'),
|
||||
]
|
||||
|
||||
# # Proposed UI URLs spec
|
||||
# path('', HomepageView)
|
||||
# path('/add', AddView)
|
||||
# path('/public', PublicIndexView)
|
||||
# path('/snapshot/:slug', SnapshotView)
|
||||
|
||||
# path('/admin', admin.site.urls)
|
||||
# path('/accounts', django.contrib.auth.urls)
|
||||
|
||||
# # Prposed REST API spec
|
||||
# # :slugs can be uuid, short_uuid, or any of the unique index_fields
|
||||
# path('api/v1/'),
|
||||
# path('api/v1/core/' [GET])
|
||||
# path('api/v1/core/snapshot/', [GET, POST, PUT]),
|
||||
# path('api/v1/core/snapshot/:slug', [GET, PATCH, DELETE]),
|
||||
# path('api/v1/core/archiveresult', [GET, POST, PUT]),
|
||||
# path('api/v1/core/archiveresult/:slug', [GET, PATCH, DELETE]),
|
||||
# path('api/v1/core/tag/', [GET, POST, PUT]),
|
||||
# path('api/v1/core/tag/:slug', [GET, PATCH, DELETE]),
|
||||
|
||||
# path('api/v1/cli/', [GET])
|
||||
# path('api/v1/cli/{add,list,config,...}', [POST]), # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
|
||||
|
||||
# path('api/v1/extractors/', [GET])
|
||||
# path('api/v1/extractors/:extractor/', [GET]),
|
||||
# path('api/v1/extractors/:extractor/:func', [GET, POST]), # pass query as args directly to chosen function
|
||||
|
||||
# future, just an idea:
|
||||
# path('api/v1/scheduler/', [GET])
|
||||
# path('api/v1/scheduler/task/', [GET, POST, PUT]),
|
||||
# path('api/v1/scheduler/task/:slug', [GET, PATCH, DELETE]),
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from django.http import HttpResponse
|
|||
from django.views import View, static
|
||||
from django.views.generic.list import ListView
|
||||
from django.views.generic import FormView
|
||||
from django.db.models import Q
|
||||
from django.contrib.auth.mixins import UserPassesTestMixin
|
||||
|
||||
from core.models import Snapshot
|
||||
|
|
@ -27,20 +28,20 @@ from ..util import base_url, ansi_to_html
|
|||
from ..index.html import snapshot_icons
|
||||
|
||||
|
||||
class MainIndex(View):
|
||||
template = 'main_index.html'
|
||||
|
||||
class HomepageView(View):
|
||||
def get(self, request):
|
||||
if request.user.is_authenticated:
|
||||
return redirect('/admin/core/snapshot/')
|
||||
|
||||
if PUBLIC_INDEX:
|
||||
return redirect('public-index')
|
||||
return redirect('/public')
|
||||
|
||||
return redirect(f'/admin/login/?next={request.path}')
|
||||
|
||||
|
||||
class LinkDetails(View):
|
||||
class SnapshotView(View):
|
||||
# render static html index from filesystem archive/<timestamp>/index.html
|
||||
|
||||
def get(self, request, path):
|
||||
# missing trailing slash -> redirect to index
|
||||
if '/' not in path:
|
||||
|
|
@ -90,8 +91,8 @@ class LinkDetails(View):
|
|||
status=404,
|
||||
)
|
||||
|
||||
class PublicArchiveView(ListView):
|
||||
template = 'snapshot_list.html'
|
||||
class PublicIndexView(ListView):
|
||||
template_name = 'public_index.html'
|
||||
model = Snapshot
|
||||
paginate_by = 100
|
||||
ordering = ['title']
|
||||
|
|
@ -107,7 +108,7 @@ class PublicArchiveView(ListView):
|
|||
qs = super().get_queryset(**kwargs)
|
||||
query = self.request.GET.get('q')
|
||||
if query:
|
||||
qs = qs.filter(title__icontains=query)
|
||||
qs = qs.filter(Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query))
|
||||
for snapshot in qs:
|
||||
snapshot.icons = snapshot_icons(snapshot)
|
||||
return qs
|
||||
|
|
@ -121,7 +122,7 @@ class PublicArchiveView(ListView):
|
|||
|
||||
|
||||
class AddView(UserPassesTestMixin, FormView):
|
||||
template_name = "add_links.html"
|
||||
template_name = "add.html"
|
||||
form_class = AddLinkForm
|
||||
|
||||
def get_initial(self):
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
|
|||
if method_name not in link.history:
|
||||
link.history[method_name] = []
|
||||
|
||||
if should_run(link, out_dir) or overwrite:
|
||||
if should_run(link, out_dir, overwrite):
|
||||
log_archive_method_started(method_name)
|
||||
|
||||
result = method_function(link=link, out_dir=out_dir)
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
if (out_dir / "archive.org.txt").exists():
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'archive.org.txt').exists():
|
||||
# if open(path, 'r').read().strip() != 'None':
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -20,16 +20,16 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_dom(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
if (out_dir / 'output.html').exists():
|
||||
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'output.html').exists():
|
||||
return False
|
||||
|
||||
return SAVE_DOM
|
||||
|
||||
|
||||
@enforce_types
|
||||
def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""print HTML of site to file using chrome --dump-html"""
|
||||
|
|
|
|||
|
|
@ -20,13 +20,13 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_favicon(link: Link, out_dir: Optional[str]=None) -> bool:
|
||||
out_dir = out_dir or link.link_dir
|
||||
if (Path(out_dir) / 'favicon.ico').exists():
|
||||
def should_save_favicon(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'favicon.ico').exists():
|
||||
return False
|
||||
|
||||
return SAVE_FAVICON
|
||||
|
||||
|
||||
@enforce_types
|
||||
def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
|
||||
"""download site favicon from google's favicon api"""
|
||||
|
|
@ -42,14 +42,13 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
|
|||
*([] if CHECK_SSL_VALIDITY else ['--insecure']),
|
||||
'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
|
||||
]
|
||||
status = 'pending'
|
||||
status = 'failed'
|
||||
timer = TimedProgress(timeout, prefix=' ')
|
||||
try:
|
||||
run(cmd, cwd=str(out_dir), timeout=timeout)
|
||||
chmod_file(output, cwd=str(out_dir))
|
||||
status = 'succeeded'
|
||||
except Exception as err:
|
||||
status = 'failed'
|
||||
output = err
|
||||
finally:
|
||||
timer.end()
|
||||
|
|
|
|||
|
|
@ -28,12 +28,12 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_git(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or link.link_dir
|
||||
def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
if (out_dir / "git").exists():
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'git').exists():
|
||||
return False
|
||||
|
||||
is_clonable_url = (
|
||||
|
|
|
|||
|
|
@ -22,11 +22,12 @@ from ..config import (
|
|||
from ..logging_util import TimedProgress
|
||||
|
||||
@enforce_types
|
||||
def should_save_headers(link: Link, out_dir: Optional[str]=None) -> bool:
|
||||
out_dir = out_dir or link.link_dir
|
||||
def should_save_headers(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'headers.json').exists():
|
||||
return False
|
||||
|
||||
output = Path(out_dir or link.link_dir) / 'headers.json'
|
||||
return not output.exists() and SAVE_HEADERS
|
||||
return SAVE_HEADERS
|
||||
|
||||
|
||||
@enforce_types
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_media(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or link.link_dir
|
||||
|
||||
def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
if (out_dir / "media").exists():
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'media').exists():
|
||||
return False
|
||||
|
||||
return SAVE_MEDIA
|
||||
|
|
|
|||
|
|
@ -37,13 +37,15 @@ def ShellError(cmd: List[str], result: CompletedProcess, lines: int=20) -> Archi
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_mercury(link: Link, out_dir: Optional[str]=None) -> bool:
|
||||
out_dir = out_dir or link.link_dir
|
||||
def should_save_mercury(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
output = Path(out_dir or link.link_dir) / 'mercury'
|
||||
return SAVE_MERCURY and MERCURY_VERSION and (not output.exists())
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'mercury').exists():
|
||||
return False
|
||||
|
||||
return SAVE_MERCURY
|
||||
|
||||
|
||||
@enforce_types
|
||||
|
|
|
|||
|
|
@ -19,12 +19,12 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_pdf(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
if (out_dir / "output.pdf").exists():
|
||||
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'output.pdf').exists():
|
||||
return False
|
||||
|
||||
return SAVE_PDF
|
||||
|
|
|
|||
|
|
@ -46,13 +46,15 @@ def get_html(link: Link, path: Path) -> str:
|
|||
return document
|
||||
|
||||
@enforce_types
|
||||
def should_save_readability(link: Link, out_dir: Optional[str]=None) -> bool:
|
||||
out_dir = out_dir or link.link_dir
|
||||
def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
output = Path(out_dir or link.link_dir) / 'readability'
|
||||
return SAVE_READABILITY and READABILITY_VERSION and (not output.exists())
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'readability').exists():
|
||||
return False
|
||||
|
||||
return SAVE_READABILITY
|
||||
|
||||
|
||||
@enforce_types
|
||||
|
|
|
|||
|
|
@ -20,12 +20,12 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_screenshot(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
if (out_dir / "screenshot.png").exists():
|
||||
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'screenshot.png').exists():
|
||||
return False
|
||||
|
||||
return SAVE_SCREENSHOT
|
||||
|
|
|
|||
|
|
@ -23,13 +23,15 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_singlefile(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
if is_static_file(link.url):
|
||||
return False
|
||||
|
||||
output = out_dir / 'singlefile.html'
|
||||
return SAVE_SINGLEFILE and SINGLEFILE_VERSION and (not output.exists())
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if not overwrite and (out_dir / 'singlefile.html').exists():
|
||||
return False
|
||||
|
||||
return SAVE_SINGLEFILE
|
||||
|
||||
|
||||
@enforce_types
|
||||
|
|
@ -37,7 +39,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
|
|||
"""download full site using single-file"""
|
||||
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
output = str(out_dir.absolute() / "singlefile.html")
|
||||
output = "singlefile.html"
|
||||
|
||||
browser_args = chrome_args(TIMEOUT=0)
|
||||
|
||||
|
|
@ -48,7 +50,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
|
|||
'--browser-executable-path={}'.format(CHROME_BINARY),
|
||||
browser_args,
|
||||
link.url,
|
||||
output
|
||||
output,
|
||||
]
|
||||
|
||||
status = 'succeeded'
|
||||
|
|
@ -69,9 +71,9 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
|
|||
)
|
||||
|
||||
# Check for common failure cases
|
||||
if (result.returncode > 0):
|
||||
if (result.returncode > 0) or not (out_dir / output).is_file():
|
||||
raise ArchiveError('SingleFile was not able to archive the page', hints)
|
||||
chmod_file(output)
|
||||
chmod_file(output, cwd=str(out_dir))
|
||||
except (Exception, OSError) as err:
|
||||
status = 'failed'
|
||||
# TODO: Make this prettier. This is necessary to run the command (escape JSON internal quotes).
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from typing import Optional
|
|||
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||
from ..util import (
|
||||
enforce_types,
|
||||
is_static_file,
|
||||
download_url,
|
||||
htmldecode,
|
||||
)
|
||||
|
|
@ -61,12 +60,9 @@ class TitleParser(HTMLParser):
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_title(link: Link, out_dir: Optional[str]=None) -> bool:
|
||||
def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
# if link already has valid title, skip it
|
||||
if link.title and not link.title.lower().startswith('http'):
|
||||
return False
|
||||
|
||||
if is_static_file(link.url):
|
||||
if not overwrite and link.title and not link.title.lower().startswith('http'):
|
||||
return False
|
||||
|
||||
return SAVE_TITLE
|
||||
|
|
@ -113,7 +109,11 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
|
|||
timestamp=link.timestamp)\
|
||||
.update(title=output)
|
||||
else:
|
||||
raise ArchiveError('Unable to detect page title')
|
||||
# if no content was returned, dont save a title (because it might be a temporary error)
|
||||
if not html:
|
||||
raise ArchiveError('Unable to detect page title')
|
||||
# output = html[:128] # use first bit of content as the title
|
||||
output = link.base_url # use the filename as the title (better UX)
|
||||
except Exception as err:
|
||||
status = 'failed'
|
||||
output = err
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@ from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
|||
from ..system import run, chmod_file
|
||||
from ..util import (
|
||||
enforce_types,
|
||||
is_static_file,
|
||||
without_scheme,
|
||||
without_fragment,
|
||||
without_query,
|
||||
path,
|
||||
|
|
@ -36,10 +34,10 @@ from ..logging_util import TimedProgress
|
|||
|
||||
|
||||
@enforce_types
|
||||
def should_save_wget(link: Link, out_dir: Optional[Path]=None) -> bool:
|
||||
def should_save_wget(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
|
||||
output_path = wget_output_path(link)
|
||||
out_dir = out_dir or Path(link.link_dir)
|
||||
if output_path and (out_dir / output_path).exists():
|
||||
if not overwrite and output_path and (out_dir / output_path).exists():
|
||||
return False
|
||||
|
||||
return SAVE_WGET
|
||||
|
|
@ -66,7 +64,7 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
|
|||
*(['--warc-file={}'.format(str(warc_path))] if SAVE_WARC else []),
|
||||
*(['--page-requisites'] if SAVE_WGET_REQUISITES else []),
|
||||
*(['--user-agent={}'.format(WGET_USER_AGENT)] if WGET_USER_AGENT else []),
|
||||
*(['--load-cookies', COOKIES_FILE] if COOKIES_FILE else []),
|
||||
*(['--load-cookies', str(COOKIES_FILE)] if COOKIES_FILE else []),
|
||||
*(['--compression=auto'] if WGET_AUTO_COMPRESSION else []),
|
||||
*([] if SAVE_WARC else ['--timestamping']),
|
||||
*([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),
|
||||
|
|
@ -105,7 +103,12 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
|
|||
if b'ERROR 500: Internal Server Error' in result.stderr:
|
||||
raise ArchiveError('500 Internal Server Error', hints)
|
||||
raise ArchiveError('Wget failed or got an error from the server', hints)
|
||||
chmod_file(output, cwd=str(out_dir))
|
||||
|
||||
if (out_dir / output).exists():
|
||||
chmod_file(output, cwd=str(out_dir))
|
||||
else:
|
||||
print(f' {out_dir}/{output}')
|
||||
raise ArchiveError('Failed to find wget output after running', hints)
|
||||
except Exception as err:
|
||||
status = 'failed'
|
||||
output = err
|
||||
|
|
@ -129,9 +132,7 @@ def wget_output_path(link: Link) -> Optional[str]:
|
|||
|
||||
See docs on wget --adjust-extension (-E)
|
||||
"""
|
||||
if is_static_file(link.url):
|
||||
return without_scheme(without_fragment(link.url))
|
||||
|
||||
|
||||
# Wget downloads can save in a number of different ways depending on the url:
|
||||
# https://example.com
|
||||
# > example.com/index.html
|
||||
|
|
@ -175,14 +176,30 @@ def wget_output_path(link: Link) -> Optional[str]:
|
|||
if html_files:
|
||||
return str(html_files[0].relative_to(link.link_dir))
|
||||
|
||||
# sometimes wget'd URLs have no ext and return non-html
|
||||
# e.g. /some/example/rss/all -> some RSS XML content)
|
||||
# /some/other/url.o4g -> some binary unrecognized ext)
|
||||
# test this with archivebox add --depth=1 https://getpocket.com/users/nikisweeting/feed/all
|
||||
last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
|
||||
for file_present in search_dir.iterdir():
|
||||
if file_present == last_part_of_url:
|
||||
return str((search_dir / file_present).relative_to(link.link_dir))
|
||||
|
||||
# Move up one directory level
|
||||
search_dir = search_dir.parent
|
||||
|
||||
if str(search_dir) == link.link_dir:
|
||||
break
|
||||
|
||||
# check for literally any file present that isnt an empty folder
|
||||
domain_dir = Path(domain(link.url).replace(":", "+"))
|
||||
files_within = list((Path(link.link_dir) / domain_dir).glob('**/*.*'))
|
||||
if files_within:
|
||||
return str((domain_dir / files_within[-1]).relative_to(link.link_dir))
|
||||
|
||||
search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
|
||||
if not search_dir.is_dir():
|
||||
return str(search_dir.relative_to(link.link_dir))
|
||||
# fallback to just the domain dir
|
||||
search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
|
||||
if search_dir.is_dir():
|
||||
return domain(link.url).replace(":", "+")
|
||||
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ __package__ = 'archivebox.index'
|
|||
|
||||
import os
|
||||
import shutil
|
||||
import json as pyjson
|
||||
from pathlib import Path
|
||||
|
||||
from itertools import chain
|
||||
|
|
@ -42,6 +41,7 @@ from .html import (
|
|||
write_html_link_details,
|
||||
)
|
||||
from .json import (
|
||||
pyjson,
|
||||
parse_json_link_details,
|
||||
write_json_link_details,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from datetime import datetime
|
|||
from typing import List, Optional, Iterator, Mapping
|
||||
from pathlib import Path
|
||||
|
||||
from django.utils.html import format_html
|
||||
from django.utils.html import format_html, mark_safe
|
||||
from collections import defaultdict
|
||||
|
||||
from .schema import Link
|
||||
|
|
@ -23,11 +23,12 @@ from ..config import (
|
|||
GIT_SHA,
|
||||
FOOTER_INFO,
|
||||
HTML_INDEX_FILENAME,
|
||||
SAVE_ARCHIVE_DOT_ORG,
|
||||
)
|
||||
|
||||
MAIN_INDEX_TEMPLATE = 'main_index.html'
|
||||
MINIMAL_INDEX_TEMPLATE = 'main_index_minimal.html'
|
||||
LINK_DETAILS_TEMPLATE = 'link_details.html'
|
||||
MAIN_INDEX_TEMPLATE = 'static_index.html'
|
||||
MINIMAL_INDEX_TEMPLATE = 'minimal_index.html'
|
||||
LINK_DETAILS_TEMPLATE = 'snapshot.html'
|
||||
TITLE_LOADING_MSG = 'Not yet archived...'
|
||||
|
||||
|
||||
|
|
@ -103,6 +104,7 @@ def link_details_template(link: Link) -> str:
|
|||
'status': 'archived' if link.is_archived else 'not yet archived',
|
||||
'status_color': 'success' if link.is_archived else 'danger',
|
||||
'oldest_archive_date': ts_to_date(link.oldest_archive_date),
|
||||
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
|
||||
})
|
||||
|
||||
@enforce_types
|
||||
|
|
@ -116,12 +118,14 @@ def render_django_template(template: str, context: Mapping[str, str]) -> str:
|
|||
def snapshot_icons(snapshot) -> str:
|
||||
from core.models import EXTRACTORS
|
||||
|
||||
# start = datetime.now()
|
||||
|
||||
archive_results = snapshot.archiveresult_set.filter(status="succeeded")
|
||||
link = snapshot.as_link()
|
||||
path = link.archive_path
|
||||
canon = link.canonical_outputs()
|
||||
output = ""
|
||||
output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{} </a>'
|
||||
output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> '
|
||||
icons = {
|
||||
"singlefile": "❶",
|
||||
"wget": "🆆",
|
||||
|
|
@ -138,27 +142,45 @@ def snapshot_icons(snapshot) -> str:
|
|||
exclude = ["favicon", "title", "headers", "archive_org"]
|
||||
# Missing specific entry for WARC
|
||||
|
||||
extractor_items = defaultdict(lambda: None)
|
||||
extractor_outputs = defaultdict(lambda: None)
|
||||
for extractor, _ in EXTRACTORS:
|
||||
for result in archive_results:
|
||||
if result.extractor == extractor:
|
||||
extractor_items[extractor] = result
|
||||
if result.extractor == extractor and result:
|
||||
extractor_outputs[extractor] = result
|
||||
|
||||
for extractor, _ in EXTRACTORS:
|
||||
if extractor not in exclude:
|
||||
exists = extractor_items[extractor] is not None
|
||||
output += output_template.format(path, canon[f"{extractor}_path"], str(exists),
|
||||
extractor, icons.get(extractor, "?"))
|
||||
existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
|
||||
# Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
|
||||
# if existing:
|
||||
# existing = (Path(path) / existing)
|
||||
# if existing.is_file():
|
||||
# existing = True
|
||||
# elif existing.is_dir():
|
||||
# existing = any(existing.glob('*.*'))
|
||||
output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
|
||||
extractor, icons.get(extractor, "?"))
|
||||
if extractor == "wget":
|
||||
# warc isn't technically it's own extractor, so we have to add it after wget
|
||||
exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
|
||||
output += output_template.format(exists[0] if exists else '#', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
|
||||
|
||||
# get from db (faster but less thurthful)
|
||||
exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
|
||||
# get from filesystem (slower but more accurate)
|
||||
# exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
|
||||
output += format_html(output_template, 'warc/', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
|
||||
|
||||
if extractor == "archive_org":
|
||||
# The check for archive_org is different, so it has to be handled separately
|
||||
target_path = Path(path) / "archive.org.txt"
|
||||
exists = target_path.exists()
|
||||
|
||||
# get from db (faster)
|
||||
exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
|
||||
# get from filesystem (slower)
|
||||
# target_path = Path(path) / "archive.org.txt"
|
||||
# exists = target_path.exists()
|
||||
output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
|
||||
"archive_org", icons.get("archive_org", "?"))
|
||||
|
||||
return format_html(f'<span class="files-icons" style="font-size: 1.1em; opacity: 0.8">{output}<span>')
|
||||
result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
|
||||
# end = datetime.now()
|
||||
# print(((end - start).total_seconds()*1000) // 1, 'ms')
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -412,12 +412,14 @@ class Link:
|
|||
"""predict the expected output paths that should be present after archiving"""
|
||||
|
||||
from ..extractors.wget import wget_output_path
|
||||
# TODO: banish this awful duplication from the codebase and import these
|
||||
# from their respective extractor files
|
||||
canonical = {
|
||||
'index_path': 'index.html',
|
||||
'favicon_path': 'favicon.ico',
|
||||
'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
|
||||
'wget_path': wget_output_path(self),
|
||||
'warc_path': 'warc',
|
||||
'warc_path': 'warc/',
|
||||
'singlefile_path': 'singlefile.html',
|
||||
'readability_path': 'readability/content.html',
|
||||
'mercury_path': 'mercury/content.html',
|
||||
|
|
@ -425,8 +427,9 @@ class Link:
|
|||
'screenshot_path': 'screenshot.png',
|
||||
'dom_path': 'output.html',
|
||||
'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
|
||||
'git_path': 'git',
|
||||
'media_path': 'media',
|
||||
'git_path': 'git/',
|
||||
'media_path': 'media/',
|
||||
'headers_path': 'headers.json',
|
||||
}
|
||||
if self.is_static:
|
||||
# static binary files like PDF and images are handled slightly differently.
|
||||
|
|
|
|||
|
|
@ -79,7 +79,6 @@ from .config import (
|
|||
ARCHIVE_DIR_NAME,
|
||||
SOURCES_DIR_NAME,
|
||||
LOGS_DIR_NAME,
|
||||
STATIC_DIR_NAME,
|
||||
JSON_INDEX_FILENAME,
|
||||
HTML_INDEX_FILENAME,
|
||||
SQL_INDEX_FILENAME,
|
||||
|
|
@ -125,10 +124,10 @@ ALLOWED_IN_OUTPUT_DIR = {
|
|||
'.virtualenv',
|
||||
'node_modules',
|
||||
'package-lock.json',
|
||||
'static',
|
||||
ARCHIVE_DIR_NAME,
|
||||
SOURCES_DIR_NAME,
|
||||
LOGS_DIR_NAME,
|
||||
STATIC_DIR_NAME,
|
||||
SQL_INDEX_FILENAME,
|
||||
JSON_INDEX_FILENAME,
|
||||
HTML_INDEX_FILENAME,
|
||||
|
|
@ -1060,6 +1059,7 @@ def server(runserver_args: Optional[List[str]]=None,
|
|||
reload: bool=False,
|
||||
debug: bool=False,
|
||||
init: bool=False,
|
||||
createsuperuser: bool=False,
|
||||
out_dir: Path=OUTPUT_DIR) -> None:
|
||||
"""Run the ArchiveBox HTTP server"""
|
||||
|
||||
|
|
@ -1068,6 +1068,9 @@ def server(runserver_args: Optional[List[str]]=None,
|
|||
if init:
|
||||
run_subcommand('init', stdin=None, pwd=out_dir)
|
||||
|
||||
if createsuperuser:
|
||||
run_subcommand('manage', subcommand_args=['createsuperuser'], pwd=out_dir)
|
||||
|
||||
# setup config for django runserver
|
||||
from . import config
|
||||
config.SHOW_PROGRESS = False
|
||||
|
|
|
|||
|
|
@ -51,9 +51,9 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
|
|||
# look inside the URL for any sub-urls, e.g. for archive.org links
|
||||
# https://web.archive.org/web/20200531203453/https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
|
||||
# -> https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
|
||||
for url in re.findall(URL_REGEX, line[1:]):
|
||||
for sub_url in re.findall(URL_REGEX, line[1:]):
|
||||
yield Link(
|
||||
url=htmldecode(url),
|
||||
url=htmldecode(sub_url),
|
||||
timestamp=str(datetime.now().timestamp()),
|
||||
title=None,
|
||||
tags=None,
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
|
|||
time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
|
||||
try:
|
||||
tags = str_between(get_row('category'), 'label="', '" />')
|
||||
except:
|
||||
except Exception:
|
||||
tags = None
|
||||
|
||||
yield Link(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from sonic import IngestClient, SearchClient
|
|||
from archivebox.util import enforce_types
|
||||
from archivebox.config import SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD, SONIC_BUCKET, SONIC_COLLECTION
|
||||
|
||||
MAX_SONIC_TEXT_LENGTH = 20000
|
||||
MAX_SONIC_TEXT_LENGTH = 2000
|
||||
|
||||
@enforce_types
|
||||
def index(snapshot_id: str, texts: List[str]):
|
||||
|
|
|
|||
|
|
@ -34,10 +34,11 @@ def get_indexable_content(results: QuerySet):
|
|||
return []
|
||||
# This should come from a plugin interface
|
||||
|
||||
# TODO: banish this duplication and get these from the extractor file
|
||||
if method == 'readability':
|
||||
return get_file_result_content(res, 'content.txt')
|
||||
elif method == 'singlefile':
|
||||
return get_file_result_content(res, '')
|
||||
return get_file_result_content(res,'',use_pwd=True)
|
||||
elif method == 'dom':
|
||||
return get_file_result_content(res,'',use_pwd=True)
|
||||
elif method == 'wget':
|
||||
|
|
|
|||
150
archivebox/templates/admin/private_index.html
Normal file
150
archivebox/templates/admin/private_index.html
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
{% extends "base.html" %}
|
||||
{% load static %}
|
||||
|
||||
{% block body %}
|
||||
<div id="toolbar">
|
||||
<form id="changelist-search" action="{% url 'public-index' %}" method="get">
|
||||
<div>
|
||||
<label for="searchbar"><img src="/static/admin/img/search.svg" alt="Search"></label>
|
||||
<input type="text" size="40" name="q" value="" id="searchbar" autofocus placeholder="Title, URL, tags, timestamp, or content...".>
|
||||
<input type="submit" value="Search" style="height: 36px; padding-top: 6px; margin: 8px"/>
|
||||
<input type="button"
|
||||
value="♺"
|
||||
title="Refresh..."
|
||||
onclick="location.href='{% url 'public-index' %}'"
|
||||
style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
|
||||
</input>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<table id="table-bookmarks">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width: 100px;">Bookmarked</th>
|
||||
<th style="width: 26vw;">Snapshot ({{object_list|length}})</th>
|
||||
<th style="width: 140px">Files</th>
|
||||
<th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for link in object_list %}
|
||||
{% include 'main_index_row.html' with link=link %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<center>
|
||||
<span class="step-links">
|
||||
{% if page_obj.has_previous %}
|
||||
<a href="{% url 'public-index' %}?page=1">« first</a>
|
||||
<a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
|
||||
{% endif %}
|
||||
|
||||
<span class="current">
|
||||
Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
|
||||
</span>
|
||||
|
||||
{% if page_obj.has_next %}
|
||||
<a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
|
||||
<a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last »</a>
|
||||
{% endif %}
|
||||
</span>
|
||||
|
||||
{% if page_obj.has_next %}
|
||||
<a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
|
||||
<a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last »</a>
|
||||
{% endif %}
|
||||
</span>
|
||||
<br>
|
||||
</center>
|
||||
{% endblock %}
|
||||
{% extends "admin/base_site.html" %}
|
||||
{% load i18n admin_urls static admin_list %}
|
||||
{% load core_tags %}
|
||||
|
||||
{% block extrastyle %}
|
||||
{{ block.super }}
|
||||
<link rel="stylesheet" type="text/css" href="{% static "admin/css/changelists.css" %}">
|
||||
{% if cl.formset %}
|
||||
<link rel="stylesheet" type="text/css" href="{% static "admin/css/forms.css" %}">
|
||||
{% endif %}
|
||||
{% if cl.formset or action_form %}
|
||||
<script src="{% url 'admin:jsi18n' %}"></script>
|
||||
{% endif %}
|
||||
{{ media.css }}
|
||||
{% if not actions_on_top and not actions_on_bottom %}
|
||||
<style>
|
||||
#changelist table thead th:first-child {width: inherit}
|
||||
</style>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
{% block extrahead %}
|
||||
{{ block.super }}
|
||||
{{ media.js }}
|
||||
{% endblock %}
|
||||
|
||||
{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %}
|
||||
|
||||
{% if not is_popup %}
|
||||
{% block breadcrumbs %}
|
||||
<div class="breadcrumbs">
|
||||
<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
|
||||
› <a href="{% url 'admin:app_list' app_label=cl.opts.app_label %}">{{ cl.opts.app_config.verbose_name }}</a>
|
||||
› {{ cl.opts.verbose_name_plural|capfirst }}
|
||||
</div>
|
||||
{% endblock %}
|
||||
{% endif %}
|
||||
|
||||
{% block coltype %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div id="content-main">
|
||||
{% block object-tools %}
|
||||
<ul class="object-tools">
|
||||
{% block object-tools-items %}
|
||||
{% change_list_object_tools %}
|
||||
{% endblock %}
|
||||
</ul>
|
||||
{% endblock %}
|
||||
{% if cl.formset and cl.formset.errors %}
|
||||
<p class="errornote">
|
||||
{% if cl.formset.total_error_count == 1 %}{% translate "Please correct the error below." %}{% else %}{% translate "Please correct the errors below." %}{% endif %}
|
||||
</p>
|
||||
{{ cl.formset.non_form_errors }}
|
||||
{% endif %}
|
||||
<div class="module{% if cl.has_filters %} filtered{% endif %}" id="changelist">
|
||||
<div class="changelist-form-container">
|
||||
{% block search %}{% search_form cl %}{% endblock %}
|
||||
{% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %}
|
||||
|
||||
<form id="changelist-form" method="post"{% if cl.formset and cl.formset.is_multipart %} enctype="multipart/form-data"{% endif %} novalidate>{% csrf_token %}
|
||||
{% if cl.formset %}
|
||||
<div>{{ cl.formset.management_form }}</div>
|
||||
{% endif %}
|
||||
|
||||
{% block result_list %}
|
||||
{% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %}
|
||||
{% comment %}
|
||||
Table grid
|
||||
{% result_list cl %}
|
||||
{% endcomment %}
|
||||
{% snapshots_grid cl %}
|
||||
{% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %}
|
||||
{% endblock %}
|
||||
{% block pagination %}{% pagination cl %}{% endblock %}
|
||||
</form>
|
||||
</div>
|
||||
{% block filters %}
|
||||
{% if cl.has_filters %}
|
||||
<div id="changelist-filter">
|
||||
<h2>{% translate 'Filter' %}</h2>
|
||||
{% if cl.has_active_filters %}<h3 id="changelist-filter-clear">
|
||||
<a href="{{ cl.clear_all_filters_qs }}">✖ {% translate "Clear all filters" %}</a>
|
||||
</h3>{% endif %}
|
||||
{% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
|
@ -88,4 +88,4 @@
|
|||
{% endblock %}
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
|
|
@ -159,4 +159,4 @@ footer {
|
|||
{% endfor %}
|
||||
</section>
|
||||
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
{% extends "base.html" %}
|
||||
{% extends "core/base.html" %}
|
||||
|
||||
{% load static %}
|
||||
{% load i18n %}
|
||||
|
|
@ -68,4 +68,6 @@
|
|||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block footer %}{% endblock %}
|
||||
|
||||
{% block sidebar %}{% endblock %}
|
||||
84
archivebox/templates/core/base.html
Normal file
84
archivebox/templates/core/base.html
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
{% load admin_urls %}
|
||||
{% load static %}
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<title>Archived Sites</title>
|
||||
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
|
||||
|
||||
<link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
|
||||
<link rel="stylesheet" type="text/css" href="{% static 'admin.css' %}">
|
||||
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
|
||||
<link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}" />
|
||||
{% block extra_head %}
|
||||
{% endblock %}
|
||||
<script src="{% static 'jquery.min.js' %}"></script>
|
||||
<script src="{% static 'jquery.dataTables.min.js' %}"></script>
|
||||
<script>
|
||||
document.addEventListener('error', function (e) {
|
||||
e.target.style.opacity = 0;
|
||||
}, true)
|
||||
jQuery(document).ready(function () {
|
||||
jQuery('#table-bookmarks').DataTable({
|
||||
searching: false,
|
||||
paging: false,
|
||||
stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
|
||||
dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
|
||||
order: [[0, 'desc']],
|
||||
iDisplayLength: 100,
|
||||
});
|
||||
});
|
||||
</script>
|
||||
<base href="{% url 'Home' %}">
|
||||
</head>
|
||||
<body>
|
||||
<div id="container">
|
||||
<div id="header">
|
||||
<div id="branding">
|
||||
<h1 id="site-name">
|
||||
<a href="{% url 'public-index' %}" class="header-archivebox" title="Last updated: {{updated}}">
|
||||
<img src="{% static 'archive.png' %}" alt="Logo" style="height: 30px"/>
|
||||
ArchiveBox
|
||||
</a>
|
||||
</h1>
|
||||
</div>
|
||||
<div id="user-tools">
|
||||
<a href="/add/">➕ Add</a> /
|
||||
<a href="/">Snapshots</a> /
|
||||
<a href="/admin/">Admin</a> /
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
|
||||
{% if user.is_authenticated %}
|
||||
|
||||
User
|
||||
<strong>{% firstof user.get_short_name user.get_username %}</strong>
|
||||
{% if user.has_usable_password %}
|
||||
<a href="{% url 'admin:password_change' %}">Change password</a> /
|
||||
{% endif %}
|
||||
<a href="{% url 'admin:logout' %}">Log out</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div id="content" class="flex">
|
||||
{% block body %}
|
||||
{% endblock %}
|
||||
</div>
|
||||
{% block footer %}
|
||||
<footer>
|
||||
<br />
|
||||
<center>
|
||||
<small>
|
||||
Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a> version
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/releases" title="Releases">v{{VERSION}}</a>.
|
||||
<br/><br/>
|
||||
{{FOOTER_INFO}}
|
||||
</small>
|
||||
</center>
|
||||
<br />
|
||||
</footer>
|
||||
{% endblock %}
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
37
archivebox/templates/core/index_row.html
Normal file
37
archivebox/templates/core/index_row.html
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
{% load static %}
|
||||
|
||||
<tr>
|
||||
<td title="{{link.timestamp}}"> {% if link.bookmarked_date %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} </td>
|
||||
<td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}">
|
||||
{% if link.is_archived %}
|
||||
<a href="archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
|
||||
{% else %}
|
||||
<a href="archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
|
||||
{% endif %}
|
||||
|
||||
<a href="archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
|
||||
<span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">{{link.title|default:'Loading...'}}</span>
|
||||
{% if link.tags_str %}
|
||||
<span class="tags" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
|
||||
{% if link.tags_str != None %}
|
||||
{{link.tags_str|default:''}}
|
||||
{% else %}
|
||||
{{ link.tags|default:'' }}
|
||||
{% endif %}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<span data-number-for="{{link.url}}" title="Fetching any missing files...">
|
||||
{% if link.icons %}
|
||||
{{link.icons}} <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
|
||||
{% else %}
|
||||
<a href="archive/{{link.timestamp}}/index.html">📄
|
||||
{{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
|
||||
</a>
|
||||
{% endif %}
|
||||
</span>
|
||||
</td>
|
||||
<td style="text-align:left"><a href="{{link.url}}">{{link.url}}</a></td>
|
||||
</tr>
|
||||
|
|
@ -16,9 +16,9 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
{% for link in links %}
|
||||
{% include "main_index_row.html" with link=link %}
|
||||
{% include "index_row.html" with link=link %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
|
|
@ -2,13 +2,21 @@
|
|||
{% load static %}
|
||||
|
||||
{% block body %}
|
||||
<br>
|
||||
<form action="{% url 'public-index' %}" method="get">
|
||||
<input name="q" type="text" placeholder="Search...">
|
||||
<button type="submit">Search</button>
|
||||
<button onclick="location.href='{% url 'public-index' %}'" type="button">
|
||||
Reload Index</button>
|
||||
</form>
|
||||
<div id="toolbar">
|
||||
<form id="changelist-search" action="{% url 'public-index' %}" method="get">
|
||||
<div>
|
||||
<label for="searchbar"><img src="/static/admin/img/search.svg" alt="Search"></label>
|
||||
<input type="text" size="40" name="q" value="" id="searchbar" autofocus placeholder="Title, URL, tags, timestamp, or content...".>
|
||||
<input type="submit" value="Search" style="height: 36px; padding-top: 6px; margin: 8px"/>
|
||||
<input type="button"
|
||||
value="♺"
|
||||
title="Refresh..."
|
||||
onclick="location.href='{% url 'public-index' %}'"
|
||||
style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
|
||||
</input>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<table id="table-bookmarks">
|
||||
<thead>
|
||||
<tr>
|
||||
|
|
@ -20,7 +28,7 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
{% for link in object_list %}
|
||||
{% include 'main_index_row.html' with link=link %}
|
||||
{% include 'index_row.html' with link=link %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
}
|
||||
.nav > div {
|
||||
min-height: 30px;
|
||||
margin: 8px 0px;
|
||||
line-height: 1.3;
|
||||
}
|
||||
.header-top a {
|
||||
text-decoration: none;
|
||||
|
|
@ -68,6 +68,11 @@
|
|||
vertical-align: -2px;
|
||||
margin-right: 4px;
|
||||
}
|
||||
.header-toggle {
|
||||
line-height: 14px;
|
||||
font-size: 70px;
|
||||
vertical-align: -8px;
|
||||
}
|
||||
|
||||
.info-row {
|
||||
margin-top: 2px;
|
||||
|
|
@ -76,24 +81,30 @@
|
|||
.info-row .alert {
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
.card {
|
||||
.header-bottom-frames .card {
|
||||
overflow: hidden;
|
||||
box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
|
||||
margin-top: 10px;
|
||||
border: 1px solid rgba(0,0,0,3);
|
||||
border-radius: 14px;
|
||||
background-color: black;
|
||||
}
|
||||
.card h4 {
|
||||
font-size: 1.4vw;
|
||||
}
|
||||
.card-body {
|
||||
font-size: 1vw;
|
||||
padding-top: 1.2vw;
|
||||
padding-left: 1vw;
|
||||
padding-right: 1vw;
|
||||
padding-bottom: 1vw;
|
||||
font-size: 15px;
|
||||
padding: 13px 10px;
|
||||
padding-bottom: 6px;
|
||||
/* padding-left: 3px; */
|
||||
/* padding-right: 3px; */
|
||||
/* padding-bottom: 3px; */
|
||||
line-height: 1.1;
|
||||
word-wrap: break-word;
|
||||
max-height: 102px;
|
||||
overflow: hidden;
|
||||
background-color: #1a1a1a;
|
||||
color: #d3d3d3;
|
||||
}
|
||||
.card-title {
|
||||
margin-bottom: 4px;
|
||||
|
|
@ -126,7 +137,7 @@
|
|||
border-top: 3px solid #aa1e55;
|
||||
}
|
||||
.card.selected-card {
|
||||
border: 2px solid orange;
|
||||
border: 1px solid orange;
|
||||
box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
|
||||
}
|
||||
.iframe-large {
|
||||
|
|
@ -174,12 +185,13 @@
|
|||
width: 98%;
|
||||
border: 1px solid rgba(0,0,0,0.2);
|
||||
box-shadow: 4px 4px 4px rgba(0,0,0,0.2);
|
||||
margin-top: 5px;
|
||||
margin-top: 0px;
|
||||
}
|
||||
.header-bottom-info {
|
||||
color: #6f6f6f;
|
||||
padding-top: 8px;
|
||||
padding-bottom: 13px;
|
||||
padding-top: 0px;
|
||||
padding-bottom: 0px;
|
||||
margin: 0px -15px;
|
||||
}
|
||||
|
||||
.header-bottom-info > div {
|
||||
|
|
@ -203,12 +215,30 @@
|
|||
margin-top: 5px;
|
||||
}
|
||||
.header-bottom-frames .card-title {
|
||||
padding-bottom: 0px;
|
||||
font-size: 1.2vw;
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
font-size: 18px;
|
||||
margin-bottom: 5px;
|
||||
display: inline-block;
|
||||
color: #d3d3d3;
|
||||
font-weight: 200;
|
||||
vertical-align: 0px;
|
||||
margin-top: -6px;
|
||||
}
|
||||
.header-bottom-frames .card-text {
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
font-size: 0.9em;
|
||||
display: inline-block;
|
||||
position: relative;
|
||||
top: -11px;
|
||||
}
|
||||
.card-text code {
|
||||
padding: .2rem .4rem;
|
||||
font-size: 90%;
|
||||
color: #bd4147;
|
||||
background-color: #101010;
|
||||
border-radius: .25rem;
|
||||
}
|
||||
|
||||
@media(max-width: 1092px) {
|
||||
|
|
@ -247,7 +277,7 @@
|
|||
</a>
|
||||
</div>
|
||||
<div class="col-lg-8">
|
||||
<img src="favicon.ico" alt="Favicon">
|
||||
<img src="favicon.ico" onerror="this.style.opacity=0" alt="Favicon">
|
||||
|
||||
{{title}}
|
||||
|
||||
|
|
@ -316,120 +346,145 @@
|
|||
</div>
|
||||
</div>
|
||||
<div class="row header-bottom-frames">
|
||||
<div class="col-lg-3">
|
||||
<div class="col-lg-2">
|
||||
<div class="card selected-card">
|
||||
<iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{archive_url}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
</a>
|
||||
<a href="{{archive_url}}" target="preview"><h4 class="card-title">Wget > WARC</h4></a>
|
||||
<p class="card-text">archive/{{domain}}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-3">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{singlefile_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
<a href="{{singlefile_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./singlefile.html</code></p>
|
||||
</a>
|
||||
<a href="{{singlefile_path}}" target="preview"><h4 class="card-title">Chrome > SingleFile</h4></a>
|
||||
<p class="card-text">archive/singlefile.html</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-3">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{archive_org_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
</a>
|
||||
<a href="{{archive_org_path}}" target="preview"><h4 class="card-title">Archive.Org</h4></a>
|
||||
<p class="card-text">web.archive.org/web/...</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-3">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{url}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
</a>
|
||||
<a href="{{url}}" target="preview"><h4 class="card-title">Original</h4></a>
|
||||
<p class="card-text">{{domain}}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<br/>
|
||||
<div class="col-lg-3">
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top pdf-frame" src="{{pdf_path}}" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{pdf_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
<a href="{{pdf_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./output.pdf</code></p>
|
||||
</a>
|
||||
<a href="{{pdf_path}}" target="preview" id="pdf-btn"><h4 class="card-title">Chrome > PDF</h4></a>
|
||||
<p class="card-text">archive/output.pdf</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-3">
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<img class="card-img-top screenshot" src="{{screenshot_path}}"></iframe>
|
||||
<img class="card-img-top" src="{{screenshot_path}}" onerror="this.style.opacity=0.2"/>
|
||||
<div class="card-body">
|
||||
<a href="{{screenshot_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
<a href="{{screenshot_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./screenshot.png</code></p>
|
||||
</a>
|
||||
<a href="{{screenshot_path}}" target="preview"><h4 class="card-title">Chrome > Screenshot</h4></a>
|
||||
<p class="card-text">archive/screenshot.png</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-3">
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{archive_url}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./{{domain}}</code></p>
|
||||
</a>
|
||||
<a href="{{archive_url}}" target="preview"><h4 class="card-title">Wget > HTML</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if SAVE_ARCHIVE_DOT_ORG %}
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{archive_org_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>🌐 web.archive.org/web/...</code></p>
|
||||
</a>
|
||||
<a href="{{archive_org_path}}" target="preview"><h4 class="card-title">Archive.Org</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>🌐 {{domain}}</code></p>
|
||||
</a>
|
||||
<a href="{{url}}" target="preview"><h4 class="card-title">Original</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{headers_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./headers.json</code></p>
|
||||
</a>
|
||||
<a href="{{headers_path}}" target="preview"><h4 class="card-title">Headers</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{dom_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
<a href="{{dom_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./output.html</code></p>
|
||||
</a>
|
||||
<a href="{{dom_path}}" target="preview"><h4 class="card-title">Chrome > HTML</h4></a>
|
||||
<p class="card-text">archive/output.html</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-3">
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{readability_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
<a href="{{readability_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./readability/content.html</code></p>
|
||||
</a>
|
||||
<a href="{{readability_path}}" target="preview"><h4 class="card-title">Readability</h4></a>
|
||||
<p class="card-text">archive/readability/...</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<br/>
|
||||
<div class="col-lg-3">
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{mercury_path}}" style="float:right" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<img src="../../static/external.png" class="external"/>
|
||||
<a href="{{mercury_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./mercury/content.html</code></p>
|
||||
</a>
|
||||
<a href="{{mercury_path}}" target="preview"><h4 class="card-title">mercury</h4></a>
|
||||
<p class="card-text">archive/mercury/...</p>
|
||||
<a href="{{mercury_path}}" target="preview"><h4 class="card-title">Mercury</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{media_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./media/*.mp4</code></p>
|
||||
</a>
|
||||
<a href="{{media_path}}" target="preview"><h4 class="card-title">Media</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-2">
|
||||
<div class="card">
|
||||
<iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
|
||||
<div class="card-body">
|
||||
<a href="{{git_path}}" title="Open in new tab..." target="_blank" rel="noopener">
|
||||
<p class="card-text"><code>./git/*.git</code></p>
|
||||
</a>
|
||||
<a href="{{git_path}}" target="preview"><h4 class="card-title">Git</h4></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<iframe sandbox="allow-same-origin allow-scripts allow-forms" class="full-page-iframe" src="{{archive_url}}" name="preview"></iframe>
|
||||
<iframe sandbox="allow-same-origin allow-scripts allow-forms" class="full-page-iframe" src="{{singlefile_path}}" name="preview"></iframe>
|
||||
|
||||
<script
|
||||
src="https://code.jquery.com/jquery-3.2.1.slim.min.js"
|
||||
|
|
@ -234,7 +234,7 @@
|
|||
</thead>
|
||||
<tbody>
|
||||
{% for link in links %}
|
||||
{% include 'main_index_row.html' with link=link %}
|
||||
{% include 'index_row.html' with link=link %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -243,7 +243,7 @@
|
|||
<center>
|
||||
<small>
|
||||
Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
|
||||
version <a href="https://github.com/ArchiveBox/ArchiveBox/tree/v{{VERSION}}" title="Git commit">v{{VERSION}}</a> |
|
||||
version <a href="https://github.com/ArchiveBox/ArchiveBox/tree/v{{version}}" title="Git commit">v{{version}}</a> |
|
||||
Download index as <a href="index.json" title="JSON summary of archived links.">JSON</a>
|
||||
<br/><br/>
|
||||
{{FOOTER_INFO}}
|
||||
|
|
@ -1,3 +1,13 @@
|
|||
header {
|
||||
font-family: "Roboto","Lucida Grande","DejaVu Sans","Bitstream Vera Sans",Verdana,Arial,sans-serif;
|
||||
font-size: 13px;
|
||||
color: white;
|
||||
height: 30px;
|
||||
}
|
||||
.header-top {
|
||||
color: white;
|
||||
}
|
||||
|
||||
.dashboard #content {
|
||||
width: 100%;
|
||||
margin-right: 0px;
|
||||
|
|
@ -60,3 +70,21 @@ ul#id_depth {
|
|||
box-sizing: border-box;
|
||||
animation: spin 2s linear infinite;
|
||||
}
|
||||
|
||||
|
||||
textarea, select {
|
||||
border-radius: 4px;
|
||||
border: 2px solid #004882;
|
||||
box-shadow: 4px 4px 4px rgba(0,0,0,0.02);
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
select option:not(:checked) {
|
||||
border: 1px dashed rgba(10,200,20,0.12);
|
||||
}
|
||||
select option:checked {
|
||||
border: 1px solid green;
|
||||
background-color: green;
|
||||
color: green;
|
||||
}
|
||||
|
||||
|
|
@ -224,7 +224,7 @@ body.model-snapshot.change-list #content .object-tools {
|
|||
100% { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
.tags > a > .tag {
|
||||
.tag {
|
||||
float: right;
|
||||
border-radius: 5px;
|
||||
background-color: #bfdfff;
|
||||
|
|
@ -232,3 +232,8 @@ body.model-snapshot.change-list #content .object-tools {
|
|||
margin-left: 4px;
|
||||
margin-top: 1px;
|
||||
}
|
||||
|
||||
.exists-False {
|
||||
opacity: 0.1;
|
||||
filter: grayscale(100%);
|
||||
}
|
||||
|
Before Width: | Height: | Size: 17 KiB After Width: | Height: | Size: 17 KiB |
|
Before Width: | Height: | Size: 1.6 KiB After Width: | Height: | Size: 1.6 KiB |
|
Before Width: | Height: | Size: 158 B After Width: | Height: | Size: 158 B |
|
Before Width: | Height: | Size: 201 B After Width: | Height: | Size: 201 B |
|
Before Width: | Height: | Size: 157 B After Width: | Height: | Size: 157 B |
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
|
|
@ -1,284 +0,0 @@
|
|||
{% load static %}
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<title>Archived Sites</title>
|
||||
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
|
||||
|
||||
<style>
|
||||
:root {
|
||||
--bg-main: #efefef;
|
||||
--accent-1: #aa1e55;
|
||||
--accent-2: #ffebeb;
|
||||
--accent-3: #efefef;
|
||||
|
||||
--text-1: #1c1c1c;
|
||||
--text-2: #eaeaea;
|
||||
--text-main: #1a1a1a;
|
||||
--font-main: "Gill Sans", Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
/* Dark Mode (WIP) */
|
||||
/*
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--accent-2: hsl(160, 100%, 96%);
|
||||
|
||||
--text-1: #eaeaea;
|
||||
--text-2: #1a1a1a;
|
||||
--bg-main: #101010;
|
||||
}
|
||||
|
||||
#table-bookmarks_wrapper,
|
||||
#table-bookmarks_wrapper img,
|
||||
tbody td:nth-child(3),
|
||||
tbody td:nth-child(3) span,
|
||||
footer {
|
||||
filter: invert(100%);
|
||||
}
|
||||
}*/
|
||||
|
||||
html,
|
||||
body {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
font-size: 18px;
|
||||
font-weight: 200;
|
||||
text-align: center;
|
||||
margin: 0px;
|
||||
padding: 0px;
|
||||
font-family: var(--font-main);
|
||||
}
|
||||
|
||||
.header-top small {
|
||||
font-weight: 200;
|
||||
color: var(--accent-3);
|
||||
}
|
||||
|
||||
.header-top {
|
||||
width: 100%;
|
||||
height: auto;
|
||||
min-height: 40px;
|
||||
margin: 0px;
|
||||
text-align: center;
|
||||
color: white;
|
||||
font-size: calc(11px + 0.84vw);
|
||||
font-weight: 200;
|
||||
padding: 4px 4px;
|
||||
border-bottom: 3px solid var(--accent-1);
|
||||
background-color: var(--accent-1);
|
||||
}
|
||||
|
||||
input[type=search] {
|
||||
width: 22vw;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #aeaeae;
|
||||
padding: 3px 5px;
|
||||
}
|
||||
|
||||
.nav>div {
|
||||
min-height: 30px;
|
||||
}
|
||||
|
||||
.header-top a {
|
||||
text-decoration: none;
|
||||
color: rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
|
||||
.header-top a:hover {
|
||||
text-decoration: none;
|
||||
color: rgba(0, 0, 0, 0.9);
|
||||
}
|
||||
|
||||
.header-top .col-lg-4 {
|
||||
text-align: center;
|
||||
padding-top: 4px;
|
||||
padding-bottom: 4px;
|
||||
}
|
||||
|
||||
.header-archivebox img {
|
||||
display: inline-block;
|
||||
margin-right: 3px;
|
||||
height: 30px;
|
||||
margin-left: 12px;
|
||||
margin-top: -4px;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
|
||||
.header-archivebox img:hover {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
#table-bookmarks_length,
|
||||
#table-bookmarks_filter {
|
||||
padding-top: 12px;
|
||||
opacity: 0.8;
|
||||
padding-left: 24px;
|
||||
padding-right: 22px;
|
||||
margin-bottom: -16px;
|
||||
}
|
||||
|
||||
table {
|
||||
padding: 6px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
table thead th {
|
||||
font-weight: 400;
|
||||
}
|
||||
|
||||
table tr {
|
||||
height: 35px;
|
||||
}
|
||||
|
||||
tbody tr:nth-child(odd) {
|
||||
background-color: var(--accent-2) !important;
|
||||
}
|
||||
|
||||
table tr td {
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
/*padding-bottom: 0.4em;*/
|
||||
/*padding-top: 0.4em;*/
|
||||
padding-left: 2px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
table tr td a {
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
table tr td img,
|
||||
table tr td object {
|
||||
display: inline-block;
|
||||
margin: auto;
|
||||
height: 24px;
|
||||
width: 24px;
|
||||
padding: 0px;
|
||||
padding-right: 5px;
|
||||
vertical-align: middle;
|
||||
margin-left: 4px;
|
||||
}
|
||||
|
||||
#table-bookmarks {
|
||||
width: 100%;
|
||||
overflow-y: scroll;
|
||||
table-layout: fixed;
|
||||
}
|
||||
|
||||
.dataTables_wrapper {
|
||||
background-color: #fafafa;
|
||||
}
|
||||
|
||||
table tr a span[data-archived~=False] {
|
||||
opacity: 0.4;
|
||||
}
|
||||
|
||||
.files-spinner {
|
||||
height: 15px;
|
||||
width: auto;
|
||||
opacity: 0.5;
|
||||
vertical-align: -2px;
|
||||
}
|
||||
|
||||
.in-progress {
|
||||
display: none;
|
||||
}
|
||||
|
||||
tr td a.favicon img {
|
||||
padding-left: 6px;
|
||||
padding-right: 12px;
|
||||
vertical-align: -4px;
|
||||
}
|
||||
|
||||
tr td a.title {
|
||||
font-size: 1.4em;
|
||||
text-decoration: none;
|
||||
color: black;
|
||||
}
|
||||
|
||||
tr td a.title small {
|
||||
background-color: var(--accent-3);
|
||||
border-radius: 4px;
|
||||
float: right
|
||||
}
|
||||
|
||||
input[type=search]::-webkit-search-cancel-button {
|
||||
-webkit-appearance: searchfield-cancel-button;
|
||||
}
|
||||
|
||||
.title-col {
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.title-col a {
|
||||
color: black;
|
||||
}
|
||||
|
||||
.exists-False {
|
||||
opacity: 0.1;
|
||||
filter: grayscale(100%);
|
||||
pointer-events: none;
|
||||
}
|
||||
</style>
|
||||
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
|
||||
<link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}" />
|
||||
{% block extra_head %}
|
||||
{% endblock %}
|
||||
<script src="{% static 'jquery.min.js' %}"></script>
|
||||
<script src="{% static 'jquery.dataTables.min.js' %}"></script>
|
||||
<script>
|
||||
document.addEventListener('error', function (e) {
|
||||
e.target.style.opacity = 0;
|
||||
}, true)
|
||||
jQuery(document).ready(function () {
|
||||
jQuery('#table-bookmarks').DataTable({
|
||||
searching: false,
|
||||
paging: false,
|
||||
stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
|
||||
dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
|
||||
order: [[0, 'desc']],
|
||||
iDisplayLength: 100,
|
||||
});
|
||||
});
|
||||
</script>
|
||||
<base href="{% url 'Home' %}">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<div class="header-top container-fluid">
|
||||
<div class="row nav">
|
||||
<div class="col-sm-2">
|
||||
<a href="{% url 'public-index' %}" class="header-archivebox" title="Last updated: {{updated}}">
|
||||
<img src="{% static 'archive.png' %}" alt="Logo" />
|
||||
ArchiveBox: Index
|
||||
</a>
|
||||
</div>
|
||||
<div class="col-sm-10" style="text-align: right">
|
||||
<a href="/add/">Add Links</a> |
|
||||
<a href="/admin/core/snapshot/">Admin</a> |
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
{% block body %}
|
||||
{% endblock %}
|
||||
<br>
|
||||
<footer>
|
||||
<br />
|
||||
<center>
|
||||
<small>
|
||||
Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a> version
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/releases" title="Releases">v{{VERSION}}</a>.
|
||||
<br/><br/>
|
||||
{{FOOTER_INFO}}
|
||||
</small>
|
||||
</center>
|
||||
<br />
|
||||
</footer>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
{% load static %}
|
||||
|
||||
<tr>
|
||||
<td title="{{link.timestamp}}"> {% if link.bookmarked_date %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} </td>
|
||||
<td class="title-col">
|
||||
{% if link.is_archived %}
|
||||
<a href="archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" class="link-favicon" decoding="async"></a>
|
||||
{% else %}
|
||||
<a href="archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" class="link-favicon" decoding="async"></a>
|
||||
{% endif %}
|
||||
<a href="archive/{{link.timestamp}}/{{link.canonical_outputs.wget_path}}" title="{{link.title}}">
|
||||
<span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">{{link.title|default:'Loading...'}}</span>
|
||||
<small style="float:right">{% if link.tags_str != None %} {{link.tags_str|default:''}} {% else %} {{ link.tags|default:'' }} {% endif %}</small>
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="archive/{{link.timestamp}}/index.html">📄
|
||||
<span data-number-for="{{link.url}}" title="Fetching any missing files...">{% if link.icons %} {{link.icons}} {% else %} {{ link.num_outputs}} {% endif %}<img src="{% static 'spinner.gif' %}" class="files-spinner" decoding="async"/></span>
|
||||
</a>
|
||||
</td>
|
||||
<td style="text-align:left"><a href="{{link.url}}">{{link.url}}</a></td>
|
||||
</tr>
|
||||
|
|
@ -1,215 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Archived Sites</title>
|
||||
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
html, body {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
font-size: 18px;
|
||||
font-weight: 200;
|
||||
text-align: center;
|
||||
margin: 0px;
|
||||
padding: 0px;
|
||||
font-family: "Gill Sans", Helvetica, sans-serif;
|
||||
}
|
||||
.header-top small {
|
||||
font-weight: 200;
|
||||
color: #efefef;
|
||||
}
|
||||
|
||||
.header-top {
|
||||
width: 100%;
|
||||
height: auto;
|
||||
min-height: 40px;
|
||||
margin: 0px;
|
||||
text-align: center;
|
||||
color: white;
|
||||
font-size: calc(11px + 0.84vw);
|
||||
font-weight: 200;
|
||||
padding: 4px 4px;
|
||||
border-bottom: 3px solid #aa1e55;
|
||||
background-color: #aa1e55;
|
||||
}
|
||||
input[type=search] {
|
||||
width: 22vw;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #aeaeae;
|
||||
padding: 3px 5px;
|
||||
}
|
||||
.nav > div {
|
||||
min-height: 30px;
|
||||
}
|
||||
.header-top a {
|
||||
text-decoration: none;
|
||||
color: rgba(0,0,0,0.6);
|
||||
}
|
||||
.header-top a:hover {
|
||||
text-decoration: none;
|
||||
color: rgba(0,0,0,0.9);
|
||||
}
|
||||
.header-top .col-lg-4 {
|
||||
text-align: center;
|
||||
padding-top: 4px;
|
||||
padding-bottom: 4px;
|
||||
}
|
||||
.header-archivebox img {
|
||||
display: inline-block;
|
||||
margin-right: 3px;
|
||||
height: 30px;
|
||||
margin-left: 12px;
|
||||
margin-top: -4px;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
.header-archivebox img:hover {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
#table-bookmarks_length, #table-bookmarks_filter {
|
||||
padding-top: 12px;
|
||||
opacity: 0.8;
|
||||
padding-left: 24px;
|
||||
padding-right: 22px;
|
||||
margin-bottom: -16px;
|
||||
}
|
||||
table {
|
||||
padding: 6px;
|
||||
width: 100%;
|
||||
}
|
||||
table thead th {
|
||||
font-weight: 400;
|
||||
}
|
||||
table tr {
|
||||
height: 35px;
|
||||
}
|
||||
tbody tr:nth-child(odd) {
|
||||
background-color: #ffebeb !important;
|
||||
}
|
||||
table tr td {
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
/*padding-bottom: 0.4em;*/
|
||||
/*padding-top: 0.4em;*/
|
||||
padding-left: 2px;
|
||||
text-align: center;
|
||||
}
|
||||
table tr td a {
|
||||
text-decoration: none;
|
||||
}
|
||||
table tr td img, table tr td object {
|
||||
display: inline-block;
|
||||
margin: auto;
|
||||
height: 24px;
|
||||
width: 24px;
|
||||
padding: 0px;
|
||||
padding-right: 5px;
|
||||
vertical-align: middle;
|
||||
margin-left: 4px;
|
||||
}
|
||||
#table-bookmarks {
|
||||
width: 100%;
|
||||
overflow-y: scroll;
|
||||
table-layout: fixed;
|
||||
}
|
||||
.dataTables_wrapper {
|
||||
background-color: #fafafa;
|
||||
}
|
||||
table tr a span[data-archived~=False] {
|
||||
opacity: 0.4;
|
||||
}
|
||||
.files-spinner {
|
||||
height: 15px;
|
||||
width: auto;
|
||||
opacity: 0.5;
|
||||
vertical-align: -2px;
|
||||
}
|
||||
.in-progress {
|
||||
display: none;
|
||||
}
|
||||
tr td a.favicon img {
|
||||
padding-left: 6px;
|
||||
padding-right: 12px;
|
||||
vertical-align: -4px;
|
||||
}
|
||||
tr td a.title {
|
||||
font-size: 1.4em;
|
||||
text-decoration:none;
|
||||
color:black;
|
||||
}
|
||||
tr td a.title small {
|
||||
background-color: #efefef;
|
||||
border-radius: 4px;
|
||||
float:right
|
||||
}
|
||||
input[type=search]::-webkit-search-cancel-button {
|
||||
-webkit-appearance: searchfield-cancel-button;
|
||||
}
|
||||
.title-col {
|
||||
text-align: left;
|
||||
}
|
||||
.title-col a {
|
||||
color: black;
|
||||
}
|
||||
</style>
|
||||
<link rel="stylesheet" href="static/bootstrap.min.css">
|
||||
<link rel="stylesheet" href="static/jquery.dataTables.min.css"/>
|
||||
<script src="static/jquery.min.js"></script>
|
||||
<script src="static/jquery.dataTables.min.js"></script>
|
||||
<script>
|
||||
document.addEventListener('error', function(e) {
|
||||
e.target.style.opacity = 0;
|
||||
}, true)
|
||||
jQuery(document).ready(function() {
|
||||
jQuery('#table-bookmarks').DataTable({
|
||||
stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
|
||||
dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
|
||||
order: [[0, 'desc']],
|
||||
iDisplayLength: 100,
|
||||
});
|
||||
});
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<div class="header-top container-fluid">
|
||||
<div class="row nav">
|
||||
<div class="col-sm-2">
|
||||
<a href="?" class="header-archivebox" title="Last updated: $time_updated">
|
||||
<img src="static/archive.png" alt="Logo"/>
|
||||
ArchiveBox
|
||||
</a>
|
||||
</div>
|
||||
<div class="col-sm-10" style="text-align: right">
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a> |
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox">Source</a> |
|
||||
<a href="https://archivebox.io">Website</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<table id="table-bookmarks">
|
||||
<thead>
|
||||
<tr class="thead-tr">
|
||||
<th style="width: 100px;">Bookmarked</th>
|
||||
<th style="width: 26vw;">Snapshot ($num_links)</th>
|
||||
<th style="width: 50px">Files</th>
|
||||
<th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>$rows</tbody>
|
||||
</table>
|
||||
<footer>
|
||||
<br/>
|
||||
<center>
|
||||
<small>
|
||||
Created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
|
||||
version <a href="https://github.com/ArchiveBox/ArchiveBox/releases" title="Releases">v$version</a>.
|
||||
<br/><br/>
|
||||
$footer_info
|
||||
</small>
|
||||
</center>
|
||||
<br/>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<tr>
|
||||
<td title="$timestamp">$bookmarked_date</td>
|
||||
<td class="title-col">
|
||||
<a href="$archive_path/index.html" class="link-url"><img src="$favicon_url" class="link-favicon" decoding="async"></a>
|
||||
<a href="$archive_path/$wget_url" title="$title">
|
||||
<span data-title-for="$url" data-archived="$is_archived">$title</span>
|
||||
<small style="float:right">$tags</small>
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href="$archive_path/index.html">📄
|
||||
<span data-number-for="$url" title="Number of extractor outputs">$num_outputs</span>
|
||||
</a>
|
||||
</td>
|
||||
<td style="text-align:left"><a href="$url">$url</a></td>
|
||||
</tr>
|
||||
|
|
@ -200,7 +200,13 @@ def get_headers(url: str, timeout: int=None) -> str:
|
|||
stream=True
|
||||
)
|
||||
|
||||
return pyjson.dumps(dict(response.headers), indent=4)
|
||||
return pyjson.dumps(
|
||||
{
|
||||
'Status-Code': response.status_code,
|
||||
**dict(response.headers),
|
||||
},
|
||||
indent=4,
|
||||
)
|
||||
|
||||
|
||||
@enforce_types
|
||||
|
|
|
|||
0
bin/build_git.sh
Executable file → Normal file
0
bin/build_git.sh
Executable file → Normal file
|
|
@ -1,24 +1,33 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Autodetect UID,GID of host user based on ownership of files in the data volume
|
||||
DATA_DIR="${DATA_DIR:-/data}"
|
||||
ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
|
||||
|
||||
USID=$(stat --format="%u" "$DATA_DIR")
|
||||
GRID=$(stat --format="%g" "$DATA_DIR")
|
||||
|
||||
# If user is not root, modify the archivebox user+files to have the same uid,gid
|
||||
if [[ "$USID" != 0 && "$GRID" != 0 ]]; then
|
||||
usermod -u "$USID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
|
||||
groupmod -g "$GRID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
|
||||
chown -R "$USID":"$GRID" "/home/$ARCHIVEBOX_USER"
|
||||
chown "$USID":"$GRID" "$DATA_DIR"
|
||||
chown "$USID":"$GRID" "$DATA_DIR/*" > /dev/null 2>&1 || true
|
||||
# Set the archivebox user UID & GID
|
||||
if [[ -n "$PUID" && "$PUID" != 0 ]]; then
|
||||
usermod -u "$PUID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
|
||||
fi
|
||||
if [[ -n "$PGID" && "$PGID" != 0 ]]; then
|
||||
groupmod -g "$PGID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
|
||||
fi
|
||||
|
||||
# Run commands as the new archivebox user in Docker.
|
||||
# Any files touched will have the same uid & gid
|
||||
# inside Docker and outside on the host machine.
|
||||
# Set the permissions of the data dir to match the archivebox user
|
||||
if [[ -d "$DATA_DIR/archive" ]]; then
|
||||
# check data directory permissions
|
||||
if [[ ! "$(stat -c %u $DATA_DIR/archive)" = "$(id -u archivebox)" ]]; then
|
||||
echo "Change in ownership detected, please be patient while we chown existing files"
|
||||
echo "This could take some time..."
|
||||
chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER -R "$DATA_DIR"
|
||||
fi
|
||||
else
|
||||
# create data directory
|
||||
mkdir -p "$DATA_DIR"
|
||||
chown -R $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
|
||||
fi
|
||||
chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
|
||||
|
||||
|
||||
# Drop permissions to run commands as the archivebox user
|
||||
if [[ "$1" == /* || "$1" == "echo" || "$1" == "archivebox" ]]; then
|
||||
# arg 1 is a binary, execute it verbatim
|
||||
# e.g. "archivebox init"
|
||||
|
|
|
|||
0
bin/release_brew.sh
Executable file → Normal file
0
bin/release_brew.sh
Executable file → Normal file
0
bin/release_deb.sh
Executable file → Normal file
0
bin/release_deb.sh
Executable file → Normal file
0
bin/release_docker.sh
Executable file → Normal file
0
bin/release_docker.sh
Executable file → Normal file
0
bin/release_docs.sh
Executable file → Normal file
0
bin/release_docs.sh
Executable file → Normal file
0
bin/release_git.sh
Executable file → Normal file
0
bin/release_git.sh
Executable file → Normal file
0
bin/release_pip.sh
Executable file → Normal file
0
bin/release_pip.sh
Executable file → Normal file
28
bin/setup.sh
28
bin/setup.sh
|
|
@ -1,6 +1,5 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
# ArchiveBox Setup Script
|
||||
# Nick Sweeting 2017 | MIT License
|
||||
# https://github.com/ArchiveBox/ArchiveBox
|
||||
|
||||
echo "[i] ArchiveBox Setup Script 📦"
|
||||
|
|
@ -8,27 +7,28 @@ echo ""
|
|||
echo " This is a helper script which installs the ArchiveBox dependencies on your system using homebrew/aptitude."
|
||||
echo " You may be prompted for a password in order to install the following:"
|
||||
echo ""
|
||||
echo " - git"
|
||||
echo " - python3, python3-pip, python3-distutils"
|
||||
echo " - curl"
|
||||
echo " - wget"
|
||||
echo " - git"
|
||||
echo " - youtube-dl"
|
||||
echo " - chromium-browser (skip this if Chrome/Chromium is already installed)"
|
||||
echo " - nodejs (used for singlefile, readability, mercury, and more)"
|
||||
echo ""
|
||||
echo " If you'd rather install these manually, you can find documentation here:"
|
||||
echo " https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
|
||||
echo ""
|
||||
echo "Press enter to continue with the automatic install, or Ctrl+C to cancel..."
|
||||
read
|
||||
|
||||
read -p "Press [enter] to continue with the automatic install, or Ctrl+C to cancel..." REPLY
|
||||
echo ""
|
||||
|
||||
# On Linux:
|
||||
if which apt-get > /dev/null; then
|
||||
echo "[+] Updating apt repos..."
|
||||
apt update -q
|
||||
echo "[+] Adding ArchiveBox apt repo to sources..."
|
||||
sudo apt install software-properties-common
|
||||
sudo add-apt-repository -u ppa:archivebox/archivebox
|
||||
echo "[+] Installing python3, wget, curl..."
|
||||
apt install git python3 python3-pip python3-distutils wget curl youtube-dl
|
||||
sudo apt install -y git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm ripgrep
|
||||
# sudo apt install archivebox
|
||||
|
||||
if which google-chrome; then
|
||||
echo "[i] You already have google-chrome installed, if you would like to download chromium instead (they work pretty much the same), follow the Manual Setup instructions"
|
||||
|
|
@ -41,13 +41,13 @@ if which apt-get > /dev/null; then
|
|||
chromium --version
|
||||
else
|
||||
echo "[+] Installing chromium..."
|
||||
apt install chromium
|
||||
sudo apt install chromium || sudo apt install chromium-browser
|
||||
fi
|
||||
|
||||
# On Mac:
|
||||
elif which brew > /dev/null; then # 🐍 eye of newt
|
||||
echo "[+] Installing python3, wget, curl (ignore 'already installed' warnings)..."
|
||||
brew install git wget curl youtube-dl
|
||||
brew install git wget curl youtube-dl ripgrep node
|
||||
if which python3; then
|
||||
if python3 -c 'import sys; raise SystemExit(sys.version_info < (3,5,0))'; then
|
||||
echo "[√] Using existing $(which python3)..."
|
||||
|
|
@ -83,7 +83,11 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
python3 -m pip install --upgrade archivebox
|
||||
npm i -g npm
|
||||
pip3 install --upgrade pip setuptools
|
||||
|
||||
pip3 install --upgrade archivebox
|
||||
npm install -g 'git+https://github.com/ArchiveBox/ArchiveBox.git'
|
||||
|
||||
# Check:
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@
|
|||
# PUBLIC_INDEX = True
|
||||
# PUBLIC_SNAPSHOTS = True
|
||||
# FOOTER_INFO = Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.
|
||||
# ACTIVE_THEME = default
|
||||
|
||||
|
||||
[ARCHIVE_METHOD_TOGGLES]
|
||||
|
|
|
|||
Loading…
Reference in a new issue