From beb0502fb47196e13910405e48da76d84cbcae48 Mon Sep 17 00:00:00 2001 From: James DiGioia Date: Sun, 10 Jan 2021 17:08:00 -0500 Subject: [PATCH 001/151] Publish tag to Docker Hub This tags the versions released on Docker Hub so we can rely on those versions rather than the sha. --- .github/workflows/docker.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1d8c14e7..c624cec3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,6 +52,19 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- + - name: Get publish tag + id: publish + run: | + if [[ $GITHUB_REF != refs/tags/* ]]; then + TAG="${GITHUB_REF##*/}" + else + TAG=$GITHUB_SHA + fi + echo ::set-output name=tag::${TAG} + env: + GITHUB_REF: ${{ github.ref }} + GITHUB_SHA: ${{ github.sha }} + - name: Build and push id: docker_build uses: docker/build-push-action@v2 @@ -62,12 +75,12 @@ jobs: push: true tags: | ${{ secrets.DOCKER_USERNAME }}/archivebox:latest - ${{ secrets.DOCKER_USERNAME }}/archivebox:${{ github.sha }} + ${{ secrets.DOCKER_USERNAME }}/archivebox:${{ steps.publish.outputs.tag }} archivebox/archivebox:latest - archivebox/archivebox:${{ github.sha }} + archivebox/archivebox:${{ steps.publish.outputs.tag }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/amd64,linux/arm64,linux/arm/v7 - + - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 8cdf43ec378a0a7928c04b821c6180eccb59c6cd Mon Sep 17 00:00:00 2001 From: James DiGioia Date: Sun, 10 Jan 2021 19:07:50 -0500 Subject: [PATCH 002/151] Fix tag logic --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c624cec3..8efe5c27 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -55,8 +55,8 @@ jobs: - name: Get publish tag id: publish run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - TAG="${GITHUB_REF##*/}" + if [[ $GITHUB_REF == refs/tags/* ]]; then + TAG="${GITHUB_REF#refs/tags/}" else TAG=$GITHUB_SHA fi From 40ce95a9e4aa0813a33b42d23fa788f92f033b5d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 11 Jan 2021 02:50:57 +0200 Subject: [PATCH 003/151] also tag image with short version --- bin/build_docker.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/build_docker.sh b/bin/build_docker.sh index 0115acdf..42fade38 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -12,6 +12,7 @@ IFS=$'\n' REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')" cd "$REPO_DIR" which docker > /dev/null @@ -20,9 +21,13 @@ echo "[+] Building docker image in the background..." docker build . -t archivebox \ -t archivebox:latest \ -t archivebox:$VERSION \ + -t archivebox:$SHORT_VERSION \ -t docker.io/nikisweeting/archivebox:latest \ -t docker.io/nikisweeting/archivebox:$VERSION \ + -t docker.io/nikisweeting/archivebox:$SHORT_VERSION \ -t docker.io/archivebox/archivebox:latest \ -t docker.io/archivebox/archivebox:$VERSION \ + -t docker.io/archivebox/archivebox:$SHORT_VERSION \ -t docker.pkg.github.com/pirate/archivebox/archivebox:latest \ - -t docker.pkg.github.com/pirate/archivebox/archivebox:$VERSION + -t docker.pkg.github.com/pirate/archivebox/archivebox:$VERSION \ + -t docker.pkg.github.com/pirate/archivebox/archivebox:$SHORT_VERSION From b5ce5b35a85aa1fa6c03869811c8dbef383f7f12 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 12 Jan 2021 12:51:18 +0200 Subject: [PATCH 004/151] fix apt install without update in deb build --- .github/workflows/debian.yml | 5 +++-- bin/build_docs.sh | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 82a635d0..6492f020 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -19,7 +19,8 @@ jobs: - name: Install packaging dependencies run: | - sudo apt install -y \ + sudo apt-get update -qq + sudo apt-get install -y \ python3 python3-dev python3-pip python3-venv python3-all \ dh-python debhelper devscripts dput software-properties-common \ python3-distutils python3-setuptools python3-wheel python3-stdeb @@ -36,7 +37,7 @@ jobs: - name: Install archivebox from deb run: | cd deb_dist/ - sudo apt install ./archivebox*.deb + sudo apt-get install ./archivebox*.deb - name: Check ArchiveBox version run: | diff --git a/bin/build_docs.sh b/bin/build_docs.sh index afc849ed..5fa220fb 100755 --- a/bin/build_docs.sh +++ b/bin/build_docs.sh @@ -20,7 +20,6 @@ fi cd "$REPO_DIR" - echo "[*] Fetching latest docs version" cd "$REPO_DIR/docs" git pull From a3008c8189d9eb798a8c11f203d6e4700876ea32 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 12 Jan 2021 12:55:38 +0200 Subject: [PATCH 005/151] fix migration failing due to null cmd_versions in older archives --- archivebox/core/migrations/0007_archiveresult.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/core/migrations/0007_archiveresult.py b/archivebox/core/migrations/0007_archiveresult.py index a780376f..ec48d3ff 100644 --- a/archivebox/core/migrations/0007_archiveresult.py +++ b/archivebox/core/migrations/0007_archiveresult.py @@ -36,7 +36,7 @@ def forwards_func(apps, schema_editor): for extractor in history: for result in history[extractor]: - ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=result["cmd"], cmd_version=result["cmd_version"], + ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=result["cmd"], cmd_version=result["cmd_version"] or 'unknown', start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"]) From 5250989e326de83803933f795596dee1f3921af2 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 12 Jan 2021 12:56:00 +0200 Subject: [PATCH 006/151] split up release script into subscripts --- bin/build_git.sh | 38 +++++++++++++++++++++++ bin/release.sh | 71 +++++++++---------------------------------- bin/release_brew.sh | 19 ++++++++++++ bin/release_deb.sh | 20 ++++++++++++ bin/release_docker.sh | 24 +++++++++++++++ bin/release_docs.sh | 25 +++++++++++++++ bin/release_git.sh | 25 +++++++++++++++ bin/release_pip.sh | 26 ++++++++++++++++ 8 files changed, 192 insertions(+), 56 deletions(-) create mode 100644 bin/build_git.sh create mode 100644 bin/release_brew.sh create mode 100644 bin/release_deb.sh create mode 100644 bin/release_docker.sh create mode 100644 bin/release_docs.sh create mode 100644 bin/release_git.sh create mode 100644 bin/release_pip.sh diff --git a/bin/build_git.sh b/bin/build_git.sh new file mode 100644 index 00000000..19e185e8 --- /dev/null +++ b/bin/build_git.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" + +cd "$REPO_DIR" +source "./.venv/bin/activate" + + +# Make sure git is clean +if [ -z "$(git status --porcelain)" ] && [[ "$(git branch --show-current)" == "master" ]]; then + git pull +else + echo "[!] Warning: git status is dirty!" + echo " Press Ctrl-C to cancel, or wait 10sec to continue..." + sleep 10 +fi + +# Bump version number in source +function bump_semver { + echo "$1" | awk -F. '{$NF = $NF + 1;} 1' | sed 's/ /./g' +} + +OLD_VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +NEW_VERSION="$(bump_semver "$OLD_VERSION")" +echo "[*] Bumping VERSION from $OLD_VERSION to $NEW_VERSION" +contents="$(jq ".version = \"$NEW_VERSION\"" "$REPO_DIR/package.json")" && \ +echo "${contents}" > package.json + diff --git a/bin/release.sh b/bin/release.sh index 12459c74..34256fad 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -11,69 +11,28 @@ set -o pipefail IFS=$'\n' REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" - cd "$REPO_DIR" -source "./.venv/bin/activate" -# Make sure git is clean -if [ -z "$(git status --porcelain)" ] && [[ "$(git branch --show-current)" == "master" ]]; then - git pull -else - echo "[!] Warning: git status is dirty!" - echo " Press Ctrl-C to cancel, or wait 10sec to continue..." - sleep 10 -fi +# Run the linters and tests +# ./bin/lint.sh +# ./bin/test.sh - -# Bump version number in source -function bump_semver { - echo "$1" | awk -F. '{$NF = $NF + 1;} 1' | sed 's/ /./g' -} - -OLD_VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" -NEW_VERSION="$(bump_semver "$OLD_VERSION")" -echo "[*] Bumping VERSION from $OLD_VERSION to $NEW_VERSION" -contents="$(jq ".version = \"$NEW_VERSION\"" "$REPO_DIR/package.json")" && \ -echo "${contents}" > package.json - - -# Build docs, python package, and docker image +# Run all the build scripts +./bin/build_git.sh ./bin/build_docs.sh ./bin/build_pip.sh ./bin/build_deb.sh +./bin/build_brew.sh ./bin/build_docker.sh +# Push relase to public repositories +./bin/release_git.sh +./bin/release_docs.sh +./bin/release_pip.sh +./bin/release_deb.sh +./bin/release_brew.sh +./bin/release_docker.sh -# Push build to github -echo "[^] Pushing source to github" -git add "$REPO_DIR/docs" -git add "$REPO_DIR/deb_dist" -git add "$REPO_DIR/pip_dist" -git add "$REPO_DIR/brew_dist" -git add "$REPO_DIR/package.json" -git add "$REPO_DIR/package-lock.json" -git commit -m "$NEW_VERSION release" -git tag -a "v$NEW_VERSION" -m "v$NEW_VERSION" -git push origin master -git push origin --tags - - -# Push releases to github -echo "[^] Uploading to test.pypi.org" -python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz} - -echo "[^] Uploading to pypi.org" -python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz} - -echo "[^] Uploading to launchpad.net" -dput archivebox "deb_dist/archivebox_${NEW_VERSION}-1_source.changes" - -echo "[^] Uploading docker image" -# docker login --username=nikisweeting -# docker login docker.pkg.github.com --username=pirate -docker push docker.io/nikisweeting/archivebox -docker push docker.io/archivebox/archivebox -docker push docker.pkg.github.com/archivebox/archivebox/archivebox - -echo "[√] Done. Published version v$NEW_VERSION" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +echo "[√] Done. Published version v$VERSION" diff --git a/bin/release_brew.sh b/bin/release_brew.sh new file mode 100644 index 00000000..526d9d59 --- /dev/null +++ b/bin/release_brew.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')" +cd "$REPO_DIR" + +# TODO +exit 0 diff --git a/bin/release_deb.sh b/bin/release_deb.sh new file mode 100644 index 00000000..dc1bff35 --- /dev/null +++ b/bin/release_deb.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')" +cd "$REPO_DIR" + + +echo "[^] Uploading to launchpad.net" +dput archivebox "deb_dist/archivebox_${VERSION}-1_source.changes" diff --git a/bin/release_docker.sh b/bin/release_docker.sh new file mode 100644 index 00000000..344a456d --- /dev/null +++ b/bin/release_docker.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')" +cd "$REPO_DIR" + + +echo "[^] Uploading docker image" +# docker login --username=nikisweeting +# docker login docker.pkg.github.com --username=pirate +docker push docker.io/nikisweeting/archivebox +docker push docker.io/archivebox/archivebox +docker push docker.pkg.github.com/archivebox/archivebox/archivebox diff --git a/bin/release_docs.sh b/bin/release_docs.sh new file mode 100644 index 00000000..114c1262 --- /dev/null +++ b/bin/release_docs.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')" +cd "$REPO_DIR" + + +echo "[^] Pushing docs to github" +cd docs/ +git commit -am "$NEW_VERSION release" +git push +git tag -a "v$NEW_VERSION" -m "v$NEW_VERSION" +git push origin master +git push origin --tags diff --git a/bin/release_git.sh b/bin/release_git.sh new file mode 100644 index 00000000..4a999e34 --- /dev/null +++ b/bin/release_git.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +cd "$REPO_DIR" + + +# Push build to github +echo "[^] Pushing release commit + tag to Github" +git commit -am "$VERSION release" +git tag -a "v$VERSION" -m "v$VERSION" +git push origin master +git push origin --tags +echo " To finish publishing the release go here:" +echo " https://github.com/ArchiveBox/ArchiveBox/releases/new" diff --git a/bin/release_pip.sh b/bin/release_pip.sh new file mode 100644 index 00000000..87323603 --- /dev/null +++ b/bin/release_pip.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")" +cd "$REPO_DIR" + + +# apt install python3 python3-all python3-dev +# pip install '.[dev]' + + +echo "[^] Uploading to test.pypi.org" +python3 -m twine upload --repository testpypi pip_dist/archivebox-${VERSION}*.{whl,tar.gz} + +echo "[^] Uploading to pypi.org" +python3 -m twine upload --repository pypi pip_dist/archivebox-${VERSION}*.{whl,tar.gz} From f50e49fa92d42cb13a05889da1adea6d726988f7 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 13 Jan 2021 05:52:59 -0500 Subject: [PATCH 007/151] require version info in all tickets --- .github/ISSUE_TEMPLATE/bug_report.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index c2bf8b23..220707b9 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,7 +10,8 @@ assignees: '' #### Describe the bug @@ -35,9 +36,11 @@ If applicable, post any relevant screenshots or copy/pasted terminal output from If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**. --> -#### Software versions +#### ArchiveBox version - - OS: ([e.g. macOS 10.14] the operating system you're running ArchiveBox on) - - ArchiveBox version: (`git rev-parse HEAD | head -c7` [e.g. d798117] commit ID of the version you're running) - - Python version: (`python3 --version` [e.g. 3.7.0]) - - Chrome version: (`chromium-browser --version` [e.g. 73.1.2.3] if relevant to bug) + +```logs +replace this line with the *full*, unshortened output of running `archivebox version` +``` + From ab6fdb83be4cef502720d073d57c3ace87d08233 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 13 Jan 2021 05:55:46 -0500 Subject: [PATCH 008/151] Update CONTRIBUTING.md --- .github/CONTRIBUTING.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 0d902dca..f78490a6 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -9,15 +9,15 @@ **Useful links:** -- https://github.com/pirate/ArchiveBox/issues -- https://github.com/pirate/ArchiveBox/pulls -- https://github.com/pirate/ArchiveBox/wiki/Roadmap -- https://github.com/pirate/ArchiveBox/wiki/Install#manual-setup +- https://github.com/ArchiveBox/ArchiveBox/issues +- https://github.com/ArchiveBox/ArchiveBox/pulls +- https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap +- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup ### Development Setup ```bash -git clone https://github.com/pirate/ArchiveBox +git clone https://github.com/ArchiveBox/ArchiveBox cd ArchiveBox # Ideally do this in a virtualenv pip install -e '.[dev]' # or use: pipenv install --dev @@ -31,6 +31,8 @@ pip install -e '.[dev]' # or use: pipenv install --dev ./bin/build.sh ``` +For more common tasks see the `Development` section at the bottom of the README. + ### Getting Help Open issues on Github or message me https://sweeting.me/#contact. From c5b7d9f2bf527c4ae42dc462c85f7974be868738 Mon Sep 17 00:00:00 2001 From: James DiGioia Date: Wed, 13 Jan 2021 09:07:12 -0500 Subject: [PATCH 009/151] Publish, minor, & major version to DockerHub --- .github/workflows/docker.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8efe5c27..2a85086a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,15 +52,24 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Get publish tag - id: publish + - name: Prepare tags to publish + id: prep run: | + # Always publish to latest. + TAGS="${{ secrets.DOCKER_USERNAME }}/archivebox:latest,archivebox/archivebox:latest" if [[ $GITHUB_REF == refs/tags/* ]]; then - TAG="${GITHUB_REF#refs/tags/}" + VERSION="${GITHUB_REF#refs/tags/}" + MINOR=${VERSION%.*} + MAJOR=${MINOR%.*} + TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$VERSION,archivebox/archivebox:$VERSION" + TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$MINOR,archivebox/archivebox:$MINOR" + TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$MAJOR,archivebox/archivebox:$MAJOR" else - TAG=$GITHUB_SHA + VERSION=$GITHUB_SHA + TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$VERSION,archivebox/archivebox:$VERSION" fi - echo ::set-output name=tag::${TAG} + + echo ::set-output name=tags::${TAGS} env: GITHUB_REF: ${{ github.ref }} GITHUB_SHA: ${{ github.sha }} @@ -73,11 +82,7 @@ jobs: file: ./Dockerfile builder: ${{ steps.buildx.outputs.name }} push: true - tags: | - ${{ secrets.DOCKER_USERNAME }}/archivebox:latest - ${{ secrets.DOCKER_USERNAME }}/archivebox:${{ steps.publish.outputs.tag }} - archivebox/archivebox:latest - archivebox/archivebox:${{ steps.publish.outputs.tag }} + tags: ${{ steps.prep.outputs.tags }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/amd64,linux/arm64,linux/arm/v7 From e9490ccfeb42acb5ef1f132155b8b5eaee779001 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 18 Jan 2021 15:14:07 -0500 Subject: [PATCH 010/151] clarify authenticated content archiving status --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 65e87230..cb7371a6 100644 --- a/README.md +++ b/README.md @@ -234,11 +234,11 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) - Easy to set up **[scheduled importing](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from multiple sources** - Uses common, **durable, [long-term formats](#saves-lots-of-useful-stuff-for-each-imported-link)** like HTML, JSON, PDF, PNG, and WARC -- ~~**Suitable for paywalled / [authenticated content](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir)** (can use your cookies)~~ (do not do this until v0.5 is released with some security fixes) - **Doesn't require a constantly-running daemon**, proxy, or native app - Provides a CLI, Python API, self-hosted web UI, and REST API (WIP) - Architected to be able to run [**many varieties of scripts during archiving**](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to extract media, summarize articles, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), expand comment threads, etc. - Can also [**mirror content to 3rd-party archiving services**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) automatically for redundancy +- ~~**Can archive paywalled / [authenticated content](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir)** (can use your cookies)~~ (doable, but not advisable until some pending security fixes are released) ## Input formats From 6c288f10e57f69d9da2208cbf2235bc4fbabe393 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 19 Jan 2021 22:02:35 -0500 Subject: [PATCH 011/151] fix README formatting for static site generator --- README.md | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index cb7371a6..b1130f42 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ docker-compose run archivebox help # to see more options Get ArchiveBox with docker on any platform First make sure you have Docker installed: https://docs.docker.com/get-docker/
+ ```bash # create a new empty directory and initalize your collection (can be anywhere) mkdir ~/archivebox && cd ~/archivebox @@ -130,6 +131,7 @@ archivebox help # to see more options ``` For other Debian-based systems or older Ubuntu systems you can add these sources to `/etc/apt/sources.list`: + ```bash deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main @@ -300,6 +302,7 @@ ArchiveBox is written in Python 3 so it requires `python3` and `pip3` available ## Caveats If you're importing URLs containing secret slugs or pages with private content (e.g Google Docs, CodiMD notepads, etc), you may want to disable some of the extractor modules to avoid leaking private URLs to 3rd party APIs during the archiving process. + ```bash # don't do this: archivebox add 'https://docs.google.com/document/d/12345somelongsecrethere' @@ -312,6 +315,7 @@ archivebox config --set CHROME_BINARY=chromium # optional: switch to chromium t ``` Be aware that malicious archived JS can also read the contents of other pages in your archive due to snapshot CSRF and XSS protections being imperfect. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) page for more details. + ```bash # visiting an archived page with malicious JS: https://127.0.0.1:8000/archive/1602401954/example.com/index.html @@ -323,6 +327,7 @@ https://127.0.0.1:8000/archive/* ``` Support for saving multiple snapshots of each site over time will be [added soon](https://github.com/ArchiveBox/ArchiveBox/issues/179) (along with the ability to view diffs of the changes between runs). For now ArchiveBox is designed to only archive each URL with each extractor type once. A workaround to take multiple snapshots of the same URL is to make them slightly different by adding a hash: + ```bash archivebox add 'https://example.com#2020-10-24' ... @@ -442,29 +447,41 @@ All contributions to ArchiveBox are welcomed! Check our [issues](https://github. ### Setup the dev environment -First, install the system dependencies from the "Bare Metal" section above. -Then you can clone the ArchiveBox repo and install -```python3 -git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox -git checkout master # or the branch you want to test +#### 1. Clone the main code repo (making sure to pull the submodules as well) + +```bash +git clone --recurse-submodules https://github.com/ArchiveBox/ArchiveBox +cd ArchiveBox +git checkout dev # or the branch you want to test git submodule update --init --recursive git pull --recurse-submodules +``` +#### 2. Option A: Install the Python, JS, and system dependencies directly on your machine + +```bash # Install ArchiveBox + python dependencies -python3 -m venv .venv && source .venv/bin/activate && pip install -e .[dev] -# or with pipenv: pipenv install --dev && pipenv shell +python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]' +# or: pipenv install --dev && pipenv shell # Install node dependencies npm install -# Optional: install extractor dependencies manually or with helper script +# Check to see if anything is missing +archivebox --version +# install any missing dependencies manually, or use the helper script: ./bin/setup.sh +``` +#### 2. Option B: Build the docker container and use that for development instead + +```bash # Optional: develop via docker by mounting the code dir into the container # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver # inside the container will reload and pick up your changes docker build . -t archivebox -docker run -it -p 8000:8000 \ +docker run -it --rm archivebox version +docker run -it --rm -p 8000:8000 \ -v $PWD/data:/data \ -v $PWD/archivebox:/app/archivebox \ archivebox server 0.0.0.0:8000 --debug --reload @@ -495,7 +512,7 @@ You can also run all these in Docker. For more examples see the Github Actions C cd archivebox/ ./manage.py makemigrations -cd data/ +cd path/to/test/data/ archivebox shell ``` (uses `pytest -s`) @@ -517,9 +534,14 @@ archivebox shell ```bash ./bin/release.sh -``` -(bumps the version, builds, and pushes a release to PyPI, Docker Hub, and Github Packages) +# or individually: +./bin/release_docs.sh +./bin/release_pip.sh +./bin/release_deb.sh +./bin/release_brew.sh +./bin/release_docker.sh +``` --- From 57d4da5ae674b8ca548cf80c7ff9c7ff8ad4371e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 19 Jan 2021 22:20:56 -0500 Subject: [PATCH 012/151] update key features list --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b1130f42..2037515f 100644 --- a/README.md +++ b/README.md @@ -232,15 +232,15 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the ## Key Features - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally -- [**Few dependencies**](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies) and [simple command line interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) +- [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies) - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) -- Easy to set up **[scheduled importing](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from multiple sources** -- Uses common, **durable, [long-term formats](#saves-lots-of-useful-stuff-for-each-imported-link)** like HTML, JSON, PDF, PNG, and WARC -- **Doesn't require a constantly-running daemon**, proxy, or native app -- Provides a CLI, Python API, self-hosted web UI, and REST API (WIP) -- Architected to be able to run [**many varieties of scripts during archiving**](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to extract media, summarize articles, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), expand comment threads, etc. -- Can also [**mirror content to 3rd-party archiving services**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) automatically for redundancy -- ~~**Can archive paywalled / [authenticated content](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir)** (can use your cookies)~~ (doable, but not advisable until some pending security fixes are released) +- Runs a [**wide variety of extractor plugins out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. youtube-dl for media, readability for article text, git for code repos, etc. +- Easy to set up **[scheduled/realtime importing](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from multiple sources** +- Uses standard, **durable, [long-term formats](#saves-lots-of-useful-stuff-for-each-imported-link)** like HTML, JSON, PDF, PNG, and WARC +- Usable as a **[oneshot CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage)**, **[self-hosted web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage)**, [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), REST API (ALPHA), and [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA) +- [**Also saves snapshots to archive.org**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode) +- Will support JS content scripts during archiving in the future, e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), expand comment threads, etc. +- Will support [content requiring a login/paywall/cookies to view](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (currently doable, but not advised until some pending security fixes are released) ## Input formats From bffbdd6d8a3b1d5cdd1a946cb9957936629c3cb3 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 19 Jan 2021 22:23:30 -0500 Subject: [PATCH 013/151] switch to using pre blocks for code within detail sections --- README.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2037515f..839ad4da 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ First make sure you have Docker installed: https://docs.docker.com/get-docker/

This is the recommended way to run ArchiveBox because it includes *all* the extractors like chrome, wget, youtube-dl, git, etc., as well as full-text search with sonic, and many other great features. -```bash +
 # create a new empty directory and initalize your collection (can be anywhere)
 mkdir ~/archivebox && cd ~/archivebox
 curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml
@@ -76,7 +76,7 @@ open http://127.0.0.1:8000
 docker-compose run archivebox add 'https://example.com'
 docker-compose run archivebox status
 docker-compose run archivebox help  # to see more options
-```
+
@@ -85,7 +85,7 @@ docker-compose run archivebox help # to see more options First make sure you have Docker installed: https://docs.docker.com/get-docker/
-```bash +
 # create a new empty directory and initalize your collection (can be anywhere)
 mkdir ~/archivebox && cd ~/archivebox
 docker run -v $PWD:/data -it archivebox/archivebox init
@@ -100,14 +100,14 @@ open http://127.0.0.1:8000
 docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 docker run -v $PWD:/data -it archivebox/archivebox status
 docker run -v $PWD:/data -it archivebox/archivebox help  # to see more options
-```
+
Get ArchiveBox with apt on Ubuntu >=20.04 -```bash +
 sudo add-apt-repository -u ppa:archivebox/archivebox
 sudo apt install archivebox
 
@@ -128,14 +128,15 @@ archivebox status
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 archivebox help  # to see more options
-```
+
For other Debian-based systems or older Ubuntu systems you can add these sources to `/etc/apt/sources.list`: -```bash +
 deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
 deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
-```
+
+ (you may need to install some other dependencies manually however)
@@ -143,7 +144,7 @@ deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
Get ArchiveBox with brew on macOS >=10.13 -```bash +
 brew install archivebox/archivebox/archivebox
 
 # create a new empty directory and initalize your collection (can be anywhere)
@@ -163,14 +164,14 @@ archivebox status
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 archivebox help  # to see more options
-```
+
Get ArchiveBox with pip on any platform -```bash +
 pip3 install archivebox
 
 # create a new empty directory and initalize your collection (can be anywhere)
@@ -191,7 +192,7 @@ archivebox status
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 archivebox help  # to see more options
-```
+
From 49491b21960262d7744012cce365ad8b21b7c0b4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 19 Jan 2021 22:32:30 -0500 Subject: [PATCH 014/151] change install instruction formatting --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 839ad4da..dfcaf19f 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,9 @@ archivebox help # to see more options
Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box) +
First make sure you have Docker installed: https://docs.docker.com/get-docker/ -

+
This is the recommended way to run ArchiveBox because it includes *all* the extractors like chrome, wget, youtube-dl, git, etc., as well as full-text search with sonic, and many other great features. 
@@ -82,7 +83,7 @@ docker-compose run archivebox help  # to see more options
 
 

 Get ArchiveBox with docker on any platform
-
+

 First make sure you have Docker installed: https://docs.docker.com/get-docker/

 
 
@@ -108,6 +109,8 @@ docker run -v $PWD:/data -it archivebox/archivebox help  # to see more options
 Get ArchiveBox with apt on Ubuntu >=20.04
 
 
+# add the repo to your sources and install the archivebox package using apt
+sudo apt install software-properties-common
 sudo add-apt-repository -u ppa:archivebox/archivebox
 sudo apt install archivebox
 
@@ -137,14 +140,19 @@ deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
 deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
 

 
+Then run `apt update; apt install archivebox; archivebox --version`.
+
 (you may need to install some other dependencies manually however)
 
 

 
 

 Get ArchiveBox with brew on macOS >=10.13
+

+First make sure you have Homebrew installed:https://brew.sh/#install

 
 
+# install the archivebox package using homebrew
 brew install archivebox/archivebox/archivebox
 
 # create a new empty directory and initalize your collection (can be anywhere)
@@ -170,8 +178,11 @@ archivebox help  # to see more options
 
 

 Get ArchiveBox with pip on any platform
+

+First make sure you have Python >= 3.7 installed: https://realpython.com/installing-python/

 
 
+# install the archivebox package using pip3
 pip3 install archivebox
 
 # create a new empty directory and initalize your collection (can be anywhere)

From ab122674a7af17b1c95e22055b5cffb68bc486fd Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 22:35:54 -0500
Subject: [PATCH 015/151] Update README.md

---
 README.md | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index dfcaf19f..1141556f 100644
--- a/README.md
+++ b/README.md
@@ -56,10 +56,10 @@ archivebox help   # to see more options
 

 Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box)
 
-

 First make sure you have Docker installed: https://docs.docker.com/get-docker/
 

-This is the recommended way to run ArchiveBox because it includes *all* the extractors like chrome, wget, youtube-dl, git, etc., as well as full-text search with sonic, and many other great features.
+This is the recommended way to run ArchiveBox because it includes all the extractors like:

+chrome, wget, youtube-dl, git, etc., full-text search w/ sonic, and many other great features.
 
 
 # create a new empty directory and initalize your collection (can be anywhere)
@@ -83,8 +83,8 @@ docker-compose run archivebox help  # to see more options
 
 

 Get ArchiveBox with docker on any platform
-

-First make sure you have Docker installed: https://docs.docker.com/get-docker/

+
+First make sure you have Docker installed: https://docs.docker.com/get-docker/
 
 
 # create a new empty directory and initalize your collection (can be anywhere)
@@ -108,6 +108,8 @@ docker run -v $PWD:/data -it archivebox/archivebox help  # to see more options
 

 Get ArchiveBox with apt on Ubuntu >=20.04
 
+First make sure you're on Ubuntu >= 20.04, or scroll down for older/non-Ubuntu instructions.
+
 
 # add the repo to your sources and install the archivebox package using apt
 sudo apt install software-properties-common
@@ -148,8 +150,8 @@ Then run `apt update; apt install archivebox; archivebox --version`.
 
 

 Get ArchiveBox with brew on macOS >=10.13
-

-First make sure you have Homebrew installed:https://brew.sh/#install

+
+First make sure you have Homebrew installed: https://brew.sh/#install
 
 
 # install the archivebox package using homebrew

From 4de49ef960feded2f56fc9250ead4e42ddca1fd1 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 22:37:57 -0500
Subject: [PATCH 016/151] Update README.md

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 1141556f..fe31e2d6 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,7 @@ archivebox help   # to see more options
 

 Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box)
 
-First make sure you have Docker installed: https://docs.docker.com/get-docker/
+First make sure you have Docker installed: https://docs.docker.com/get-docker/
 

 This is the recommended way to run ArchiveBox because it includes all the extractors like:

 chrome, wget, youtube-dl, git, etc., full-text search w/ sonic, and many other great features.
@@ -84,7 +84,7 @@ docker-compose run archivebox help  # to see more options
 

 Get ArchiveBox with docker on any platform
 
-First make sure you have Docker installed: https://docs.docker.com/get-docker/
+First make sure you have Docker installed: https://docs.docker.com/get-docker/
 
 
 # create a new empty directory and initalize your collection (can be anywhere)
@@ -108,7 +108,7 @@ docker run -v $PWD:/data -it archivebox/archivebox help  # to see more options
 

 Get ArchiveBox with apt on Ubuntu >=20.04
 
-First make sure you're on Ubuntu >= 20.04, or scroll down for older/non-Ubuntu instructions.
+First make sure you're on Ubuntu >= 20.04, or scroll down for older/non-Ubuntu instructions.
 
 
 # add the repo to your sources and install the archivebox package using apt
@@ -151,7 +151,7 @@ Then run `apt update; apt install archivebox; archivebox --version`.
 

 Get ArchiveBox with brew on macOS >=10.13
 
-First make sure you have Homebrew installed: https://brew.sh/#install
+First make sure you have Homebrew installed: https://brew.sh/#install
 
 
 # install the archivebox package using homebrew
@@ -180,8 +180,8 @@ archivebox help  # to see more options
 
 

 Get ArchiveBox with pip on any platform
-

-First make sure you have Python >= 3.7 installed: https://realpython.com/installing-python/

+
+First make sure you have Python >= 3.7 installed: https://realpython.com/installing-python/
 
 
 # install the archivebox package using pip3

From 4f440f2f0fd2b83ed61a44fd40df411007415bd9 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 22:46:46 -0500
Subject: [PATCH 017/151] tweak formatting of pre code blocks

---
 README.md | 39 +++++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index fe31e2d6..495442ec 100644
--- a/README.md
+++ b/README.md
@@ -45,9 +45,8 @@ mkdir ~/archivebox && cd ~/archivebox    # this can be anywhere
 archivebox init
 
 archivebox add 'https://example.com'
-archivebox add --depth=1 'https://example.com'
-archivebox schedule --every=day https://getpocket.com/users/USERNAME/feed/all
-archivebox oneshot --extract=title,favicon,media https://www.youtube.com/watch?v=dQw4w9WgXcQ
+archivebox schedule --every=day --depth=1 'https://getpocket.com/users/USERNAME/feed/all'
+archivebox oneshot --extract=title,favicon,media 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
 archivebox help   # to see more options
 ```
 
@@ -57,27 +56,27 @@ archivebox help   # to see more options
 Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box)
 
 First make sure you have Docker installed: https://docs.docker.com/get-docker/
-

-This is the recommended way to run ArchiveBox because it includes all the extractors like:

-chrome, wget, youtube-dl, git, etc., full-text search w/ sonic, and many other great features.
 
-
+

 # create a new empty directory and initalize your collection (can be anywhere)
 mkdir ~/archivebox && cd ~/archivebox
-curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml
+curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml'
 docker-compose run archivebox init
 docker-compose run archivebox --version
 
 # start the webserver and open the UI (optional)
 docker-compose run archivebox manage createsuperuser
 docker-compose up -d
-open http://127.0.0.1:8000
+open 'http://127.0.0.1:8000'
 
 # you can also add links and manage your archive via the CLI:
 docker-compose run archivebox add 'https://example.com'
 docker-compose run archivebox status
 docker-compose run archivebox help  # to see more options
-

+

+
+This is the recommended way to run ArchiveBox because it includes all the extractors like:

+chrome, wget, youtube-dl, git, etc., full-text search w/ sonic, and many other great features.
 
 

 
@@ -86,7 +85,7 @@ docker-compose run archivebox help  # to see more options
 
 First make sure you have Docker installed: https://docs.docker.com/get-docker/
 
-
+

 # create a new empty directory and initalize your collection (can be anywhere)
 mkdir ~/archivebox && cd ~/archivebox
 docker run -v $PWD:/data -it archivebox/archivebox init
@@ -101,7 +100,7 @@ open http://127.0.0.1:8000
 docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'
 docker run -v $PWD:/data -it archivebox/archivebox status
 docker run -v $PWD:/data -it archivebox/archivebox help  # to see more options
-

+

 
 

 
@@ -110,7 +109,7 @@ docker run -v $PWD:/data -it archivebox/archivebox help  # to see more options
 
 First make sure you're on Ubuntu >= 20.04, or scroll down for older/non-Ubuntu instructions.
 
-
+

 # add the repo to your sources and install the archivebox package using apt
 sudo apt install software-properties-common
 sudo add-apt-repository -u ppa:archivebox/archivebox
@@ -133,14 +132,14 @@ archivebox status
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 archivebox help  # to see more options
-

+

 
 For other Debian-based systems or older Ubuntu systems you can add these sources to `/etc/apt/sources.list`:
 
-
+

 deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
 deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main
-

+

 
 Then run `apt update; apt install archivebox; archivebox --version`.
 
@@ -153,7 +152,7 @@ Then run `apt update; apt install archivebox; archivebox --version`.
 
 First make sure you have Homebrew installed: https://brew.sh/#install
 
-
+

 # install the archivebox package using homebrew
 brew install archivebox/archivebox/archivebox
 
@@ -174,7 +173,7 @@ archivebox status
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 archivebox help  # to see more options
-

+

 
 

 
@@ -183,7 +182,7 @@ archivebox help  # to see more options
 
 First make sure you have Python >= 3.7 installed: https://realpython.com/installing-python/
 
-
+

 # install the archivebox package using pip3
 pip3 install archivebox
 
@@ -205,7 +204,7 @@ archivebox status
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 archivebox help  # to see more options
-

+

 
 

  

From 4ae1a8beb126b3736493d4027c125bcf79824bb0 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 23:07:01 -0500
Subject: [PATCH 018/151] fix bolding

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 495442ec..322d86be 100644
--- a/README.md
+++ b/README.md
@@ -247,10 +247,10 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally
 - [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies) 
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
-- Runs a [**wide variety of extractor plugins out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. youtube-dl for media, readability for article text, git for code repos, etc.
-- Easy to set up **[scheduled/realtime importing](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from multiple sources**
-- Uses standard, **durable, [long-term formats](#saves-lots-of-useful-stuff-for-each-imported-link)** like HTML, JSON, PDF, PNG, and WARC
-- Usable as a **[oneshot CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage)**, **[self-hosted web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage)**, [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), REST API (ALPHA), and [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
+- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): media w/ youtube-dl, articles w/ readability, code w/ git, etc.
+- [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from many types of sources
+- [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC
+- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), REST API (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
 - [**Also saves snapshots to archive.org**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Will support JS content scripts during archiving in the future, e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), expand comment threads, etc.
 - Will support [content requiring a login/paywall/cookies to view](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (currently doable, but not advised until some pending security fixes are released)

From 6fe6a48dd7414ce4af9692c25ca56473117b4560 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 23:17:54 -0500
Subject: [PATCH 019/151] Update README.md

---
 README.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 322d86be..6aacaa88 100644
--- a/README.md
+++ b/README.md
@@ -30,7 +30,9 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 
 Your archive can be managed through the command line with commands like `archivebox add`, through the built-in Web UI `archivebox server`, or via the Python library API (beta). It can ingest bookmarks from a browser or service like Pocket/Pinboard, your entire browsing history, RSS feeds, or URLs one at a time. You can also schedule regular/realtime imports with `archivebox schedule`.
 
-The main index is a self-contained `index.sqlite3` file, and each snapshot is stored as a folder `data/archive//`, with an easy-to-read `index.html` and `index.json` within. For each page, ArchiveBox auto-extracts many types of assets/media and saves them in standard formats, with out-of-the-box support for: several types of HTML snapshots (wget, Chrome headless, singlefile), PDF snapshotting, screenshotting, WARC archiving, git repositories, images, audio, video, subtitles, article text, and more. The snapshots are browseable and managable offline through the filesystem, the built-in webserver, or the Python library API.
+Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within. Snapshots are browseable and managable offline through the filesystem, the built-in webserver, or the Python API.
+
+For each archived URL, ArchiveBox saves: several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more](#output-formats).
 
 ### Quickstart
 
@@ -251,9 +253,9 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from many types of sources
 - [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC
 - [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), REST API (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
-- [**Also saves snapshots to archive.org**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
-- Will support JS content scripts during archiving in the future, e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), expand comment threads, etc.
-- Will support [content requiring a login/paywall/cookies to view](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (currently doable, but not advised until some pending security fixes are released)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released)
+- Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), [expand threads](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
 
 ## Input formats
 

From eef865ac1d72e7f02b733b27d10798bac55a0fe8 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 23:23:52 -0500
Subject: [PATCH 020/151] simplify intro section

---
 README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 6aacaa88..32eb95f2 100644
--- a/README.md
+++ b/README.md
@@ -28,11 +28,12 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
 
-Your archive can be managed through the command line with commands like `archivebox add`, through the built-in Web UI `archivebox server`, or via the Python library API (beta). It can ingest bookmarks from a browser or service like Pocket/Pinboard, your entire browsing history, RSS feeds, or URLs one at a time. You can also schedule regular/realtime imports with `archivebox schedule`.
+Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within.
 
-Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within. Snapshots are browseable and managable offline through the filesystem, the built-in webserver, or the Python API.
+For each URL added (`archivebox add https://example.com`), ArchiveBox saves: several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more](#output-formats).
 
-For each archived URL, ArchiveBox saves: several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more](#output-formats).
+Archived site snapshots are browseable and managable offline with the CLI commands like `archivebox status`, directly through the filesystem `./archive/` folders, via the built-in web UI `archivebox server`, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha).
+You can `archivebox add` or `archivebox schedule` regular imports of URLs from browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or just add URLs one at a time manually.
 
 ### Quickstart
 

From 64c091a317ba355826f609657eb908d3a305bfb7 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 23:26:52 -0500
Subject: [PATCH 021/151] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 32eb95f2..a677c4c3 100644
--- a/README.md
+++ b/README.md
@@ -30,10 +30,10 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 
 Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within.
 
-For each URL added (`archivebox add https://example.com`), ArchiveBox saves: several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more](#output-formats).
+For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
+You can use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [from many other sources...](#input-formats)
 
-Archived site snapshots are browseable and managable offline with the CLI commands like `archivebox status`, directly through the filesystem `./archive/` folders, via the built-in web UI `archivebox server`, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha).
-You can `archivebox add` or `archivebox schedule` regular imports of URLs from browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or just add URLs one at a time manually.
+Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha).
 
 ### Quickstart
 

From 77827f7b5904d372ccdd53397f82bc7a18a14809 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 23:31:59 -0500
Subject: [PATCH 022/151] link to REST API info issue

---
 README.md | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index a677c4c3..e4c9acce 100644
--- a/README.md
+++ b/README.md
@@ -31,9 +31,9 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within.
 
 For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
-You can use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [from many other sources...](#input-formats)
+You can use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
 
-Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha).
+Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha).
 
 ### Quickstart
 
@@ -230,7 +230,7 @@ ArchiveBox is a command line tool, self-hostable web-archiving server, and Pytho
 
 To use ArchiveBox you start by creating a folder for your data to live in (it can be anywhere on your system), and running `archivebox init` inside of it. That will create a sqlite3 index and an `ArchiveBox.conf` file. After that, you can continue to add/export/manage/etc using the CLI `archivebox help`, or you can run the Web UI (recommended). If you only want to archive a single site, you can run `archivebox oneshot` to avoid having to create a whole collection.
 
-The CLI is considered "stable", the ArchiveBox Python API and REST APIs are "beta", and the [desktop app](https://github.com/ArchiveBox/desktop) is "alpha".
+The [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) is considered "stable", the ArchiveBox [Python API](https://docs.archivebox.io/en/latest/modules.html) and [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) are "alpha", and the [desktop app](https://github.com/ArchiveBox/desktop) is "alpha".
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer). You can also self-host your archivebox server on a public domain to provide archive.org-style public access to your site snapshots.
 
@@ -248,12 +248,12 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally
-- [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies) 
+- [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies) 
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
-- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): media w/ youtube-dl, articles w/ readability, code w/ git, etc.
-- [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from many types of sources
+- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): media w/ youtube-dl, articles w/ readability, code w/ git, [and more...](#output-formats)
+- [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC
-- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), REST API (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
+- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released)
 - Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), [expand threads](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
@@ -443,8 +443,8 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 - [Chromium Install](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install)
 - [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview)
 - [Troubleshooting](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
-- [Python API](https://docs.archivebox.io/en/latest/modules.html)
-- REST API (coming soon...)
+- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
+- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)
 
 ## More Info
 

From 732a4eed1b8c66f142edbf6547e1f6095bb294f4 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Tue, 19 Jan 2021 23:32:42 -0500
Subject: [PATCH 023/151] also info about config file

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e4c9acce..776490a0 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
 
-Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within.
+Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, `ArchiveBox.conf` config file, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within.
 
 For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
 You can use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)

From 5c7842ffb3e481e566c91a770a1d5cad61169c34 Mon Sep 17 00:00:00 2001
From: Dan Arnfield 
Date: Wed, 20 Jan 2021 09:24:34 -0600
Subject: [PATCH 024/151] Fix dependency dict entries

---
 archivebox/config.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/archivebox/config.py b/archivebox/config.py
index 9a3f9a77..8c05ef26 100644
--- a/archivebox/config.py
+++ b/archivebox/config.py
@@ -775,7 +775,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
             'version': config['PYTHON_VERSION'],
             'hash': bin_hash(config['PYTHON_BINARY']),
             'enabled': True,
-            'is_valid': bool(config['DJANGO_VERSION']),
+            'is_valid': bool(config['PYTHON_VERSION']),
         },
         'DJANGO_BINARY': {
             'path': bin_path(config['DJANGO_BINARY']),
@@ -787,7 +787,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
         'CURL_BINARY': {
             'path': bin_path(config['CURL_BINARY']),
             'version': config['CURL_VERSION'],
-            'hash': bin_hash(config['PYTHON_BINARY']),
+            'hash': bin_hash(config['CURL_BINARY']),
             'enabled': config['USE_CURL'],
             'is_valid': bool(config['CURL_VERSION']),
         },
@@ -803,7 +803,7 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
             'version': config['NODE_VERSION'],
             'hash': bin_hash(config['NODE_BINARY']),
             'enabled': config['USE_NODE'],
-            'is_valid': bool(config['SINGLEFILE_VERSION']),
+            'is_valid': bool(config['NODE_VERSION']),
         },
         'SINGLEFILE_BINARY': {
             'path': bin_path(config['SINGLEFILE_BINARY']),

From bbb6cc89e96fb10a2cb8a7f2a3ea84f4ec9ba3d5 Mon Sep 17 00:00:00 2001
From: Mario Campos 
Date: Wed, 20 Jan 2021 11:23:40 -0600
Subject: [PATCH 025/151] Create codeql-analysis.yml

---
 .github/workflows/codeql-analysis.yml | 32 +++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 .github/workflows/codeql-analysis.yml

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 00000000..66e331b2
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,32 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev ]
+  schedule:
+    - cron: '43 1 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        queries: security-extended
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

From 1659b47bb0834614cb0de50e4d1a6fa03b1bfc27 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 14:14:16 -0500
Subject: [PATCH 026/151] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 776490a0..fa104761 100644
--- a/README.md
+++ b/README.md
@@ -271,9 +271,9 @@ archivebox add --depth=1 'https://example.com/some/downloads.html'
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 ```
 
--  Browser history or bookmarks exports (Chrome, Firefox, Safari, IE, Opera, and more)
+-  [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [bookmarks exports](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (instructions: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), and more)
 -  RSS, XML, JSON, CSV, SQL, HTML, Markdown, TXT, or any other text-based format
--  Pocket, Pinboard, Instapaper, Shaarli, Delicious, Reddit Saved Posts, Wallabag, Unmark.it, OneTab, and more
+-  [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), Delicious, [Reddit Saved Posts](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), OneTab, [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 

From d8f6d4d51795b860c1f80980114cd5cf623ac8b7 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 14:17:44 -0500
Subject: [PATCH 027/151] Update README.md

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fa104761..56a13ccb 100644
--- a/README.md
+++ b/README.md
@@ -271,8 +271,9 @@ archivebox add --depth=1 'https://example.com/some/downloads.html'
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 ```
 
--  [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [bookmarks exports](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (instructions: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), and more)
--  RSS, XML, JSON, CSV, SQL, HTML, Markdown, TXT, or any other text-based format
+
+-  TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file)
+-  [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive))
 -  [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), Delicious, [Reddit Saved Posts](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), OneTab, [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.

From 14df0cbb7c8e948c46a69a398af4ec2e502dc7dd Mon Sep 17 00:00:00 2001
From: jdcaballerov <743513+jdcaballerov@users.noreply.github.com>
Date: Wed, 20 Jan 2021 14:51:46 -0500
Subject: [PATCH 028/151] Update sonic.py

Sonic buffer accepts 20.000 bytes not unicode characters, since the chunking here is on unicode characters, sending 20.000 characters will overflow sonic's buffer.
UTF-8 can take up to 6 bytes, so sending less than (20.000 / 6) rounded minus should be ok.
---
 archivebox/search/backends/sonic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/search/backends/sonic.py b/archivebox/search/backends/sonic.py
index f0beaddd..f3ef6628 100644
--- a/archivebox/search/backends/sonic.py
+++ b/archivebox/search/backends/sonic.py
@@ -5,7 +5,7 @@ from sonic import IngestClient, SearchClient
 from archivebox.util import enforce_types
 from archivebox.config import SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD, SONIC_BUCKET, SONIC_COLLECTION
 
-MAX_SONIC_TEXT_LENGTH = 20000
+MAX_SONIC_TEXT_LENGTH = 2000
 
 @enforce_types
 def index(snapshot_id: str, texts: List[str]):

From c16cfe740b0f0f2a95b2e0128be0099ad8e15757 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:19:22 -0500
Subject: [PATCH 029/151] add screenshot grid

---
 README.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/README.md b/README.md
index 56a13ccb..05072589 100644
--- a/README.md
+++ b/README.md
@@ -210,6 +210,7 @@ archivebox help  # to see more options
 

 
 

+
  
 ---
  
@@ -221,6 +222,43 @@ archivebox help  # to see more options
 For more information, see the full Quickstart guide, Usage, and Configuration docs.
 
 
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+brew install archivebox

+archivebox version
+		
+archivebox init

+		
+archivebox add
+
+archivebox data dir
+

+archivebox server
+
+archivebox server add
+
+archivebox server list
+
+archivebox server detail
+

+

+
 ---
 
 

From 6733388c8692c73a3f772890ed0ce46f13bba795 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:24:10 -0500
Subject: [PATCH 030/151] Update README.md

---
 README.md | 51 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 05072589..d9b2dd13 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,20 @@ You can use `archivebox schedule` to ingest URLs regularly from your browser boo
 
 Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha).
 
+

+. . . . . . . . . . . . . . . . . . . . . . . . . . . .
+

+Demo | Usage
+

+. . . . . . . . . . . . . . . . . . . . . . . . . . . .
+

+CLI Screenshot
+Desktop index screenshot
+Desktop details page Screenshot
+Desktop details page Screenshot

+

+

+
 ### Quickstart
 
 It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`).
@@ -222,6 +236,20 @@ archivebox help  # to see more options
 For more information, see the full Quickstart guide, Usage, and Configuration docs.
 
 
+---
+
+
+# Overview
+
+ArchiveBox is a command line tool, self-hostable web-archiving server, and Python library all-in-one. It can be installed on Docker, macOS, and Linux/BSD, and Windows. You can download and install it as a Debian/Ubuntu package, Homebrew package, Python3 package, or a Docker image. No matter which install method you choose, they all provide the same CLI, Web UI, and on-disk data format.
+
+To use ArchiveBox you start by creating a folder for your data to live in (it can be anywhere on your system), and running `archivebox init` inside of it. That will create a sqlite3 index and an `ArchiveBox.conf` file. After that, you can continue to add/export/manage/etc using the CLI `archivebox help`, or you can run the Web UI (recommended). If you only want to archive a single site, you can run `archivebox oneshot` to avoid having to create a whole collection.
+
+The [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) is considered "stable", the ArchiveBox [Python API](https://docs.archivebox.io/en/latest/modules.html) and [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) are "alpha", and the [desktop app](https://github.com/ArchiveBox/desktop) is "alpha".
+
+At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer). You can also self-host your archivebox server on a public domain to provide archive.org-style public access to your site snapshots.
+
+## Screenshots
 
 

 
@@ -259,29 +287,6 @@ For more information, see the 
-CLI Screenshot
-Desktop index screenshot
-Desktop details page Screenshot
-Desktop details page Screenshot

-Demo | Usage
-

-. . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

-
 
 ## Key Features
 

From ff311f63e9221eba8c92662fc88d55eaabc6ff2d Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:30:49 -0500
Subject: [PATCH 031/151] Update README.md

---
 README.md | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index d9b2dd13..1517d4e5 100644
--- a/README.md
+++ b/README.md
@@ -40,12 +40,9 @@ Archived content is browseable and managable locally with the CLI commands like
 

 Demo | Usage
 

-. . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

-CLI Screenshot
-Desktop index screenshot
-Desktop details page Screenshot
-Desktop details page Screenshot

+cli init screenshot
+server snapshot admin screenshot
+server snapshot details page screenshot

 
 

 
@@ -277,10 +274,10 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 archivebox server add
 
 

From 98aee266f54e70d1a63d33155a00ffbba536dd58 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:35:28 -0500
Subject: [PATCH 032/151] Update README.md

---
 README.md | 83 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 42 insertions(+), 41 deletions(-)

diff --git a/README.md b/README.md
index 1517d4e5..cccdeb34 100644
--- a/README.md
+++ b/README.md
@@ -38,11 +38,11 @@ Archived content is browseable and managable locally with the CLI commands like
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
 

-Demo | Usage
+Demo | Screenshots | Usage
 

 cli init screenshot
 server snapshot admin screenshot
-server snapshot details page screenshot

+server snapshot details page screenshot

 

 

 
@@ -246,45 +246,6 @@ The [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) is cons
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer). You can also self-host your archivebox server on a public domain to provide archive.org-style public access to your site snapshots.
 
-## Screenshots
-
-

-

-archivebox server list
+archivebox server list
 
 
-archivebox server detail
+archivebox server detail
 
 

 

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

-brew install archivebox

-archivebox version
-		
-archivebox init

-		
-archivebox add
-
-archivebox data dir
-

-archivebox server
-
-archivebox server add
-
-archivebox server list
-
-archivebox server detail
-

-

-
-
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally
@@ -391,6 +352,46 @@ archivebox add 'https://example.com#2020-10-24'
 archivebox add 'https://example.com#2020-10-25'
 ```
 
+
+## Screenshots
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+brew install archivebox

+archivebox version
+		
+archivebox init

+		
+archivebox add
+
+archivebox data dir
+

+archivebox server
+
+archivebox server add
+
+archivebox server list
+
+archivebox server detail
+

+

+
+
 ---
 
 


From d753f9eb274d5d44f811f8c9f0f0bfb3f242ee0b Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:37:06 -0500
Subject: [PATCH 033/151] Update README.md

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cccdeb34..663ecf2b 100644
--- a/README.md
+++ b/README.md
@@ -38,11 +38,14 @@ Archived content is browseable and managable locally with the CLI commands like
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
 

-Demo | Screenshots | Usage
-

 cli init screenshot
 server snapshot admin screenshot
 server snapshot details page screenshot

+

+. . . . . . . . . . . . . . . . . . . . . . . . . . . .
+

+Demo | Screenshots | Usage
+

 

 

 

From befac97f524e461f43f372cbb745c07f6f2c1f0f Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:37:36 -0500
Subject: [PATCH 034/151] Update README.md

---
 README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 663ecf2b..a4a71fab 100644
--- a/README.md
+++ b/README.md
@@ -37,17 +37,16 @@ Archived content is browseable and managable locally with the CLI commands like
 
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

+


 cli init screenshot
 server snapshot admin screenshot
-server snapshot details page screenshot

+server snapshot details page screenshot
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
 

 Demo | Screenshots | Usage
 

 

-

 
 ### Quickstart
 

From cfe2145184a520fbdbcc8b7a4b6f3dcacf4a8cba Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:45:01 -0500
Subject: [PATCH 035/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a4a71fab..2f07dc35 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ You can use `archivebox schedule` to ingest URLs regularly from your browser boo
 Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha).
 
 

-. . . . . . . . . . . . . . . . . . . . . . . . . . . .
+bookshelf graphic
 


 cli init screenshot
 server snapshot admin screenshot

From 29d139a0d71db952a2ed59930274e45c844e4538 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:45:56 -0500
Subject: [PATCH 036/151] Update README.md

---
 README.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 2f07dc35..a8a6d2a5 100644
--- a/README.md
+++ b/README.md
@@ -36,15 +36,17 @@ You can use `archivebox schedule` to ingest URLs regularly from your browser boo
 Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha).
 
 

+

 bookshelf graphic
 


+Demo | Screenshots | Usage
+

+. . . . . . . . . . . . . . . . . . . . . . . . . . . .
+

 cli init screenshot
 server snapshot admin screenshot
 server snapshot details page screenshot
 

-. . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

-Demo | Screenshots | Usage
 

 

 

From b93a006ebbf8c2d71d94652cf4990a3f568d7c3b Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 15:57:31 -0500
Subject: [PATCH 037/151] Update README.md

---
 README.md | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index a8a6d2a5..7d3f0937 100644
--- a/README.md
+++ b/README.md
@@ -38,16 +38,17 @@ Archived content is browseable and managable locally with the CLI commands like
 

 

 bookshelf graphic
-


-Demo | Screenshots | Usage
+

+Demo | Screenshots | Usage
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

+


 cli init screenshot
 server snapshot admin screenshot
 server snapshot details page screenshot
 

 

+grass
 

 
 ### Quickstart
@@ -226,13 +227,20 @@ archivebox help  # to see more options
 
 

 
+

+

+grass
+

  
 ---
  
 

-
+
+
+
+

+. . . . . . . . . . . . . . . . . . . . . . . . . . . .
 

-
 DEMO: archivebox.zervice.io/  
 For more information, see the full Quickstart guide, Usage, and Configuration docs.
 


From 7f0629097bdb0a43a230dba3e392478ee67a994b Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:02:00 -0500
Subject: [PATCH 038/151] Update README.md

---
 README.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 7d3f0937..243e17df 100644
--- a/README.md
+++ b/README.md
@@ -235,14 +235,15 @@ archivebox help  # to see more options
 ---
  
 

-
-
-
+
+
+
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
 

-DEMO: archivebox.zervice.io/  
-For more information, see the full Quickstart guide, Usage, and Configuration docs.
+DEMO: https://archivebox.zervice.io

+Quickstart | Usage | Configuration
+

 

 
 ---

From 38bee553644c380663ffa2decd84145186218150 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:02:21 -0500
Subject: [PATCH 039/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 243e17df..345cbfdc 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ Archived content is browseable and managable locally with the CLI commands like
 
 

 

-bookshelf graphic
+bookshelf graphicbookshelf graphic
 

 Demo | Screenshots | Usage
 


From 615458fa376c67e45b09d304ac9aa68eadea8f71 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:04:55 -0500
Subject: [PATCH 040/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 345cbfdc..ab5ccdc1 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ Archived content is browseable and managable locally with the CLI commands like
 

 

 bookshelf graphicbookshelf graphic
-

+


 Demo | Screenshots | Usage
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .

From c49d85c8e36cff5414567c39415045e4c7bbefdc Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:06:39 -0500
Subject: [PATCH 041/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ab5ccdc1..5e1b451f 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ Archived content is browseable and managable locally with the CLI commands like
 
 

 

-bookshelf graphicbookshelf graphic
+bookshelf graphiclogobookshelf graphic
 


 Demo | Screenshots | Usage
 


From 580ea0ecf80a02cfe1897da5e4bde4ae646f7cf5 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:07:24 -0500
Subject: [PATCH 042/151] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 5e1b451f..7fa83d1d 100644
--- a/README.md
+++ b/README.md
@@ -36,8 +36,8 @@ You can use `archivebox schedule` to ingest URLs regularly from your browser boo
 Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha).
 
 

-

-bookshelf graphiclogobookshelf graphic
+


+bookshelf graphic   logo   bookshelf graphic
 


 Demo | Screenshots | Usage
 


From 8fb556ea3755c146d9017386882067f303739e6f Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:10:39 -0500
Subject: [PATCH 043/151] Update README.md

---
 README.md | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 7fa83d1d..83f8b98b 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ Archived content is browseable and managable locally with the CLI commands like
 server snapshot details page screenshot
 

 

-grass
+grass
 

 
 ### Quickstart
@@ -69,7 +69,7 @@ archivebox oneshot --extract=title,favicon,media 'https://www.youtube.com/watch?
 archivebox help   # to see more options
 ```
 
-*(click to expand the sections below for full setup instructions)*
+*(click to expand the ► sections below for full setup instructions)*
 
 

 Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box)
@@ -229,24 +229,8 @@ archivebox help  # to see more options
 
 

 

-grass
+grass
 

- 
----
- 
-

-
-
-
-

-. . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

-DEMO: https://archivebox.zervice.io

-Quickstart | Usage | Configuration
-

-

-
----
 
 
 # Overview
@@ -259,6 +243,18 @@ The [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) is cons
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer). You can also self-host your archivebox server on a public domain to provide archive.org-style public access to your site snapshots.
 
+

+
+
+
+

+. . . . . . . . . . . . . . . . . . . . . . . . . . . .
+

+DEMO: https://archivebox.zervice.io

+Quickstart | Usage | Configuration
+

+

+
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally

From 665d5c2155014e974c457c3c71b4e258b295bdce Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:24:56 -0500
Subject: [PATCH 044/151] Update README.md

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 83f8b98b..e66e8caf 100644
--- a/README.md
+++ b/README.md
@@ -284,7 +284,7 @@ archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 
 -  TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file)
 -  [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive))
--  [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), Delicious, [Reddit Saved Posts](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), OneTab, [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
+-  [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved Posts](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -318,6 +318,10 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
 
 It does everything out-of-the-box by default, but you can disable or tweak [individual archive methods](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) via environment variables or config file.
 
+

+dependencies graphic
+

+
 ## Dependencies
 
 You don't need to install all the dependencies, ArchiveBox will automatically enable the relevant modules based on whatever you have available, but it's recommended to use the official [Docker image](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) with everything preinstalled.

From 22bf08def8413823a5b4687c1363e30c0fdf5338 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:25:34 -0500
Subject: [PATCH 045/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e66e8caf..2e276541 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,7 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
 It does everything out-of-the-box by default, but you can disable or tweak [individual archive methods](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) via environment variables or config file.
 
 

-dependencies graphic
+dependencies graphic
 

 
 ## Dependencies

From be6b4b3066d830fc528a7fd2f2ce815d74e93894 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:28:53 -0500
Subject: [PATCH 046/151] Update README.md

---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index 2e276541..4e3a4be9 100644
--- a/README.md
+++ b/README.md
@@ -290,6 +290,10 @@ See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usa
 
 It also includes a built-in scheduled import feature with `archivebox schedule` and browser bookmarklet, so you can pull in URLs from RSS feeds, websites, or the filesystem regularly/on-demand.
 
+

+dependencies graphic
+

+
 ## Output formats
 
 All of ArchiveBox's state (including the index, snapshot data, and config file) is stored in a single folder called the "ArchiveBox data folder". All `archivebox` CLI commands must be run from inside this folder, and you first create it by running `archivebox init`.
@@ -318,6 +322,8 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
 
 It does everything out-of-the-box by default, but you can disable or tweak [individual archive methods](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) via environment variables or config file.
 
+---
+
 

 dependencies graphic
 

@@ -365,6 +371,9 @@ archivebox add 'https://example.com#2020-10-24'
 archivebox add 'https://example.com#2020-10-25'
 ```
 
+

+lego graphic
+

 
 ## Screenshots
 

From 163b8b01ce0dc021da34d23d31cfc5e5a98af696 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:31:10 -0500
Subject: [PATCH 047/151] Update README.md

---
 README.md | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 4e3a4be9..7c726554 100644
--- a/README.md
+++ b/README.md
@@ -268,6 +268,12 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 - Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released)
 - Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), [expand threads](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
 
+---
+
+

+lego
+

+
 ## Input formats
 
 ArchiveBox supports many input formats for URLs, including Pocket & Pinboard exports, Browser bookmarks, Browser history, plain text, HTML, markdown, and more!
@@ -290,10 +296,6 @@ See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usa
 
 It also includes a built-in scheduled import feature with `archivebox schedule` and browser bookmarklet, so you can pull in URLs from RSS feeds, websites, or the filesystem regularly/on-demand.
 
-

-dependencies graphic
-

-
 ## Output formats
 
 All of ArchiveBox's state (including the index, snapshot data, and config file) is stored in a single folder called the "ArchiveBox data folder". All `archivebox` CLI commands must be run from inside this folder, and you first create it by running `archivebox init`.
@@ -322,12 +324,12 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
 
 It does everything out-of-the-box by default, but you can disable or tweak [individual archive methods](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) via environment variables or config file.
 
----
-
 

-dependencies graphic
+lego graphic
 

 
+---
+
 ## Dependencies
 
 You don't need to install all the dependencies, ArchiveBox will automatically enable the relevant modules based on whatever you have available, but it's recommended to use the official [Docker image](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) with everything preinstalled.
@@ -371,10 +373,14 @@ archivebox add 'https://example.com#2020-10-24'
 archivebox add 'https://example.com#2020-10-25'
 ```
 
+---
+
 

-lego graphic
+dependencies graphic
 

 
+---
+
 ## Screenshots
 
 


From ba9cff6b8d05985e0d960e83947a0b688c5c2150 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:37:14 -0500
Subject: [PATCH 048/151] Update README.md

---
 README.md | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 7c726554..57eced53 100644
--- a/README.md
+++ b/README.md
@@ -373,17 +373,13 @@ archivebox add 'https://example.com#2020-10-24'
 archivebox add 'https://example.com#2020-10-25'
 ```
 
----
-
-

-dependencies graphic
-

 
 ---
 
 ## Screenshots
 
 

+
 
@@ -419,14 +415,12 @@ archivebox add 'https://example.com#2020-10-25'
 

 
 

 

 
-
 ---
 
 

-
+paisley graphic
 

 
----
 
 # Background & Motivation
 

From ff3195f70e44ae869c24e2add7cc9a49ce5583de Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:42:11 -0500
Subject: [PATCH 049/151] Update README.md

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 57eced53..41d5ada1 100644
--- a/README.md
+++ b/README.md
@@ -482,6 +482,10 @@ Whether you want to learn which organizations are the big players in the web arc
 
 ---
 
+

+documentation graphicdocumentation graphic
+

+
 # Documentation
 
 

From 071d6bb3244a32d5e642ed8da359eb2115264373 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:44:36 -0500
Subject: [PATCH 050/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 41d5ada1..dcdef464 100644
--- a/README.md
+++ b/README.md
@@ -483,7 +483,7 @@ Whether you want to learn which organizations are the big players in the web arc
 ---
 
 

-documentation graphicdocumentation graphic
+documentation graphic
 

 
 # Documentation

From 8f44cda350f3632e39a94d1bfb3cdf0c1fce4976 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:46:25 -0500
Subject: [PATCH 051/151] Update README.md

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index dcdef464..4b02b690 100644
--- a/README.md
+++ b/README.md
@@ -443,6 +443,11 @@ ArchiveBox archives the sites in **several different formats** beyond what publi
 
 All the archived links are stored by date bookmarked in `./archive/`, and everything is indexed nicely with JSON & HTML files. The intent is for all the content to be viewable with common software in 50 - 100 years without needing to run ArchiveBox in a VM.
 
+

+

+dependencies graphic
+

+
 ## Comparison to Other Projects
 
 ▶ **Check out our [community page](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) for an index of web archiving initiatives and projects.**

From 4a7981213ba17ad32b1a1980e521050539545095 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 16:50:49 -0500
Subject: [PATCH 052/151] Update README.md

---
 README.md | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 4b02b690..4a38f972 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ Archived content is browseable and managable locally with the CLI commands like
 server snapshot details page screenshot
 

 

-grass
+grassgrass
 

 
 ### Quickstart
@@ -229,7 +229,7 @@ archivebox help  # to see more options
 
 

 

-grass
+grassgrass
 

 
 
@@ -417,9 +417,7 @@ archivebox add 'https://example.com#2020-10-25'
 
 ---
 
-

-paisley graphic
-

+paisley graphic
 
 
 # Background & Motivation
@@ -443,10 +441,9 @@ ArchiveBox archives the sites in **several different formats** beyond what publi
 
 All the archived links are stored by date bookmarked in `./archive/`, and everything is indexed nicely with JSON & HTML files. The intent is for all the content to be viewable with common software in 50 - 100 years without needing to run ArchiveBox in a VM.
 
-

-

-dependencies graphic
-

+---
+
+dependencies graphic
 
 ## Comparison to Other Projects
 
@@ -487,9 +484,7 @@ Whether you want to learn which organizations are the big players in the web arc
 
 ---
 
-

-documentation graphic
-

+documentation graphic
 
 # Documentation
 

From 8fc8f7217fb9ee683e96e84e18c0b740d6be60a1 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 17:03:34 -0500
Subject: [PATCH 053/151] Update README.md

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index 4a38f972..a00dbc4e 100644
--- a/README.md
+++ b/README.md
@@ -414,6 +414,7 @@ archivebox add 'https://example.com#2020-10-25'
 
 
 

+

 
 ---
 
@@ -482,6 +483,8 @@ Whether you want to learn which organizations are the big players in the web arc
 - Learn why archiving the internet is important by reading the "[On the Importance of Web Archiving](https://parameters.ssrc.org/2018/09/on-the-importance-of-web-archiving/)" blog post.
 - Or reach out to me for questions and comments via [@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) or [@theSquashSH](https://twitter.com/thesquashSH) on Twitter.
 
+

+
 ---
 
 documentation graphic
@@ -523,8 +526,12 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 - [Background & Motivation](https://github.com/ArchiveBox/ArchiveBox#background--motivation)
 - [Web Archiving Community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 
+

+
 ---
 
+development
+
 # ArchiveBox Development
 
 All contributions to ArchiveBox are welcomed! Check our [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) for things to work on, and please open an issue to discuss your proposed implementation before working on things! Otherwise we may have to close your PR if it doesn't align with our roadmap.

From f92e184043f535bd9d6b017e49fcaaf1a934acb4 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 17:05:05 -0500
Subject: [PATCH 054/151] Update README.md

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index a00dbc4e..791db6a3 100644
--- a/README.md
+++ b/README.md
@@ -442,10 +442,6 @@ ArchiveBox archives the sites in **several different formats** beyond what publi
 
 All the archived links are stored by date bookmarked in `./archive/`, and everything is indexed nicely with JSON & HTML files. The intent is for all the content to be viewable with common software in 50 - 100 years without needing to run ArchiveBox in a VM.
 
----
-
-dependencies graphic
-
 ## Comparison to Other Projects
 
 ▶ **Check out our [community page](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) for an index of web archiving initiatives and projects.**
@@ -464,6 +460,10 @@ Unlike crawler software that starts from a seed URL and works outwards, or publi
 
 Because ArchiveBox is designed to ingest a firehose of browser history and bookmark feeds to a local disk, it can be much more disk-space intensive than a centralized service like the Internet Archive or Archive.today. However, as storage space gets cheaper and compression improves, you should be able to use it continuously over the years without having to delete anything. In my experience, ArchiveBox uses about 5gb per 1000 articles, but your milage may vary depending on which options you have enabled and what types of sites you're archiving. By default, it archives everything in as many formats as possible, meaning it takes more space than a using a single method, but more content is accurately replayable over extended periods of time. Storage requirements can be reduced by using a compressed/deduplicated filesystem like ZFS/BTRFS, or by setting `SAVE_MEDIA=False` to skip audio & video files.
 
+

+dependencies graphic
+

+
 ## Learn more
 
 Whether you want to learn which organizations are the big players in the web archiving space, want to find a specific open-source tool for your web archiving need, or just want to see where archivists hang out online, our Community Wiki page serves as an index of the broader web archiving community. Check it out to learn about some of the coolest web archiving projects and communities on the web!

From 92c42dede6ca1e4aa9cb602ab0e2436ecccbb28f Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 17:19:01 -0500
Subject: [PATCH 055/151] Update README.md

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 791db6a3..0f99b36c 100644
--- a/README.md
+++ b/README.md
@@ -328,6 +328,8 @@ It does everything out-of-the-box by default, but you can disable or tweak [indi
 lego graphic
 

 
+

+
 ---
 
 ## Dependencies
@@ -338,6 +340,12 @@ If you so choose, you can also install ArchiveBox and its dependencies directly
 
 ArchiveBox is written in Python 3 so it requires `python3` and `pip3` available on your system. It also uses a set of optional, but highly recommended external dependencies for archiving sites: `wget` (for plain HTML, static files, and WARC saving), `chromium` (for screenshots, PDFs, JS execution, and more), `youtube-dl` (for audio and video), `git` (for cloning git repos), and `nodejs` (for readability and singlefile), and more.
 
+

+
+---
+
+security graphic
+
 ## Caveats
 
 If you're importing URLs containing secret slugs or pages with private content (e.g Google Docs, CodiMD notepads, etc), you may want to disable some of the extractor modules to avoid leaking private URLs to 3rd party APIs during the archiving process.

From aef871fe643205bffca8ec3f1953498fc3c1b0c2 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 17:19:56 -0500
Subject: [PATCH 056/151] Update README.md

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 0f99b36c..09ace153 100644
--- a/README.md
+++ b/README.md
@@ -381,9 +381,12 @@ archivebox add 'https://example.com#2020-10-24'
 archivebox add 'https://example.com#2020-10-25'
 ```
 
+

 
 ---
 
+

+
 ## Screenshots
 
 


From 5f69198f387d1d00c97c69c41ba0387fad636759 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 18:34:11 -0500
Subject: [PATCH 057/151] Update README.md

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 09ace153..e57c0d88 100644
--- a/README.md
+++ b/README.md
@@ -429,6 +429,8 @@ archivebox add 'https://example.com#2020-10-25'
 
 ---
 
+

+
 paisley graphic
 
 

From 02bdb3bdeb615f39c4e336117bda8c0992fdf73a Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 18:42:10 -0500
Subject: [PATCH 058/151] fix DATABASE_NAME posixpath

---
 archivebox/cli/archivebox_schedule.py | 1 +
 archivebox/core/settings.py           | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py
index ec5e9146..568b25b9 100644
--- a/archivebox/cli/archivebox_schedule.py
+++ b/archivebox/cli/archivebox_schedule.py
@@ -42,6 +42,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--depth', # '-d',
         type=int,
+        choices=[0, 1],
         default=0,
         help='Depth to archive to [0] or 1, see "add" command help for more info.',
     )
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index e8ed6b16..bfc0cdc3 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -101,7 +101,7 @@ TEMPLATES = [
 ################################################################################
 
 DATABASE_FILE = Path(OUTPUT_DIR) / SQL_INDEX_FILENAME
-DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", DATABASE_FILE)
+DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(DATABASE_FILE))
 
 DATABASES = {
     'default': {

From 72e2c7b95da85759c9029ee99260866f2e62889d Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 18:44:28 -0500
Subject: [PATCH 059/151] use relative imports for util

---
 archivebox/core/admin.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index 832bea38..e00e988c 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -11,13 +11,14 @@ from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
 from django import forms
 
+from .util import htmldecode, urldecode, ansi_to_html
+
 from core.models import Snapshot, Tag
 from core.forms import AddLinkForm, TagField
 
 from core.mixins import SearchResultsAdminMixin
 
 from index.html import snapshot_icons
-from util import htmldecode, urldecode, ansi_to_html
 from logging_util import printable_filesize
 from main import add, remove
 from config import OUTPUT_DIR

From a07ed3989e3bfd7cc4f4247669ea9addeaa594b7 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 19:02:31 -0500
Subject: [PATCH 060/151] fix import path

---
 archivebox/core/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index e00e988c..f641b177 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -11,7 +11,7 @@ from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
 from django import forms
 
-from .util import htmldecode, urldecode, ansi_to_html
+from ..util import htmldecode, urldecode, ansi_to_html
 
 from core.models import Snapshot, Tag
 from core.forms import AddLinkForm, TagField

From 6c35b12fb9374be9740bf6062f75de3a1d0ced30 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 19:07:21 -0500
Subject: [PATCH 061/151] remove assets folder

---
 assets/css/style.scss | 50 -------------------------------------------
 1 file changed, 50 deletions(-)
 delete mode 100644 assets/css/style.scss

diff --git a/assets/css/style.scss b/assets/css/style.scss
deleted file mode 100644
index 44657267..00000000
--- a/assets/css/style.scss
+++ /dev/null
@@ -1,50 +0,0 @@
----
----
-
-@import "{{ site.theme }}";
-
-div.shell {
-    width: 80%;
-    max-width: 1300px;
-    min-width: 300px;
-}
-
-span.banner-fix {
-    width: 80%;
-    max-width: 1300px;
-    min-width: 300px;
-}
-
-header h1 {
-    background-color: #aa1f55;
-    padding-bottom: 15px;
-    font-weight: 200px;
-}
-header h2 {
-    background-color: #aa1f55;
-    font-family: 'Open Sans';
-}
-
-#main_content div[align=center] h1 {
-    display: none;
-}
-#main_content img {
-    box-shadow: 4px 4px 4px rgba(0,0,0,0.1);
-    border-radius: 8px;
-    border: 0px;
-    vertical-align: top;
-}
-#main_content em img {
-    display: block;
-    margin-top: -83px;
-    padding: 0px;
-    margin-bottom: 20px;
-}
-
-#main_content img[alt=comparison] {
-    margin: 25px;
-}
-
-#forkme_banner {
-    opacity: 0.1;
-}

From ef7711ffa0f8c3a8db73811953160a49102bff04 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 19:13:53 -0500
Subject: [PATCH 062/151] fix cookies file arg is path

---
 .github/workflows/pip.yml     | 2 +-
 archivebox/extractors/wget.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml
index 36153189..8d8e3f91 100644
--- a/.github/workflows/pip.yml
+++ b/.github/workflows/pip.yml
@@ -1,4 +1,4 @@
-name: Build pip package
+name: Build Pip package
 
 on:
   workflow_dispatch:
diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index b7adbea0..ec252123 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -66,7 +66,7 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
         *(['--warc-file={}'.format(str(warc_path))] if SAVE_WARC else []),
         *(['--page-requisites'] if SAVE_WGET_REQUISITES else []),
         *(['--user-agent={}'.format(WGET_USER_AGENT)] if WGET_USER_AGENT else []),
-        *(['--load-cookies', COOKIES_FILE] if COOKIES_FILE else []),
+        *(['--load-cookies', str(COOKIES_FILE)] if COOKIES_FILE else []),
         *(['--compression=auto'] if WGET_AUTO_COMPRESSION else []),
         *([] if SAVE_WARC else ['--timestamping']),
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),

From 9163615d75f60b594001c79def6379e26e5af77b Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 19:34:06 -0500
Subject: [PATCH 063/151] Update README.md

---
 README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e57c0d88..af65f4a8 100644
--- a/README.md
+++ b/README.md
@@ -17,11 +17,13 @@
 
 
 
-
 
 
 
-
+

+
+Language grade: Python
+Language grade: JavaScript
 
 

 


From 0b9b4c1a2cf31909a1896ff1f3b6941a0d74ddc0 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 19:36:49 -0500
Subject: [PATCH 064/151] Update README.md

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index af65f4a8..833622b9 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,8 @@
 
 Language grade: Python
 Language grade: JavaScript
+Total alerts
+
 
 

 


From 6ed6e2e45dda9f841c4d2eb4c257d6d0f66bbeee Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 19:45:33 -0500
Subject: [PATCH 065/151] center all banners

---
 README.md | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 833622b9..0f356470 100644
--- a/README.md
+++ b/README.md
@@ -348,7 +348,9 @@ ArchiveBox is written in Python 3 so it requires `python3` and `pip3` available
 
 ---
 
+

 security graphic
+

 
 ## Caveats
 
@@ -435,8 +437,9 @@ archivebox add 'https://example.com#2020-10-25'
 
 

 
+

 paisley graphic
-
+

 
 # Background & Motivation
 
@@ -477,9 +480,10 @@ Unlike crawler software that starts from a seed URL and works outwards, or publi
 
 Because ArchiveBox is designed to ingest a firehose of browser history and bookmark feeds to a local disk, it can be much more disk-space intensive than a centralized service like the Internet Archive or Archive.today. However, as storage space gets cheaper and compression improves, you should be able to use it continuously over the years without having to delete anything. In my experience, ArchiveBox uses about 5gb per 1000 articles, but your milage may vary depending on which options you have enabled and what types of sites you're archiving. By default, it archives everything in as many formats as possible, meaning it takes more space than a using a single method, but more content is accurately replayable over extended periods of time. Storage requirements can be reduced by using a compressed/deduplicated filesystem like ZFS/BTRFS, or by setting `SAVE_MEDIA=False` to skip audio & video files.
 
+

 

 dependencies graphic
-

+

 
 ## Learn more
 
@@ -504,7 +508,9 @@ Whether you want to learn which organizations are the big players in the web arc
 
 ---
 
+

 documentation graphic
+

 
 # Documentation
 
@@ -547,7 +553,9 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 
 ---
 
+

 development
+

 
 # ArchiveBox Development
 

From 80738a368370fb42674d1504c7ccc9e3f79c9b44 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 20:09:07 -0500
Subject: [PATCH 066/151] add link to LGTM alerts

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 0f356470..c8c00d48 100644
--- a/README.md
+++ b/README.md
@@ -561,6 +561,8 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 
 All contributions to ArchiveBox are welcomed! Check our [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) for things to work on, and please open an issue to discuss your proposed implementation before working on things! Otherwise we may have to close your PR if it doesn't align with our roadmap.
 
+Low hanging fruit / easy first tickets: Total alerts
+
 ### Setup the dev environment
 
 #### 1. Clone the main code repo (making sure to pull the submodules as well)

From 4761533a807f1eec76f7e2da11d82ea45cd454c1 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 20:19:13 -0500
Subject: [PATCH 067/151] remove overview section

---
 README.md | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index c8c00d48..4da6a501 100644
--- a/README.md
+++ b/README.md
@@ -32,12 +32,15 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
 
-Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, `ArchiveBox.conf` config file, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within.
+Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, `ArchiveBox.conf` config file, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within. If you only want to archive a single site, you can run `archivebox oneshot` to avoid having to create a whole collection.
 
 For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
 You can use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
 
-Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha).
+Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha), directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha). It can be installed on Docker, macOS, and Linux/BSD, and Windows. No matter which install method you choose, they all provide the same CLI, Web UI, and on-disk data format.
+
+You can also self-host your `archivebox server` on a public domain to provide archive.org-style public access to your snapshots.
+At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
 
 

 


@@ -237,16 +240,6 @@ archivebox help  # to see more options
 

 
 
-# Overview
-
-ArchiveBox is a command line tool, self-hostable web-archiving server, and Python library all-in-one. It can be installed on Docker, macOS, and Linux/BSD, and Windows. You can download and install it as a Debian/Ubuntu package, Homebrew package, Python3 package, or a Docker image. No matter which install method you choose, they all provide the same CLI, Web UI, and on-disk data format.
-
-To use ArchiveBox you start by creating a folder for your data to live in (it can be anywhere on your system), and running `archivebox init` inside of it. That will create a sqlite3 index and an `ArchiveBox.conf` file. After that, you can continue to add/export/manage/etc using the CLI `archivebox help`, or you can run the Web UI (recommended). If you only want to archive a single site, you can run `archivebox oneshot` to avoid having to create a whole collection.
-
-The [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) is considered "stable", the ArchiveBox [Python API](https://docs.archivebox.io/en/latest/modules.html) and [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) are "alpha", and the [desktop app](https://github.com/ArchiveBox/desktop) is "alpha".
-
-At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer). You can also self-host your archivebox server on a public domain to provide archive.org-style public access to your site snapshots.
-
 

 
 

From b5cbd35dee10515121c3da0a419dcd2a8e4fb66f Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 20:34:27 -0500
Subject: [PATCH 068/151] Update README.md

---
 README.md | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 4da6a501..a794a2bb 100644
--- a/README.md
+++ b/README.md
@@ -30,16 +30,23 @@
 

 

 
-ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
+ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects. For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
 
-Running `archivebox init` in a folder creates a collection with a self-contained `index.sqlite3` index, `ArchiveBox.conf` config file, and folders for each snapshot under `./archive//`, with human-readable `index.html` and `index.json` files within. If you only want to archive a single site, you can run `archivebox oneshot` to avoid having to create a whole collection.
+**First steps:**
 
-For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
-You can use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
+1. Get ArchiveBox (see Quickstart below)
+2. `archivebox init` in a new empty folder to create a collection
+3. `archivebox add 'https://example.com'` to start adding URLs to snapshot in your collection
+4. `archivebox server` to self-host an admin Web UI with your repository of snapshots (archive.org-style)
 
-Archived content is browseable and managable locally with the CLI commands like `archivebox status` or `archivebox list ...`, via the built-in web UI `archivebox server`, [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha), directly through the filesystem `./archive/` folders, or via the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha) or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha). It can be installed on Docker, macOS, and Linux/BSD, and Windows. No matter which install method you choose, they all provide the same CLI, Web UI, and on-disk data format.
+**Next steps:**
+
+- use `archivebox oneshot` to archive a single URL without starting a whole collection
+- use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
+- use `archivebox status`, `archivebox list ...`, `archivebox version` to see more information about your setup
+- browse `./archive//` and view archived content directly from the filesystem
+- or use the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha)
 
-You can also self-host your `archivebox server` on a public domain to provide archive.org-style public access to your snapshots.
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
 
 

@@ -60,21 +67,11 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 
 ### Quickstart
 
-It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`).
+It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`). There is also an [Electron desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha). No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
 
-```bash
-pip3 install archivebox
-archivebox --version
-# install extras as-needed, or use one of full setup methods below to get everything out-of-the-box
-
-mkdir ~/archivebox && cd ~/archivebox    # this can be anywhere
-archivebox init
-
-archivebox add 'https://example.com'
-archivebox schedule --every=day --depth=1 'https://getpocket.com/users/USERNAME/feed/all'
-archivebox oneshot --extract=title,favicon,media 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
-archivebox help   # to see more options
-```
+1. Install ArchiveBox: `apt/brew/pip3 install archivebox`
+2. Start a collection: `archivebox init`
+3. Start archiving: `archivebox add 'https://example.com'`
 
 *(click to expand the ► sections below for full setup instructions)*
 

From 2a90f58818d38aaf4b89ba94d743785f23eed7e4 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 20:44:17 -0500
Subject: [PATCH 069/151] Update README.md

---
 README.md | 41 +++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index a794a2bb..f491ad08 100644
--- a/README.md
+++ b/README.md
@@ -30,29 +30,38 @@
 

 

 
-ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects. For each URL added with `archivebox add`, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
+ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
 
-**First steps:**
+**How it works:**
 
-1. Get ArchiveBox (see Quickstart below)
-2. `archivebox init` in a new empty folder to create a collection
-3. `archivebox add 'https://example.com'` to start adding URLs to snapshot in your collection
-4. `archivebox server` to self-host an admin Web UI with your repository of snapshots (archive.org-style)
-
-**Next steps:**
-
-- use `archivebox oneshot` to archive a single URL without starting a whole collection
-- use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
-- use `archivebox status`, `archivebox list ...`, `archivebox version` to see more information about your setup
-- browse `./archive//` and view archived content directly from the filesystem
-- or use the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha)
-
-At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
+1. Get ArchiveBox
+   (see Quickstart below)
+2. `archivebox init`
+   Run this in an empty folder to init a collection
+3. `archivebox add 'https://example.com'`
+   Start adding URLs to snapshot in your collection. For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
+4. `archivebox server`
+   Self-host an admin Web UI with your repository of snapshots (archive.org-style).
 
 

 


 bookshelf graphic   logo   bookshelf graphic
 


+

+
+**⚡️ Common tasks:**
+
+- use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
+- use `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive
+- use `archivebox oneshot` archive single URLs without starting a whole collection
+- use `archivebox status`, `archivebox list ...`, `archivebox remove` to manage Snapshots in the archive
+- use `archivebox config`, `archivebox version`, `archivebox help` to administer your ArchiveBox install
+- browse `./archive//` and view archived content directly from the filesystem
+
+At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
+
+

+


 Demo | Screenshots | Usage
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .

From dbdd16d79dc16c140af3b524152b3496a7bbcdd1 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 20:52:21 -0500
Subject: [PATCH 070/151] Update README.md

---
 README.md | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index f491ad08..1116deff 100644
--- a/README.md
+++ b/README.md
@@ -32,16 +32,16 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
 
-**How it works:**
+**🔢 How it works:**
 
-1. Get ArchiveBox
-   (see Quickstart below)
+1. Get ArchiveBox: Docker, Apt, Brew, Pip
+   ([see below](#Quickstart))
 2. `archivebox init`
-   Run this in an empty folder to init a collection
+   Run this in an empty folder to init a collection (or use `archivebox oneshot`).
 3. `archivebox add 'https://example.com'`
-   Start adding URLs to snapshot in your collection. For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
+   Start adding URLs to archive. For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
 4. `archivebox server`
-   Self-host an admin Web UI with your repository of snapshots (archive.org-style).
+   Self-host an admin Web UI with your repository of snapshots (archive.org-style) or browse `./archive//` and view archived content directly from the filesystem.
 
 

 


@@ -49,19 +49,20 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 


 

 
-**⚡️ Common tasks:**
+**⚡️ CLI Usage:**
+
+- run `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
+- run `archivebox config`, `archivebox version`, `archivebox init` to administer your ArchiveBox install
+- run `archivebox status`, `archivebox list`, `archivebox remove` to manage Snapshots in the archive
+- run `archivebox oneshot` archive single URLs without starting a whole collection
+- run `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive
+- run `archivebox manage createsuperuser`, `archivebox server` to manage the web UI
 
-- use `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
-- use `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive
-- use `archivebox oneshot` archive single URLs without starting a whole collection
-- use `archivebox status`, `archivebox list ...`, `archivebox remove` to manage Snapshots in the archive
-- use `archivebox config`, `archivebox version`, `archivebox help` to administer your ArchiveBox install
-- browse `./archive//` and view archived content directly from the filesystem
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
 
 

-


+

 Demo | Screenshots | Usage
 

 . . . . . . . . . . . . . . . . . . . . . . . . . . . .

From dfc1e80330e8ae714a25920236d6fa0cb6405e2d Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 20:57:47 -0500
Subject: [PATCH 071/151] Update README.md

---
 README.md | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 1116deff..1ed00c4a 100644
--- a/README.md
+++ b/README.md
@@ -32,16 +32,15 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
 
-**🔢 How it works:**
+**🔢  How it works:**
 
 1. Get ArchiveBox: Docker, Apt, Brew, Pip
    ([see below](#Quickstart))
-2. `archivebox init`
-   Run this in an empty folder to init a collection (or use `archivebox oneshot`).
-3. `archivebox add 'https://example.com'`
-   Start adding URLs to archive. For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
-4. `archivebox server`
-   Self-host an admin Web UI with your repository of snapshots (archive.org-style) or browse `./archive//` and view archived content directly from the filesystem.
+2. `archivebox init`: Run this in an empty folder
+3. `archivebox add 'https://example.com'`: Start adding URLs to archive.  
+    For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
+4. `archivebox server`: Run the webserver and open the admin UI
+    Or browse `./archive//` and view archived content directly from the filesystem.
 
 

 


@@ -49,15 +48,18 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 


 

 
-**⚡️ CLI Usage:**
+**⚡️  CLI Usage:**
 
-- run `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
-- run `archivebox config`, `archivebox version`, `archivebox init` to administer your ArchiveBox install
-- run `archivebox status`, `archivebox list`, `archivebox remove` to manage Snapshots in the archive
-- run `archivebox oneshot` archive single URLs without starting a whole collection
-- run `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive
-- run `archivebox manage createsuperuser`, `archivebox server` to manage the web UI
+```bash
+archivebox --version
+archivebox help
+```
 
+- `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
+- `archivebox config`, `archivebox version`, `archivebox init`, `archivebox manage createsuperuser` to administer your ArchiveBox install
+- `archivebox status`, `archivebox list`, `archivebox remove` to manage Snapshots in the archive
+- `archivebox oneshot` archive single URLs without starting a whole collection
+- `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
 

From 142e300a32e19bf02f5a717531ae3aaf1eba7085 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:02:14 -0500
Subject: [PATCH 072/151] Update README.md

---
 README.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 1ed00c4a..1b207e86 100644
--- a/README.md
+++ b/README.md
@@ -38,9 +38,9 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
    ([see below](#Quickstart))
 2. `archivebox init`: Run this in an empty folder
 3. `archivebox add 'https://example.com'`: Start adding URLs to archive.  
-    For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
-4. `archivebox server`: Run the webserver and open the admin UI
-    Or browse `./archive//` and view archived content directly from the filesystem.
+    For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
+4. `archivebox server`: Run the webserver and open the admin UI  
+    Or browse `./archive//` and view archived content directly from the filesystem.
 
 

 


@@ -51,12 +51,13 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 **⚡️  CLI Usage:**
 
 ```bash
+# archivebox [subcommand] [--args]
 archivebox --version
 archivebox help
 ```
 
-- `archivebox schedule` to ingest URLs regularly from your browser boorkmarks/history, a service like Pocket/Pinboard, RSS feeds, or [and more...](#input-formats)
-- `archivebox config`, `archivebox version`, `archivebox init`, `archivebox manage createsuperuser` to administer your ArchiveBox install
+- `archivebox schedule` to pull in fresh URLs in regularly from [boorkmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats)
+- `archivebox init`, `archivebox version`, `archivebox config`, `archivebox manage` to administer your ArchiveBox install
 - `archivebox status`, `archivebox list`, `archivebox remove` to manage Snapshots in the archive
 - `archivebox oneshot` archive single URLs without starting a whole collection
 - `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive

From f796ec22e2c31643b60dadebeb10288d4811fb05 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:04:56 -0500
Subject: [PATCH 073/151] Update README.md

---
 README.md | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 1b207e86..29d22204 100644
--- a/README.md
+++ b/README.md
@@ -34,13 +34,16 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 
 **🔢  How it works:**
 
-1. Get ArchiveBox: Docker, Apt, Brew, Pip
-   ([see below](#Quickstart))
-2. `archivebox init`: Run this in an empty folder
+First Get ArchiveBox via Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
+```bash
+apt/brew/pip3 install archivebox
+```
+
+1. `archivebox init`: Run this in an empty folder
 3. `archivebox add 'https://example.com'`: Start adding URLs to archive.  
-    For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats)
 4. `archivebox server`: Run the webserver and open the admin UI  
-    Or browse `./archive//` and view archived content directly from the filesystem.
+
+For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats). Open the web UI at http://127.0.0.1:8000 to manage your collection, or browse `./archive//` and view archived content directly from the filesystem.
 
 

 



From 464a6b4eb3ef42accc3eabd7f3c54b329316293b Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:09:00 -0500
Subject: [PATCH 074/151] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 29d22204..df3943ca 100644
--- a/README.md
+++ b/README.md
@@ -59,11 +59,11 @@ archivebox --version
 archivebox help
 ```
 
+- `archivebox init/version/status/config/manage` to administer your collection
+- `archivebox add/remove/update/list` to manage Snapshots in the archive
 - `archivebox schedule` to pull in fresh URLs in regularly from [boorkmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats)
-- `archivebox init`, `archivebox version`, `archivebox config`, `archivebox manage` to administer your ArchiveBox install
-- `archivebox status`, `archivebox list`, `archivebox remove` to manage Snapshots in the archive
 - `archivebox oneshot` archive single URLs without starting a whole collection
-- `archivebox shell`, the `index.sqlite3`, [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha) to interact with your archive
+- `archivebox shell` call the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or browse the `index.sqlite3` DB
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
 

From 8c3c52e1a1e5877b36c28e0666e332d0800a913e Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:14:06 -0500
Subject: [PATCH 075/151] Update README.md

---
 README.md | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index df3943ca..0d6f99da 100644
--- a/README.md
+++ b/README.md
@@ -83,11 +83,9 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 
 ### Quickstart
 
-It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`). There is also an [Electron desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha). No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
+It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`). There is also an [Electron desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha).
 
-1. Install ArchiveBox: `apt/brew/pip3 install archivebox`
-2. Start a collection: `archivebox init`
-3. Start archiving: `archivebox add 'https://example.com'`
+No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
 
 *(click to expand the ► sections below for full setup instructions)*
 
@@ -251,15 +249,15 @@ archivebox help  # to see more options
 

 grassgrass
 

-
+

 
 

 
 
 
-

+


 . . . . . . . . . . . . . . . . . . . . . . . . . . . .
-

+


 DEMO: https://archivebox.zervice.io

 Quickstart | Usage | Configuration
 

@@ -270,13 +268,13 @@ archivebox help  # to see more options
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up for anything, stores all data locally
 - [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies) 
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
-- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): media w/ youtube-dl, articles w/ readability, code w/ git, [and more...](#output-formats)
+- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (youtube-dl), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC
 - [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#submit_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released)
-- Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. to block ads, [scroll pages](https://github.com/ArchiveBox/ArchiveBox/issues/80), [close modals](https://github.com/ArchiveBox/ArchiveBox/issues/175), [expand threads](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
+- Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hiding](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expander](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
 
 ---
 

From 554719e91276148a4b59952ce474d2cbe467b66f Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:15:00 -0500
Subject: [PATCH 076/151] Update README.md

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 0d6f99da..8982a5e7 100644
--- a/README.md
+++ b/README.md
@@ -87,6 +87,14 @@ It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (wi
 
 No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
 
+
+
+1. Install ArchiveBox: `apt/brew/pip3 install archivebox`
+2. Start a collection: `archivebox init`
+3. Start archiving: `archivebox add 'https://example.com'`
+
+
+
 *(click to expand the ► sections below for full setup instructions)*
 
 


From 1bf8367fd54e36fcda2d3b7f54a4e7746f89d7d7 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:21:08 -0500
Subject: [PATCH 077/151] Update README.md

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 8982a5e7..0b4aea2a 100644
--- a/README.md
+++ b/README.md
@@ -30,9 +30,9 @@
 

 

 
-ArchiveBox is a powerful self-hosted internet archiving solution written in Python 3. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on the configuration and the content it detects.
+ArchiveBox is a powerful self-hosted internet archiving solution written in Python. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on setup and content within.
 
-**🔢  How it works:**
+#### 🔢  Quickstart
 
 First Get ArchiveBox via Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
 ```bash
@@ -51,7 +51,7 @@ For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrom
 


 

 
-**⚡️  CLI Usage:**
+#### ⚡️  CLI Usage
 
 ```bash
 # archivebox [subcommand] [--args]
@@ -81,9 +81,9 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 grassgrass
 

 
-### Quickstart
+### Install
 
-It works on Linux/BSD (Intel and ARM CPUs with `docker`/`apt`/`pip3`), macOS (with `docker`/`brew`/`pip3`), and Windows (beta with `docker`/`pip3`). There is also an [Electron desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha).
+ArchiveBox should work on most systems: Linux/BSD (Intel & ARM CPUs) w/ `docker`/`apt`/`pip3`, macOS w/ `docker`/`brew`/`pip3`, and Windows w/ `docker`/`pip3` (beta). There is also a cross-platform [Electron desktop app](https://github.com/ArchiveBox/electron-archivebox) (in alpha).
 
 No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
 

From 218ab8aa625cd19e1992c84ad4e715fc18c612ba Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:26:24 -0500
Subject: [PATCH 078/151] Update README.md

---
 README.md | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 0b4aea2a..ea897f9b 100644
--- a/README.md
+++ b/README.md
@@ -83,17 +83,11 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 
 ### Install
 
-ArchiveBox should work on most systems: Linux/BSD (Intel & ARM CPUs) w/ `docker`/`apt`/`pip3`, macOS w/ `docker`/`brew`/`pip3`, and Windows w/ `docker`/`pip3` (beta). There is also a cross-platform [Electron desktop app](https://github.com/ArchiveBox/electron-archivebox) (in alpha).
+**Supported Systems:** (x86/ARM 32bit & 64bit)
 
-No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
-
-
-
-1. Install ArchiveBox: `apt/brew/pip3 install archivebox`
-2. Start a collection: `archivebox init`
-3. Start archiving: `archivebox add 'https://example.com'`
-
-
+ - Linux/BSD w/ `docker`/`apt`/`pip3`/`brew`
+ - macOS w/ `docker`/`brew`/`pip3`
+ - Windows w/ `docker`/`pip3` (beta)
 
 *(click to expand the ► sections below for full setup instructions)*
 
@@ -253,6 +247,16 @@ archivebox help  # to see more options
 
 

 
+No matter which install method you choose, they all roughly follow this 3-step process and all provide the same CLI, Web UI, and on-disk data format.
+
+
+
+1. Install ArchiveBox: `apt/brew/pip3 install archivebox`
+2. Start a collection: `archivebox init`
+3. Start archiving: `archivebox add 'https://example.com'`
+
+
+
 

 

 grassgrass

From fbdd3fff0bcc86e25975d4dcc4d80c5b8d95b205 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:29:29 -0500
Subject: [PATCH 079/151] Update README.md

---
 README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index ea897f9b..15e08615 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on setup and content within.
 
-#### 🔢  Quickstart
+#### 🔢  Intro
 
 First Get ArchiveBox via Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
 ```bash
@@ -43,7 +43,8 @@ apt/brew/pip3 install archivebox
 3. `archivebox add 'https://example.com'`: Start adding URLs to archive.  
 4. `archivebox server`: Run the webserver and open the admin UI  
 
-For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats). Open the web UI at http://127.0.0.1:8000 to manage your collection, or browse `./archive//` and view archived content directly from the filesystem.
+For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).  
+Open the web UI at http://127.0.0.1:8000 to manage your collection, or browse `./archive//` and view archived content directly from the filesystem.
 
 

 


@@ -81,9 +82,9 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 grassgrass
 

 
-### Install
+### Quickstart
 
-**Supported Systems:** (x86/ARM 32bit & 64bit)
+**Supported Systems:** (x86/ARM 32bit/64bit)
 
  - Linux/BSD w/ `docker`/`apt`/`pip3`/`brew`
  - macOS w/ `docker`/`brew`/`pip3`

From 03f389b6a12f6858ace0caca079aee65e4f7f346 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:34:23 -0500
Subject: [PATCH 080/151] Update README.md

---
 README.md | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 15e08615..222a3d6a 100644
--- a/README.md
+++ b/README.md
@@ -32,19 +32,27 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on setup and content within.
 
-#### 🔢  Intro
+#### 🔢  Overview
 
 First Get ArchiveBox via Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
+
 ```bash
 apt/brew/pip3 install archivebox
 ```
 
-1. `archivebox init`: Run this in an empty folder
-3. `archivebox add 'https://example.com'`: Start adding URLs to archive.  
-4. `archivebox server`: Run the webserver and open the admin UI  
+Then use the `archivebox` CLI to set up your archive and start the web UI.
 
-For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).  
-Open the web UI at http://127.0.0.1:8000 to manage your collection, or browse `./archive//` and view archived content directly from the filesystem.
+```bash
+archivebox init                         # run this in an empty folder
+archivebox add 'https://example.com'    # start adding URLs to archive
+```
+
+For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).
+
+```bash
+archivebox server 0.0.0.0:8000          # run the admin UI webserver  
+ls ./archive/*/index.json               # or browse via the filesystem
+```
 
 

 



From 444edc22b72b6bb4844ce07b2eec325fefa3a0ac Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:38:19 -0500
Subject: [PATCH 081/151] Update README.md

---
 README.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 222a3d6a..67b89577 100644
--- a/README.md
+++ b/README.md
@@ -34,23 +34,26 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 
 #### 🔢  Overview
 
-First Get ArchiveBox via Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
+Get ArchiveBox via [Docker Compose (recommended)](#Quickstart), Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
 
 ```bash
 apt/brew/pip3 install archivebox
 ```
 
-Then use the `archivebox` CLI to set up your archive and start the web UI.
+Then use the `archivebox` CLI to set up your archive and add URLs.
 
 ```bash
 archivebox init                         # run this in an empty folder
 archivebox add 'https://example.com'    # start adding URLs to archive
+archivebox schedule --every=day --depth=1 https://example.com/rss.xml
 ```
 
 For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).
 
 ```bash
-archivebox server 0.0.0.0:8000          # run the admin UI webserver  
+archivebox server 0.0.0.0:8000          # run the admin UI webserver
+open http://127.0.0.1:8000/admin/
+
 ls ./archive/*/index.json               # or browse via the filesystem
 ```
 

From f931fb30d408c0cdfbffb2d3c9f99896a519dedf Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:39:44 -0500
Subject: [PATCH 082/151] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 67b89577..d3e9d9e2 100644
--- a/README.md
+++ b/README.md
@@ -93,14 +93,14 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 grassgrass
 

 
-### Quickstart
-
 **Supported Systems:** (x86/ARM 32bit/64bit)
 
  - Linux/BSD w/ `docker`/`apt`/`pip3`/`brew`
  - macOS w/ `docker`/`brew`/`pip3`
  - Windows w/ `docker`/`pip3` (beta)
 
+### Quickstart
+
 *(click to expand the ► sections below for full setup instructions)*
 
 


From 6805a4688e538c7f9ee903a2303826bd63b1c83e Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:43:02 -0500
Subject: [PATCH 083/151] Update README.md

---
 README.md | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index d3e9d9e2..888721fd 100644
--- a/README.md
+++ b/README.md
@@ -93,15 +93,13 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 grassgrass
 

 
-**Supported Systems:** (x86/ARM 32bit/64bit)
-
- - Linux/BSD w/ `docker`/`apt`/`pip3`/`brew`
- - macOS w/ `docker`/`brew`/`pip3`
- - Windows w/ `docker`/`pip3` (beta)
 
 ### Quickstart
 
-*(click to expand the ► sections below for full setup instructions)*
+**Supported Systems:** Linux/BSD, macOS, Windows (x86/ARM 32bit/64bit)  
+**Distributions:** `docker`/`apt`/`brew`/`pip3`/`npm` (in order of completeness)
+
+*(click to expand your preferred ► distribution below for full setup instructions)*
 
 

 Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box)

From 4547dc477628f882274a62583975820831d2f183 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:50:33 -0500
Subject: [PATCH 084/151] condense outputs

---
 README.md | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 888721fd..49ea9d04 100644
--- a/README.md
+++ b/README.md
@@ -337,17 +337,15 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
 ```
 
 - **Index:** `index.html` & `index.json` HTML and JSON index files containing metadata and details
-- **Title:** `title` title of the site
-- **Favicon:** `favicon.ico` favicon of the site
-- **Headers:** `headers.json` Any HTTP headers the site returns are saved in a json file
-- **SingleFile:** `singlefile.html` HTML snapshot rendered with headless Chrome using SingleFile
-- **WGET Clone:** `example.com/page-name.html` wget clone of the site, with .html appended if not present
-- **WARC:** `warc/.gz` gzipped WARC of all the resources fetched while archiving
-- **PDF:** `output.pdf` Printed PDF of site using headless chrome
-- **Screenshot:** `screenshot.png` 1440x900 screenshot of site using headless chrome
-- **DOM Dump:** `output.html` DOM Dump of the HTML after rendering using headless chrome
-- **Readability:** `article.html/json` Article text extraction using Readability
-- **URL to Archive.org:** `archive.org.txt` A link to the saved site on archive.org
+- **Title**, **Favicon**, **Headers** Response headers, site favicon, and parsed site title
+- **Wget Clone:** `example.com/page-name.html` wget clone of the site with  `warc/.gz`
+- Chrome Headless
+  - **SingleFile:** `singlefile.html` HTML snapshot rendered with headless Chrome using SingleFile
+  - **PDF:** `output.pdf` Printed PDF of site using headless chrome
+  - **Screenshot:** `screenshot.png` 1440x900 screenshot of site using headless chrome
+  - **DOM Dump:** `output.html` DOM Dump of the HTML after rendering using headless chrome
+  - **Readability:** `article.html/json` Article text extraction using Readability
+- **Archive.org Permalink:** `archive.org.txt` A link to the saved site on archive.org
 - **Audio & Video:** `media/` all audio/video files + playlists, including subtitles & metadata with youtube-dl
 - **Source Code:** `git/` clone of any repository found on github, bitbucket, or gitlab links
 - _More coming soon! See the [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap)..._

From c867443112ac1462bae36819b012debfbf674ed4 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:52:14 -0500
Subject: [PATCH 085/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 49ea9d04..34574e15 100644
--- a/README.md
+++ b/README.md
@@ -364,7 +364,7 @@ It does everything out-of-the-box by default, but you can disable or tweak [indi
 
 You don't need to install all the dependencies, ArchiveBox will automatically enable the relevant modules based on whatever you have available, but it's recommended to use the official [Docker image](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) with everything preinstalled.
 
-If you so choose, you can also install ArchiveBox and its dependencies directly on any Linux or macOS systems using the [automated setup script](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart) or the [system package manager](https://github.com/ArchiveBox/ArchiveBox/wiki/Install).
+If you so choose, you can also install ArchiveBox and its dependencies directly on any Linux or macOS systems using the [system package manager](https://github.com/ArchiveBox/ArchiveBox/wiki/Install) or by running the [automated setup script](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart).
 
 ArchiveBox is written in Python 3 so it requires `python3` and `pip3` available on your system. It also uses a set of optional, but highly recommended external dependencies for archiving sites: `wget` (for plain HTML, static files, and WARC saving), `chromium` (for screenshots, PDFs, JS execution, and more), `youtube-dl` (for audio and video), `git` (for cloning git repos), and `nodejs` (for readability and singlefile), and more.
 

From b8d89b14e164244bfa537b81045099a9f3bb2c66 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:54:06 -0500
Subject: [PATCH 086/151] Update README.md

---
 README.md | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 34574e15..faff4573 100644
--- a/README.md
+++ b/README.md
@@ -38,11 +38,7 @@ Get ArchiveBox via [Docker Compose (recommended)](#Quickstart), Docker, Apt, Bre
 
 ```bash
 apt/brew/pip3 install archivebox
-```
 
-Then use the `archivebox` CLI to set up your archive and add URLs.
-
-```bash
 archivebox init                         # run this in an empty folder
 archivebox add 'https://example.com'    # start adding URLs to archive
 archivebox schedule --every=day --depth=1 https://example.com/rss.xml
@@ -51,9 +47,7 @@ archivebox schedule --every=day --depth=1 https://example.com/rss.xml
 For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).
 
 ```bash
-archivebox server 0.0.0.0:8000          # run the admin UI webserver
-open http://127.0.0.1:8000/admin/
-
+archivebox server 0.0.0.0:8000          # open http://127.0.0.1:8000/
 ls ./archive/*/index.json               # or browse via the filesystem
 ```
 
@@ -360,6 +354,8 @@ It does everything out-of-the-box by default, but you can disable or tweak [indi
 
 ---
 
+

+
 ## Dependencies
 
 You don't need to install all the dependencies, ArchiveBox will automatically enable the relevant modules based on whatever you have available, but it's recommended to use the official [Docker image](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker) with everything preinstalled.

From e02974f85d2686f62223bf43067bd8108ab80721 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:57:30 -0500
Subject: [PATCH 087/151] Update README.md

---
 README.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/README.md b/README.md
index faff4573..96899e6c 100644
--- a/README.md
+++ b/README.md
@@ -32,9 +32,7 @@
 
 ArchiveBox is a powerful self-hosted internet archiving solution written in Python. You feed it URLs of pages you want to archive, and it saves them to disk in a variety of formats depending on setup and content within.
 
-#### 🔢  Overview
-
-Get ArchiveBox via [Docker Compose (recommended)](#Quickstart), Docker, Apt, Brew, Pip, etc. ([see below](#Quickstart)).
+**🔢  Run ArchiveBox via [Docker Compose (recommended)](#Quickstart), Docker, Apt, Brew, or Pip ([see below](#Quickstart)).**
 
 ```bash
 apt/brew/pip3 install archivebox

From f4d59449af35d54ad341f6ca67d575006a5f9bc3 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 21:59:07 -0500
Subject: [PATCH 088/151] Update README.md

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 96899e6c..f0f13678 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,8 @@ archivebox server 0.0.0.0:8000          # open http://127.0.0.1:8000/
 ls ./archive/*/index.json               # or browse via the filesystem
 ```
 
+You can then manage your snapshots via the filesystem, CLI, web UI, or Python API (alpha).
+
 

 


 bookshelf graphic   logo   bookshelf graphic

From cfa9559d81bdc84474f1e39b830571af43a00d33 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 22:01:19 -0500
Subject: [PATCH 089/151] Update README.md

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index f0f13678..f6ec3f02 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,8 @@ ls ./archive/*/index.json               # or browse via the filesystem
 
 You can then manage your snapshots via the filesystem, CLI, web UI, or Python API (alpha).
 
+At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
+
 

 


 bookshelf graphic   logo   bookshelf graphic
@@ -71,8 +73,6 @@ archivebox help
 - `archivebox oneshot` archive single URLs without starting a whole collection
 - `archivebox shell` call the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or browse the `index.sqlite3` DB
 
-At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
-
 

 

 Demo | Screenshots | Usage
@@ -90,10 +90,10 @@ At the end of the day, the goal is to sleep soundly knowing that the part of the
 
 ### Quickstart
 
-**Supported Systems:** Linux/BSD, macOS, Windows (x86/ARM 32bit/64bit)  
-**Distributions:** `docker`/`apt`/`brew`/`pip3`/`npm` (in order of completeness)
+**🖥  Supported Systems:** Linux/BSD, macOS, Windows (x86/ARM 32bit/64bit)  
+**📦  Distributions:** `docker`/`apt`/`brew`/`pip3`/`npm` (in order of completeness)
 
-*(click to expand your preferred ► distribution below for full setup instructions)*
+*(click to expand your preferred **► `distribution`** below for full setup instructions)*
 
 

 Get ArchiveBox with docker-compose on any platform (recommended, everything included out-of-the-box)

From dfb726b9ce93b05cdd615ed076f1aa3b61c4c67f Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 22:04:32 -0500
Subject: [PATCH 090/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f6ec3f02..b99ac9e1 100644
--- a/README.md
+++ b/README.md
@@ -90,7 +90,7 @@ archivebox help
 
 ### Quickstart
 
-**🖥  Supported Systems:** Linux/BSD, macOS, Windows (x86/ARM 32bit/64bit)  
+**🖥  Supported OSs:** Linux/BSD, macOS, Windows      **🎮  Supported CPUs:** x86/ARM 32bit/64bit
 **📦  Distributions:** `docker`/`apt`/`brew`/`pip3`/`npm` (in order of completeness)
 
 *(click to expand your preferred **► `distribution`** below for full setup instructions)*

From fc2023b6de769578570293284e981ece821f74b3 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 22:06:50 -0500
Subject: [PATCH 091/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b99ac9e1..671c61ec 100644
--- a/README.md
+++ b/README.md
@@ -90,7 +90,7 @@ archivebox help
 
 ### Quickstart
 
-**🖥  Supported OSs:** Linux/BSD, macOS, Windows      **🎮  Supported CPUs:** x86/ARM 32bit/64bit
+**🖥  Supported OSs:** Linux/BSD, macOS, Windows      **🎮  CPU Architectures:** x86, amd64, arm7, arm8 (raspi >=3)
 **📦  Distributions:** `docker`/`apt`/`brew`/`pip3`/`npm` (in order of completeness)
 
 *(click to expand your preferred **► `distribution`** below for full setup instructions)*

From 553c3ca2192e8ce3b42b018ba3bba10e13494748 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 20 Jan 2021 22:09:09 -0500
Subject: [PATCH 092/151] Update README.md

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 671c61ec..d219da7a 100644
--- a/README.md
+++ b/README.md
@@ -292,6 +292,8 @@ No matter which install method you choose, they all roughly follow this 3-step p
 - Planned: support for archiving [content requiring a login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (working, but ill-advised until some pending fixes are released)
 - Planned: support for running [JS scripts during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51), e.g. adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hiding](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expander](https://github.com/ArchiveBox/ArchiveBox/issues/345), etc.
 
+

+
 ---
 
 

@@ -314,7 +316,7 @@ archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 
 -  TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file)
 -  [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive))
--  [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved Posts](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
+-  [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -581,7 +583,8 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 
 All contributions to ArchiveBox are welcomed! Check our [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) for things to work on, and please open an issue to discuss your proposed implementation before working on things! Otherwise we may have to close your PR if it doesn't align with our roadmap.
 
-Low hanging fruit / easy first tickets: Total alerts
+Low hanging fruit / easy first tickets:

+Total alerts
 
 ### Setup the dev environment
 

From 5420903102981a49b97c90e61a2f6959fd49614b Mon Sep 17 00:00:00 2001
From: Dan Arnfield 
Date: Thu, 21 Jan 2021 15:45:11 -0600
Subject: [PATCH 093/151] Refactor `should_save_extractor` methods to accept
 `overwrite` parameter

---
 archivebox/extractors/__init__.py    |  2 +-
 archivebox/extractors/archive_org.py |  6 +++---
 archivebox/extractors/dom.py         | 10 +++++-----
 archivebox/extractors/favicon.py     |  8 ++++----
 archivebox/extractors/git.py         |  6 +++---
 archivebox/extractors/headers.py     |  9 +++++----
 archivebox/extractors/media.py       |  7 +++----
 archivebox/extractors/mercury.py     | 10 ++++++----
 archivebox/extractors/pdf.py         |  8 ++++----
 archivebox/extractors/readability.py | 10 ++++++----
 archivebox/extractors/screenshot.py  |  8 ++++----
 archivebox/extractors/singlefile.py  | 10 ++++++----
 archivebox/extractors/title.py       |  8 ++++----
 archivebox/extractors/wget.py        |  4 ++--
 14 files changed, 56 insertions(+), 50 deletions(-)

diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py
index a4acef0b..15968097 100644
--- a/archivebox/extractors/__init__.py
+++ b/archivebox/extractors/__init__.py
@@ -102,7 +102,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                 if method_name not in link.history:
                     link.history[method_name] = []
 
-                if should_run(link, out_dir) or overwrite:
+                if should_run(link, out_dir, overwrite):
                     log_archive_method_started(method_name)
 
                     result = method_function(link=link, out_dir=out_dir)
diff --git a/archivebox/extractors/archive_org.py b/archivebox/extractors/archive_org.py
index f5598d6f..1f382190 100644
--- a/archivebox/extractors/archive_org.py
+++ b/archivebox/extractors/archive_org.py
@@ -25,12 +25,12 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
+def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
 
-    if (out_dir / "archive.org.txt").exists():
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'archive.org.txt').exists():
         # if open(path, 'r').read().strip() != 'None':
         return False
 
diff --git a/archivebox/extractors/dom.py b/archivebox/extractors/dom.py
index babbe71c..ec2df073 100644
--- a/archivebox/extractors/dom.py
+++ b/archivebox/extractors/dom.py
@@ -20,16 +20,16 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_dom(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
+def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
-    
-    if (out_dir / 'output.html').exists():
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'output.html').exists():
         return False
 
     return SAVE_DOM
-    
+
 @enforce_types
 def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print HTML of site to file using chrome --dump-html"""
diff --git a/archivebox/extractors/favicon.py b/archivebox/extractors/favicon.py
index 5e7c1fb0..3a4aeea7 100644
--- a/archivebox/extractors/favicon.py
+++ b/archivebox/extractors/favicon.py
@@ -20,13 +20,13 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_favicon(link: Link, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or link.link_dir
-    if (Path(out_dir) / 'favicon.ico').exists():
+def should_save_favicon(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'favicon.ico').exists():
         return False
 
     return SAVE_FAVICON
-    
+
 @enforce_types
 def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download site favicon from google's favicon api"""
diff --git a/archivebox/extractors/git.py b/archivebox/extractors/git.py
index fd20d4b6..efef37c2 100644
--- a/archivebox/extractors/git.py
+++ b/archivebox/extractors/git.py
@@ -28,12 +28,12 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_git(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or link.link_dir
+def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
 
-    if (out_dir / "git").exists():
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'git').exists():
         return False
 
     is_clonable_url = (
diff --git a/archivebox/extractors/headers.py b/archivebox/extractors/headers.py
index 4e69dec1..91dcb8e3 100644
--- a/archivebox/extractors/headers.py
+++ b/archivebox/extractors/headers.py
@@ -22,11 +22,12 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 @enforce_types
-def should_save_headers(link: Link, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or link.link_dir
+def should_save_headers(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'headers.json').exists():
+        return False
 
-    output = Path(out_dir or link.link_dir) / 'headers.json'
-    return not output.exists() and SAVE_HEADERS
+    return SAVE_HEADERS
 
 
 @enforce_types
diff --git a/archivebox/extractors/media.py b/archivebox/extractors/media.py
index 3792fd2a..1c0a21ba 100644
--- a/archivebox/extractors/media.py
+++ b/archivebox/extractors/media.py
@@ -21,13 +21,12 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_media(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or link.link_dir
-
+def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
 
-    if (out_dir / "media").exists():
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'media').exists():
         return False
 
     return SAVE_MEDIA
diff --git a/archivebox/extractors/mercury.py b/archivebox/extractors/mercury.py
index 07c02420..d9e32c0a 100644
--- a/archivebox/extractors/mercury.py
+++ b/archivebox/extractors/mercury.py
@@ -37,13 +37,15 @@ def ShellError(cmd: List[str], result: CompletedProcess, lines: int=20) -> Archi
 
 
 @enforce_types
-def should_save_mercury(link: Link, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or link.link_dir
+def should_save_mercury(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
 
-    output = Path(out_dir or link.link_dir) / 'mercury'
-    return SAVE_MERCURY and MERCURY_VERSION and (not output.exists())
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'mercury').exists():
+        return False
+
+    return SAVE_MERCURY
 
 
 @enforce_types
diff --git a/archivebox/extractors/pdf.py b/archivebox/extractors/pdf.py
index 1b0201e3..7138206c 100644
--- a/archivebox/extractors/pdf.py
+++ b/archivebox/extractors/pdf.py
@@ -19,12 +19,12 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_pdf(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
+def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
-    
-    if (out_dir / "output.pdf").exists():
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'output.pdf').exists():
         return False
 
     return SAVE_PDF
diff --git a/archivebox/extractors/readability.py b/archivebox/extractors/readability.py
index 9da620b4..6e48cd9a 100644
--- a/archivebox/extractors/readability.py
+++ b/archivebox/extractors/readability.py
@@ -46,13 +46,15 @@ def get_html(link: Link, path: Path) -> str:
         return document
 
 @enforce_types
-def should_save_readability(link: Link, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or link.link_dir
+def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
 
-    output = Path(out_dir or link.link_dir) / 'readability'
-    return SAVE_READABILITY and READABILITY_VERSION and (not output.exists())
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'readability').exists():
+        return False
+
+    return SAVE_READABILITY
 
 
 @enforce_types
diff --git a/archivebox/extractors/screenshot.py b/archivebox/extractors/screenshot.py
index 325584eb..cc748bf6 100644
--- a/archivebox/extractors/screenshot.py
+++ b/archivebox/extractors/screenshot.py
@@ -20,12 +20,12 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_screenshot(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
+def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
-    
-    if (out_dir / "screenshot.png").exists():
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'screenshot.png').exists():
         return False
 
     return SAVE_SCREENSHOT
diff --git a/archivebox/extractors/singlefile.py b/archivebox/extractors/singlefile.py
index 2e5c3896..8d9b36be 100644
--- a/archivebox/extractors/singlefile.py
+++ b/archivebox/extractors/singlefile.py
@@ -23,13 +23,15 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_singlefile(link: Link, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
+def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     if is_static_file(link.url):
         return False
 
-    output = out_dir / 'singlefile.html'
-    return SAVE_SINGLEFILE and SINGLEFILE_VERSION and (not output.exists())
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'singlefile.html').exists():
+        return False
+
+    return SAVE_SINGLEFILE
 
 
 @enforce_types
diff --git a/archivebox/extractors/title.py b/archivebox/extractors/title.py
index 28cb128f..816c0484 100644
--- a/archivebox/extractors/title.py
+++ b/archivebox/extractors/title.py
@@ -61,12 +61,12 @@ class TitleParser(HTMLParser):
 
 
 @enforce_types
-def should_save_title(link: Link, out_dir: Optional[str]=None) -> bool:
-    # if link already has valid title, skip it
-    if link.title and not link.title.lower().startswith('http'):
+def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
-    if is_static_file(link.url):
+    # if link already has valid title, skip it
+    if not overwrite and link.title and not link.title.lower().startswith('http'):
         return False
 
     return SAVE_TITLE
diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index ec252123..ee8744b2 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -36,10 +36,10 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_wget(link: Link, out_dir: Optional[Path]=None) -> bool:
+def should_save_wget(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
     output_path = wget_output_path(link)
     out_dir = out_dir or Path(link.link_dir)
-    if output_path and (out_dir / output_path).exists():
+    if not overwrite and output_path and (out_dir / output_path).exists():
         return False
 
     return SAVE_WGET

From 7e8d02a96951f59dd2d828e940cdf60ef9d4a383 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 10:26:39 -0500
Subject: [PATCH 094/151] Update README.md

---
 README.md | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index d219da7a..a8eb5a65 100644
--- a/README.md
+++ b/README.md
@@ -37,19 +37,21 @@ ArchiveBox is a powerful self-hosted internet archiving solution written in Pyth
 ```bash
 apt/brew/pip3 install archivebox
 
-archivebox init                         # run this in an empty folder
-archivebox add 'https://example.com'    # start adding URLs to archive
-archivebox schedule --every=day --depth=1 https://example.com/rss.xml
+archivebox init                       # run this in an empty folder
+archivebox add 'https://example.com'  # start adding URLs to archive
+curl https://example.com/rss.xml | archivebox add  # or add via stdin
+archivebox schedule --every=day https://example.com/rss.xml
 ```
 
 For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).
 
 ```bash
-archivebox server 0.0.0.0:8000          # open http://127.0.0.1:8000/
-ls ./archive/*/index.json               # or browse via the filesystem
+archivebox server 0.0.0.0:8000         # use the web UI http://127.0.0.1:8000/
+archivebox list 'https://example.com'  # use the CLI commands (--help for more)
+ls ./archive/*/index.json              # or browse directly via the filesystem
 ```
 
-You can then manage your snapshots via the filesystem, CLI, web UI, or Python API (alpha).
+You can then manage your snapshots via the [filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout), [CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [Web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [SQLite DB](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py) (`./index.sqlite3`), [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha).
 
 At the end of the day, the goal is to sleep soundly knowing that the part of the internet you care about will be automatically preserved in multiple, durable long-term formats that will be accessible for decades (or longer).
 
@@ -71,7 +73,7 @@ archivebox help
 - `archivebox add/remove/update/list` to manage Snapshots in the archive
 - `archivebox schedule` to pull in fresh URLs in regularly from [boorkmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats)
 - `archivebox oneshot` archive single URLs without starting a whole collection
-- `archivebox shell` call the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha), or browse the `index.sqlite3` DB
+- `archivebox shell` open a REPL to use the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
 
 

 


From 15c7fa2f31ed91717d8dcd85642b47176f3222b3 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 10:29:35 -0500
Subject: [PATCH 095/151] fix wiki anchor link capitalization

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index a8eb5a65..68a077a4 100644
--- a/README.md
+++ b/README.md
@@ -516,13 +516,13 @@ Whether you want to learn which organizations are the big players in the web arc
 
 
 - [Community Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
-  - [The Master Lists](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#The-Master-Lists)  
+  - [The Master Lists](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#the-master-lists)  
     _Community-maintained indexes of archiving tools and institutions._
-  - [Web Archiving Software](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)  
+  - [Web Archiving Software](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)  
     _Open source tools and projects in the internet archiving space._
-  - [Reading List](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Reading-List)  
+  - [Reading List](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#reading-list)  
     _Articles, posts, and blogs relevant to ArchiveBox and web archiving in general._
-  - [Communities](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Communities)  
+  - [Communities](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#communities)  
     _A collection of the most active internet archiving communities and initiatives._
 - Check out the ArchiveBox [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) and [Changelog](https://github.com/ArchiveBox/ArchiveBox/wiki/Changelog)
 - Learn why archiving the internet is important by reading the "[On the Importance of Web Archiving](https://parameters.ssrc.org/2018/09/on-the-importance-of-web-archiving/)" blog post.

From bd05f0e02900fc57c4b04aa79ea651d85d521ba9 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 10:31:37 -0500
Subject: [PATCH 096/151] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 68a077a4..aca20bcc 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 

 
-
ArchiveBox
The open-source self-hosted web archive.

+
ArchiveBox
Open-source self-hosted web archiving.

 
 ▶️ Quickstart |
 Demo |

From 7ab62d0b2c57393067f4d7ca0991f9c64dc7f2bf Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 13:03:57 -0500
Subject: [PATCH 097/151] update setup script

---
 bin/setup.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/bin/setup.sh b/bin/setup.sh
index e87c9571..e8136687 100755
--- a/bin/setup.sh
+++ b/bin/setup.sh
@@ -8,12 +8,13 @@ echo ""
 echo "    This is a helper script which installs the ArchiveBox dependencies on your system using homebrew/aptitude."
 echo "    You may be prompted for a password in order to install the following:"
 echo ""
-echo "        - git"
 echo "        - python3, python3-pip, python3-distutils"
 echo "        - curl"
 echo "        - wget"
+echo "        - git"
 echo "        - youtube-dl"
 echo "        - chromium-browser  (skip this if Chrome/Chromium is already installed)"
+echo "        - nodejs            (used for singlefile, readability, mercury, and more)"
 echo ""
 echo "    If you'd rather install these manually, you can find documentation here:"
 echo "        https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
@@ -26,9 +27,9 @@ echo ""
 # On Linux:
 if which apt-get > /dev/null; then
     echo "[+] Updating apt repos..."
-    apt update -q
+    sudo apt update -qq
     echo "[+] Installing python3, wget, curl..."
-    apt install git python3 python3-pip python3-distutils wget curl youtube-dl
+    sudo apt install git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm
 
     if which google-chrome; then
         echo "[i] You already have google-chrome installed, if you would like to download chromium instead (they work pretty much the same), follow the Manual Setup instructions"
@@ -41,13 +42,13 @@ if which apt-get > /dev/null; then
         chromium --version
     else
         echo "[+] Installing chromium..."
-        apt install chromium
+        sudo apt install chromium
     fi
 
 # On Mac:
 elif which brew > /dev/null; then   # 🐍 eye of newt
     echo "[+] Installing python3, wget, curl  (ignore 'already installed' warnings)..."
-    brew install git wget curl youtube-dl
+    brew install git wget curl youtube-dl node
     if which python3; then
         if python3 -c 'import sys; raise SystemExit(sys.version_info < (3,5,0))'; then
             echo "[√] Using existing $(which python3)..."
@@ -84,6 +85,7 @@ else
 fi
 
 python3 -m pip install --upgrade archivebox
+npm install -g 'git+https://github.com/ArchiveBox/ArchiveBox.git' 
 
 # Check:
 echo ""

From d9366774b004d0261b15223751709c32ad7b5fd9 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 13:37:25 -0500
Subject: [PATCH 098/151] Update setup.sh

---
 bin/setup.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/bin/setup.sh b/bin/setup.sh
index e8136687..47156d6a 100755
--- a/bin/setup.sh
+++ b/bin/setup.sh
@@ -19,9 +19,7 @@ echo ""
 echo "    If you'd rather install these manually, you can find documentation here:"
 echo "        https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
 echo ""
-echo "Press enter to continue with the automatic install, or Ctrl+C to cancel..."
-read
-
+read -p "Press [enter] to continue with the automatic install, or Ctrl+C to cancel..."
 echo ""
 
 # On Linux:
@@ -29,7 +27,7 @@ if which apt-get > /dev/null; then
     echo "[+] Updating apt repos..."
     sudo apt update -qq
     echo "[+] Installing python3, wget, curl..."
-    sudo apt install git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm
+    sudo apt install -y git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm
 
     if which google-chrome; then
         echo "[i] You already have google-chrome installed, if you would like to download chromium instead (they work pretty much the same), follow the Manual Setup instructions"
@@ -42,7 +40,7 @@ if which apt-get > /dev/null; then
         chromium --version
     else
         echo "[+] Installing chromium..."
-        sudo apt install chromium
+        sudo apt install chromium || sudo apt install chromium-browser
     fi
 
 # On Mac:

From b1144f4a819e426c9dff9ac7e21b1d161cebcc1e Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 13:40:14 -0500
Subject: [PATCH 099/151] Update setup.sh

---
 bin/setup.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/bin/setup.sh b/bin/setup.sh
index 47156d6a..20eb86a0 100755
--- a/bin/setup.sh
+++ b/bin/setup.sh
@@ -1,6 +1,5 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # ArchiveBox Setup Script
-# Nick Sweeting 2017 | MIT License
 # https://github.com/ArchiveBox/ArchiveBox
 
 echo "[i] ArchiveBox Setup Script 📦"
@@ -19,7 +18,7 @@ echo ""
 echo "    If you'd rather install these manually, you can find documentation here:"
 echo "        https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
 echo ""
-read -p "Press [enter] to continue with the automatic install, or Ctrl+C to cancel..."
+read -p "Press [enter] to continue with the automatic install, or Ctrl+C to cancel..." REPLY
 echo ""
 
 # On Linux:

From bfe8f4c539aaa9d629db430abcb960ce9851bc56 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 13:41:16 -0500
Subject: [PATCH 100/151] Update setup.sh

---
 bin/setup.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/setup.sh b/bin/setup.sh
index 20eb86a0..2dc2054a 100755
--- a/bin/setup.sh
+++ b/bin/setup.sh
@@ -81,7 +81,10 @@ else
     exit 1
 fi
 
-python3 -m pip install --upgrade archivebox
+npm i -g npm
+pip3 install --upgrade pip setuptools
+
+pip3 install --upgrade archivebox
 npm install -g 'git+https://github.com/ArchiveBox/ArchiveBox.git' 
 
 # Check:

From 5cf7efaade691c72b8cea6ec39befa508259ffe9 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 13:47:26 -0500
Subject: [PATCH 101/151] Update setup.sh

---
 bin/setup.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/setup.sh b/bin/setup.sh
index 2dc2054a..304c96c5 100755
--- a/bin/setup.sh
+++ b/bin/setup.sh
@@ -23,10 +23,12 @@ echo ""
 
 # On Linux:
 if which apt-get > /dev/null; then
-    echo "[+] Updating apt repos..."
-    sudo apt update -qq
+    echo "[+] Adding ArchiveBox apt repo to sources..."
+    sudo apt install software-properties-common
+    sudo add-apt-repository -u ppa:archivebox/archivebox
     echo "[+] Installing python3, wget, curl..."
-    sudo apt install -y git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm
+    sudo apt install -y git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm ripgrep
+    # sudo apt install archivebox
 
     if which google-chrome; then
         echo "[i] You already have google-chrome installed, if you would like to download chromium instead (they work pretty much the same), follow the Manual Setup instructions"
@@ -45,7 +47,7 @@ if which apt-get > /dev/null; then
 # On Mac:
 elif which brew > /dev/null; then   # 🐍 eye of newt
     echo "[+] Installing python3, wget, curl  (ignore 'already installed' warnings)..."
-    brew install git wget curl youtube-dl node
+    brew install git wget curl youtube-dl ripgrep node
     if which python3; then
         if python3 -c 'import sys; raise SystemExit(sys.version_info < (3,5,0))'; then
             echo "[√] Using existing $(which python3)..."

From c6f0b8e6b3be00d2f8ad915624e31175a1bb9573 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 23 Jan 2021 17:10:58 -0500
Subject: [PATCH 102/151] link dev time and money in new issue template

---
 .github/ISSUE_TEMPLATE/feature_request.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 3361571d..5378139f 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -45,6 +45,6 @@ workarounds, or other software you've considered using to fix the problem.
 
 ---
 
- - [ ] I'm willing to contribute dev time / money to fix this issue
+ - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
  - [ ] I like ArchiveBox so far / would recommend it to a friend
  - [ ] I've had a lot of difficulty getting ArchiveBox set up

From 1989275944ae5d4194270f842c7ebddd44c79291 Mon Sep 17 00:00:00 2001
From: Preston Maness 
Date: Sat, 23 Jan 2021 20:32:56 -0600
Subject: [PATCH 103/151] Fix issue #617 by using mark_safe in combination with
 format_html

I have no experience with Django, so all I'm really going off of is this
stackoverflow

https://stackoverflow.com/a/64498319

which cited this bit of Django documentation:

https://docs.djangoproject.com/en/3.1/ref/utils/#django.utils.html.format_html

After using this method, I no longer get the 500 error or KeyError
exception, and can browse the local server and interact with the single
entry in it (the problematic URL in ArchiveBox#617 with curly braces).

Whether this is the "right" method or not, I have no idea. But it is at
least a start.
---
 archivebox/index/html.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index a62e2c7e..6db8435c 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -4,7 +4,7 @@ from datetime import datetime
 from typing import List, Optional, Iterator, Mapping
 from pathlib import Path
 
-from django.utils.html import format_html
+from django.utils.html import format_html, mark_safe
 from collections import defaultdict
 
 from .schema import Link
@@ -161,4 +161,4 @@ def snapshot_icons(snapshot) -> str:
             output += '{} '.format(canon["archive_org_path"], str(exists),
                                                                                         "archive_org", icons.get("archive_org", "?"))
 
-    return format_html(f'{output}')
+    return format_html('{}', mark_safe(output))

From 9764a8ed9bce0e5abc936cacbb5bac0bc1615863 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 22 Jan 2021 14:06:01 -0500
Subject: [PATCH 104/151] check for non html files from wget

---
 archivebox/extractors/wget.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index ee8744b2..15923ac3 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -175,11 +175,22 @@ def wget_output_path(link: Link) -> Optional[str]:
                 if html_files:
                     return str(html_files[0].relative_to(link.link_dir))
 
+                # sometimes wget'd URLs have no ext and return non-html
+                # e.g. /some/example/rss/all -> some RSS XML content)
+                #      /some/other/url.o4g   -> some binary unrecognized ext)
+                # test this with archivebox add --depth=1 https://getpocket.com/users/nikisweeting/feed/all
+                last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
+                for file_present in os.listdir(search_dir):
+                    if file_present == last_part_of_url:
+                        return os.path.join(path_from_link_dir, file_present)
+
         # Move up one directory level
         search_dir = search_dir.parent
 
         if str(search_dir) == link.link_dir:
             break
+
+
     
     search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
     if not search_dir.is_dir():

From ea209fa575ec021edea6482832969a7fd6ec5f7b Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Mon, 25 Jan 2021 18:55:37 -0500
Subject: [PATCH 105/151] make permission chowning on docker start less fancy

---
 bin/docker_entrypoint.sh | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh
index 29fcb646..865a9929 100755
--- a/bin/docker_entrypoint.sh
+++ b/bin/docker_entrypoint.sh
@@ -1,24 +1,25 @@
 #!/usr/bin/env bash
 
-# Autodetect UID,GID of host user based on ownership of files in the data volume
 DATA_DIR="${DATA_DIR:-/data}"
 ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
 
-USID=$(stat --format="%u" "$DATA_DIR")
-GRID=$(stat --format="%g" "$DATA_DIR")
-
-# If user is not root, modify the archivebox user+files to have the same uid,gid
-if [[ "$USID" != 0 && "$GRID" != 0 ]]; then
-    usermod -u "$USID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
-    groupmod -g "$GRID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
-    chown -R "$USID":"$GRID" "/home/$ARCHIVEBOX_USER"
-    chown "$USID":"$GRID" "$DATA_DIR"
-    chown "$USID":"$GRID" "$DATA_DIR/*" > /dev/null 2>&1 || true
+# Set the permissions of the data dir to match the archivebox user
+if [[ -d "$DATA_DIR/archive" ]]; then
+    # check data directory permissions
+    if [[ ! "$(stat -c %u $DATA_DIR/archive)" = "$(id -u archivebox)" ]]; then
+        echo "Change in ownership detected, please be patient while we chown existing files"
+        echo "This could take some time..."
+        chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER -R "$DATA_DIR"
+    fi
+else
+    # create data directory
+    mkdir -p "$DATA_DIR"
+    chown -R $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 fi
+chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 
-# Run commands as the new archivebox user in Docker.
-#   Any files touched will have the same uid & gid
-#   inside Docker and outside on the host machine.
+
+# Drop permissions to run commands as the archivebox user
 if [[ "$1" == /* || "$1" == "echo" || "$1" == "archivebox" ]]; then
     # arg 1 is a binary, execute it verbatim
     # e.g. "archivebox init"

From b647581115b601459962ae66a6898a9b6c483c9b Mon Sep 17 00:00:00 2001
From: Preston Maness 
Date: Mon, 25 Jan 2021 20:47:57 -0600
Subject: [PATCH 106/151] Update archivebox/index/html.py

mark_safe is dangerous, as the URL's filename could have malicious HTML fragments in it.

Co-authored-by: Nick Sweeting 
---
 archivebox/index/html.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 6db8435c..27940cb2 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -161,4 +161,4 @@ def snapshot_icons(snapshot) -> str:
             output += '{} '.format(canon["archive_org_path"], str(exists),
                                                                                         "archive_org", icons.get("archive_org", "?"))
 
-    return format_html('{}', mark_safe(output))
+    return format_html('{}', output)

From 1810426774ebea0d350fe3926278da60680b8d08 Mon Sep 17 00:00:00 2001
From: Preston Maness 
Date: Mon, 25 Jan 2021 21:16:06 -0600
Subject: [PATCH 107/151] Remove now-unused mark_safe import

---
 archivebox/index/html.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 27940cb2..12eab62a 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -4,7 +4,7 @@ from datetime import datetime
 from typing import List, Optional, Iterator, Mapping
 from pathlib import Path
 
-from django.utils.html import format_html, mark_safe
+from django.utils.html import format_html
 from collections import defaultdict
 
 from .schema import Link

From 15e58bd366ccda6ccc02d5a28e44c81b91ad42b6 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 27 Jan 2021 11:27:40 -0500
Subject: [PATCH 108/151] fix using os.path calls on pathlib paths

---
 archivebox/extractors/wget.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index 15923ac3..04886575 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -180,9 +180,9 @@ def wget_output_path(link: Link) -> Optional[str]:
                 #      /some/other/url.o4g   -> some binary unrecognized ext)
                 # test this with archivebox add --depth=1 https://getpocket.com/users/nikisweeting/feed/all
                 last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
-                for file_present in os.listdir(search_dir):
+                for file_present in search_dir.iterdir():
                     if file_present == last_part_of_url:
-                        return os.path.join(path_from_link_dir, file_present)
+                        return search_dir / file_present
 
         # Move up one directory level
         search_dir = search_dir.parent

From 1442e73f2249d98dbc75c932b6c398956333d744 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Wed, 27 Jan 2021 14:50:31 -0500
Subject: [PATCH 109/151] add css file back for public site

---
 assets/css/style.scss | 50 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 assets/css/style.scss

diff --git a/assets/css/style.scss b/assets/css/style.scss
new file mode 100644
index 00000000..44657267
--- /dev/null
+++ b/assets/css/style.scss
@@ -0,0 +1,50 @@
+---
+---
+
+@import "{{ site.theme }}";
+
+div.shell {
+    width: 80%;
+    max-width: 1300px;
+    min-width: 300px;
+}
+
+span.banner-fix {
+    width: 80%;
+    max-width: 1300px;
+    min-width: 300px;
+}
+
+header h1 {
+    background-color: #aa1f55;
+    padding-bottom: 15px;
+    font-weight: 200px;
+}
+header h2 {
+    background-color: #aa1f55;
+    font-family: 'Open Sans';
+}
+
+#main_content div[align=center] h1 {
+    display: none;
+}
+#main_content img {
+    box-shadow: 4px 4px 4px rgba(0,0,0,0.1);
+    border-radius: 8px;
+    border: 0px;
+    vertical-align: top;
+}
+#main_content em img {
+    display: block;
+    margin-top: -83px;
+    padding: 0px;
+    margin-bottom: 20px;
+}
+
+#main_content img[alt=comparison] {
+    margin: 25px;
+}
+
+#forkme_banner {
+    opacity: 0.1;
+}

From db96e7d75b4f1c78e2a385d3453422132a11f2ec Mon Sep 17 00:00:00 2001
From: Dan Arnfield 
Date: Thu, 28 Jan 2021 08:37:15 -0600
Subject: [PATCH 110/151] Set archivebox UID/GID via envvars

---
 bin/docker_entrypoint.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh
index 865a9929..b806c0b2 100755
--- a/bin/docker_entrypoint.sh
+++ b/bin/docker_entrypoint.sh
@@ -3,6 +3,14 @@
 DATA_DIR="${DATA_DIR:-/data}"
 ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
 
+# Set the archivebox user UID & GID
+if [[ -n "$ARCHIVEBOX_UID" && "$ARCHIVEBOX_UID" != 0 ]]; then
+    usermod -u "$ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+fi
+if [[ -n "$ARCHIVEBOX_GID" && "$ARCHIVEBOX_GID" != 0 ]]; then
+    groupmod -g "$ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+fi
+
 # Set the permissions of the data dir to match the archivebox user
 if [[ -d "$DATA_DIR/archive" ]]; then
     # check data directory permissions

From 187ca7e2b5093586c7c2397eac90c2ccc3256b04 Mon Sep 17 00:00:00 2001
From: Dan Arnfield 
Date: Thu, 28 Jan 2021 08:48:21 -0600
Subject: [PATCH 111/151] Change env vars to linuxserver.io convention

---
 bin/docker_entrypoint.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh
index b806c0b2..65a4c1f6 100755
--- a/bin/docker_entrypoint.sh
+++ b/bin/docker_entrypoint.sh
@@ -4,11 +4,11 @@ DATA_DIR="${DATA_DIR:-/data}"
 ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
 
 # Set the archivebox user UID & GID
-if [[ -n "$ARCHIVEBOX_UID" && "$ARCHIVEBOX_UID" != 0 ]]; then
-    usermod -u "$ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+if [[ -n "$PUID" && "$PUID" != 0 ]]; then
+    usermod -u "$PUID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
 fi
-if [[ -n "$ARCHIVEBOX_GID" && "$ARCHIVEBOX_GID" != 0 ]]; then
-    groupmod -g "$ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+if [[ -n "$PGID" && "$PGID" != 0 ]]; then
+    groupmod -g "$PGID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
 fi
 
 # Set the permissions of the data dir to match the archivebox user

From 5adde91a47e8638780e2b24018f573efda19af93 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 20:49:59 -0500
Subject: [PATCH 112/151] add guide for running in debug mode

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index aca20bcc..83da2690 100644
--- a/README.md
+++ b/README.md
@@ -635,6 +635,14 @@ docker run -it --rm -p 8000:8000 \
 See the `./bin/` folder and read the source of the bash scripts within.
 You can also run all these in Docker. For more examples see the Github Actions CI/CD tests that are run: `.github/workflows/*.yaml`.
 
+#### Run in DEBUG mode
+
+```bash
+archivebox config --set DEBUG=True
+# or
+archivebox server --debug ...
+```
+
 #### Run the linters
 
 ```bash

From 39ec77e46c7b88cdf2fcdd8eab768d0037a6cd6a Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 22:27:02 -0500
Subject: [PATCH 113/151] add createsuperuser flag to server command

---
 archivebox/cli/archivebox_server.py | 6 ++++++
 archivebox/main.py                  | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py
index dbacf7e5..a4d96dc9 100644
--- a/archivebox/cli/archivebox_server.py
+++ b/archivebox/cli/archivebox_server.py
@@ -43,6 +43,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Run archivebox init before starting the server',
     )
+    parser.add_argument(
+        '--createsuperuser',
+        action='store_true',
+        help='Run archivebox manage createsuperuser before starting the server',
+    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
@@ -51,6 +56,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         reload=command.reload,
         debug=command.debug,
         init=command.init,
+        createsuperuser=command.createsuperuser,
         out_dir=pwd or OUTPUT_DIR,
     )
 
diff --git a/archivebox/main.py b/archivebox/main.py
index eb8cd6a0..c666f5d6 100644
--- a/archivebox/main.py
+++ b/archivebox/main.py
@@ -1060,6 +1060,7 @@ def server(runserver_args: Optional[List[str]]=None,
            reload: bool=False,
            debug: bool=False,
            init: bool=False,
+           createsuperuser: bool=False,
            out_dir: Path=OUTPUT_DIR) -> None:
     """Run the ArchiveBox HTTP server"""
 
@@ -1068,6 +1069,9 @@ def server(runserver_args: Optional[List[str]]=None,
     if init:
         run_subcommand('init', stdin=None, pwd=out_dir)
 
+    if createsuperuser:
+        run_subcommand('manage', subcommand_args=['createsuperuser'], pwd=out_dir)
+
     # setup config for django runserver
     from . import config
     config.SHOW_PROGRESS = False

From f0040580c82a8f0d3fc280ebac1fb5baf8949efb Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 22:27:17 -0500
Subject: [PATCH 114/151] fix files icons escaping

---
 archivebox/index/html.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 12eab62a..0ba8e7c1 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -4,7 +4,7 @@ from datetime import datetime
 from typing import List, Optional, Iterator, Mapping
 from pathlib import Path
 
-from django.utils.html import format_html
+from django.utils.html import format_html, mark_safe
 from collections import defaultdict
 
 from .schema import Link
@@ -147,12 +147,12 @@ def snapshot_icons(snapshot) -> str:
     for extractor, _ in EXTRACTORS:
         if extractor not in exclude:
             exists = extractor_items[extractor] is not None
-            output += output_template.format(path, canon[f"{extractor}_path"], str(exists),
+            output += format_html(output_template, path, canon[f"{extractor}_path"], str(exists),
                                              extractor, icons.get(extractor, "?"))
         if extractor == "wget":
             # warc isn't technically it's own extractor, so we have to add it after wget
             exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
-            output += output_template.format(exists[0] if exists else '#', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
+            output += format_html(output_template, exists[0] if exists else '#', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
 
         if extractor == "archive_org":
             # The check for archive_org is different, so it has to be handled separately
@@ -161,4 +161,4 @@ def snapshot_icons(snapshot) -> str:
             output += '{} '.format(canon["archive_org_path"], str(exists),
                                                                                         "archive_org", icons.get("archive_org", "?"))
 
-    return format_html('{}', output)
+    return format_html('{}', mark_safe(output))

From 6a8f6992d8af00fba9181677fb5cf07c6876a304 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 22:28:10 -0500
Subject: [PATCH 115/151] reuse admin styling for pubic index and add page

---
 archivebox/themes/default/base.html           | 286 +++---------------
 .../themes/default/core/snapshot_list.html    |  22 +-
 archivebox/themes/default/main_index_row.html |  22 +-
 archivebox/themes/default/static/add.css      |  28 ++
 4 files changed, 104 insertions(+), 254 deletions(-)

diff --git a/archivebox/themes/default/base.html b/archivebox/themes/default/base.html
index a70430ea..48043a3f 100644
--- a/archivebox/themes/default/base.html
+++ b/archivebox/themes/default/base.html
@@ -1,3 +1,4 @@
+{% load admin_urls %}
 {% load static %}
 
 
@@ -7,222 +8,8 @@
     Archived Sites
     
 
-    
+    
+    
     
     
             {% block extra_head %}
@@ -247,38 +34,49 @@
         
     
     
-        

-            

-                

-                    

+        

+            
+                

+                    ➕  Add  / 
+                    Snapshots  / 
+                    Admin  / 
+                    Docs
+                    {% if user.is_authenticated %}
+                           
+                        User
+                        {% firstof user.get_short_name user.get_username %}    
+                        {% if user.has_usable_password %}
+                            Change password /
+                        {% endif %}
+                        Log out
+                    {% endif %}
                 

             

-        

-        {% block body %}
-        {% endblock %}
-        

-        
+            

+                {% block body %}
+                {% endblock %}
+            

+            
+        

     
     
     
diff --git a/archivebox/themes/default/core/snapshot_list.html b/archivebox/themes/default/core/snapshot_list.html
index ce2b2faa..84abee7d 100644
--- a/archivebox/themes/default/core/snapshot_list.html
+++ b/archivebox/themes/default/core/snapshot_list.html
@@ -2,13 +2,21 @@
 {% load static %}
 
 {% block body %}
-    

-    

-        
-        
-        
-    

+    

+        
+    

     
diff --git a/archivebox/themes/default/main_index_row.html b/archivebox/themes/default/main_index_row.html
index 5e21a8c1..7ca14479 100644
--- a/archivebox/themes/default/main_index_row.html
+++ b/archivebox/themes/default/main_index_row.html
@@ -10,13 +10,29 @@
         {% endif %}
             
                 {{link.title|default:'Loading...'}}
-                {% if link.tags_str != None %} {{link.tags_str|default:''}} {% else %} {{ link.tags|default:'' }} {% endif %}
+                
+                    
+                        
+                            {% if link.tags_str != None %}
+                                {{link.tags_str|default:''}}
+                            {% else %}
+                                {{ link.tags|default:'' }}
+                            {% endif %}
+                        
+                    
+                
-
\ No newline at end of file
+
diff --git a/archivebox/themes/default/static/add.css b/archivebox/themes/default/static/add.css
index b128bf4b..875c61bc 100644
--- a/archivebox/themes/default/static/add.css
+++ b/archivebox/themes/default/static/add.css
@@ -1,3 +1,13 @@
+header {
+    font-family: "Roboto","Lucida Grande","DejaVu Sans","Bitstream Vera Sans",Verdana,Arial,sans-serif;
+    font-size: 13px;
+    color: white;
+    height: 30px;
+}
+.header-top {
+  color: white;
+}
+
 .dashboard #content {
   width: 100%;
   margin-right: 0px;
@@ -60,3 +70,21 @@ ul#id_depth {
   box-sizing: border-box;
   animation: spin 2s linear infinite;
 }
+
+
+textarea, select {
+  border-radius: 4px;
+  border: 2px solid #004882;
+  box-shadow:  4px 4px 4px rgba(0,0,0,0.02);
+  width: 100%;
+}
+
+select option:not(:checked) {
+  border: 1px dashed rgba(10,200,20,0.12);
+}
+select option:checked {
+  border: 1px solid green;
+  background-color: green;
+  color: green;
+}
+

From 7d8fe66d439f9f6a05b665ce98ab6a34092ea306 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 22:35:21 -0500
Subject: [PATCH 116/151] consistent tags styling

---
 archivebox/themes/default/main_index.html     |  8 ++++++
 archivebox/themes/default/main_index_row.html | 25 +++++++++----------
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/archivebox/themes/default/main_index.html b/archivebox/themes/default/main_index.html
index 95af1963..85753b31 100644
--- a/archivebox/themes/default/main_index.html
+++ b/archivebox/themes/default/main_index.html
@@ -185,6 +185,14 @@
             .title-col a {
                 color: black;
             }
+            .tags {
+                float: right;
+                border-radius: 5px;
+                background-color: #bfdfff;
+                padding: 2px 5px;
+                margin-left: 4px;
+                margin-top: 1px;
+            }
         
         
         
diff --git a/archivebox/themes/default/main_index_row.html b/archivebox/themes/default/main_index_row.html
index 7ca14479..cfbcbfe8 100644
--- a/archivebox/themes/default/main_index_row.html
+++ b/archivebox/themes/default/main_index_row.html
@@ -8,20 +8,19 @@
         {% else %}
             
         {% endif %}
-            
-                {{link.title|default:'Loading...'}}
-                
-                    
-                        
-                            {% if link.tags_str != None %}
-                                {{link.tags_str|default:''}}
-                            {% else %}
-                                {{ link.tags|default:'' }}
-                            {% endif %}
-                        
-                    
+
+        
+            {{link.title|default:'Loading...'}}
+            {% if link.tags_str %}
+                
+                    {% if link.tags_str != None %}
+                        {{link.tags_str|default:''}}
+                    {% else %}
+                        {{ link.tags|default:'' }}
+                    {% endif %}
                 
-            
+            {% endif %}
+        
     
     
-    
diff --git a/archivebox/themes/default/static/admin.css b/archivebox/themes/default/static/admin.css
index 181c06de..142e1b89 100644
--- a/archivebox/themes/default/static/admin.css
+++ b/archivebox/themes/default/static/admin.css
@@ -224,7 +224,7 @@ body.model-snapshot.change-list #content .object-tools {
   100% { transform: rotate(360deg); }
 }
 
-.tags > a > .tag {
+.tag {
     float: right;
     border-radius: 5px;
     background-color: #bfdfff;
@@ -232,3 +232,8 @@ body.model-snapshot.change-list #content .object-tools {
     margin-left: 4px;
     margin-top: 1px;
 }
+
+.exists-False {
+    opacity: 0.1;
+    filter: grayscale(100%);
+}

From d7df9e58eaa6bd48681196f867217c950fd51b49 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 23:15:05 -0500
Subject: [PATCH 118/151] hide footer on add page

---
 archivebox/core/forms.py                 | 22 ++++++++++++++++++++
 archivebox/themes/default/add_links.html |  2 ++
 archivebox/themes/default/base.html      | 26 +++++++++++++-----------
 3 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py
index 86b29bb7..ed584c68 100644
--- a/archivebox/core/forms.py
+++ b/archivebox/core/forms.py
@@ -22,10 +22,32 @@ class AddLinkForm(forms.Form):
     url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
     depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
     archive_methods = forms.MultipleChoiceField(
+        label="Archive methods (select at least 1, otherwise all will be used by default)",
         required=False,
         widget=forms.SelectMultiple,
         choices=ARCHIVE_METHODS,
     )
+    # TODO: hook these up to the view and put them 
+    # in a collapsible UI section labeled "Advanced"
+    #
+    # exclude_patterns = forms.CharField(
+    #     label="Exclude patterns",
+    #     min_length='1',
+    #     required=False,
+    #     initial=URL_BLACKLIST,
+    # )
+    # timeout = forms.IntegerField(
+    #     initial=TIMEOUT,
+    # )
+    # overwrite = forms.BooleanField(
+    #     label="Overwrite any existing Snapshots",
+    #     initial=False,
+    # )
+    # index_only = forms.BooleanField(
+    #     label="Add URLs to index without Snapshotting",
+    #     initial=False,
+    # )
+
 class TagWidgetMixin:
     def format_value(self, value):
         if value is not None and not isinstance(value, str):
diff --git a/archivebox/themes/default/add_links.html b/archivebox/themes/default/add_links.html
index 0b384f5c..fa8b441f 100644
--- a/archivebox/themes/default/add_links.html
+++ b/archivebox/themes/default/add_links.html
@@ -68,4 +68,6 @@
     
 {% endblock %}
 
+{% block footer %}{% endblock %}
+
 {% block sidebar %}{% endblock %}
diff --git a/archivebox/themes/default/base.html b/archivebox/themes/default/base.html
index 48043a3f..c6eda60f 100644
--- a/archivebox/themes/default/base.html
+++ b/archivebox/themes/default/base.html
@@ -64,18 +64,20 @@
                 {% block body %}
                 {% endblock %}
             
-            
+            {% block footer %}
+                
+            {% endblock %}
         
     
     

From f3ade5f5cc001c4f59b990eff314467a38657e8e Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 29 Jan 2021 00:13:59 -0500
Subject: [PATCH 119/151] document new createsuperuser flag on archivebox
 server

---
 README.md | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 83da2690..7e0363a8 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@ archivebox schedule --every=day https://example.com/rss.xml
 For each URL added, ArchiveBox saves several types of HTML snapshot (wget, Chrome headless, singlefile), a PDF, a screenshot, a WARC archive, any git repositories, images, audio, video, subtitles, article text, [and more...](#output-formats).
 
 ```bash
-archivebox server 0.0.0.0:8000         # use the web UI http://127.0.0.1:8000/
+archivebox server --createsuperuser 0.0.0.0:8000   # use the interactive web UI
 archivebox list 'https://example.com'  # use the CLI commands (--help for more)
 ls ./archive/*/index.json              # or browse directly via the filesystem
 ```
@@ -137,8 +137,7 @@ docker run -v $PWD:/data -it archivebox/archivebox init
 docker run -v $PWD:/data -it archivebox/archivebox --version
 
 # start the webserver and open the UI (optional)
-docker run -v $PWD:/data -it archivebox/archivebox manage createsuperuser
-docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox server 0.0.0.0:8000
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox server --createsuperuser 0.0.0.0:8000
 open http://127.0.0.1:8000
 
 # you can also add links and manage your archive via the CLI:
@@ -167,8 +166,7 @@ archivebox init
 archivebox --version
 
 # start the webserver and open the web UI (optional)
-archivebox manage createsuperuser
-archivebox server 0.0.0.0:8000
+archivebox server --createsuperuser 0.0.0.0:8000
 open http://127.0.0.1:8000
 
 # you can also add URLs and manage the archive via the CLI and filesystem:
@@ -208,8 +206,7 @@ archivebox init
 archivebox --version
 
 # start the webserver and open the web UI (optional)
-archivebox manage createsuperuser
-archivebox server 0.0.0.0:8000
+archivebox server --createsuperuser 0.0.0.0:8000
 open http://127.0.0.1:8000
 
 # you can also add URLs and manage the archive via the CLI and filesystem:
@@ -239,8 +236,7 @@ archivebox --version
 # Install any missing extras like wget/git/chrome/etc. manually as needed
 
 # start the webserver and open the web UI (optional)
-archivebox manage createsuperuser
-archivebox server 0.0.0.0:8000
+archivebox server --createsuperuser 0.0.0.0:8000
 open http://127.0.0.1:8000
 
 # you can also add URLs and manage the archive via the CLI and filesystem:

From 3227f54b525f5a6abb5f9f127651260596d15b3d Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 29 Jan 2021 00:15:15 -0500
Subject: [PATCH 120/151] limit youtubedl download size to 750m and stop
 splitting out audio files

---
 archivebox/config.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/archivebox/config.py b/archivebox/config.py
index 8c05ef26..dc014ed5 100644
--- a/archivebox/config.py
+++ b/archivebox/config.py
@@ -116,16 +116,15 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--write-annotations',
                                                                 '--write-thumbnail',
                                                                 '--no-call-home',
-                                                                '--user-agent',
                                                                 '--all-subs',
-                                                                '--extract-audio',
-                                                                '--keep-video',
+                                                                '--yes-playlist',
+                                                                '--continue',
                                                                 '--ignore-errors',
                                                                 '--geo-bypass',
-                                                                '--audio-format', 'mp3',
-                                                                '--audio-quality', '320K',
-                                                                '--embed-thumbnail',
-                                                                '--add-metadata']},
+                                                                '--add-metadata',
+                                                                '--max-filesize=750m',
+                                                                ]},
+                                                                    
 
         'WGET_ARGS':                {'type': list,  'default': ['--no-verbose',
                                                                 '--adjust-extension',

From f6c3683ab812e21e529f8cd27468c7ffa2a65da5 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 29 Jan 2021 00:15:28 -0500
Subject: [PATCH 121/151] fix snapshot favicon loading spinner height

---
 archivebox/index/html.py                      | 2 +-
 archivebox/themes/default/main_index_row.html | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 91ff83cd..28f25fde 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -161,4 +161,4 @@ def snapshot_icons(snapshot) -> str:
             output += '{} '.format(canon["archive_org_path"], str(exists),
                                                                                         "archive_org", icons.get("archive_org", "?"))
 
-    return format_html('{}', mark_safe(output))
+    return format_html('{}', mark_safe(output))
diff --git a/archivebox/themes/default/main_index_row.html b/archivebox/themes/default/main_index_row.html
index cb821f61..bcc3e112 100644
--- a/archivebox/themes/default/main_index_row.html
+++ b/archivebox/themes/default/main_index_row.html
@@ -6,7 +6,7 @@
         {% if link.is_archived %}
             
         {% else %}
-            
+            
         {% endif %}
 
         
@@ -28,7 +28,7 @@
                 {{link.icons}} {{link.num_outputs}}
             {% else %}
                 📄 
-                    {{link.num_outputs}} 
+                    {{link.num_outputs}} 
                 
             {% endif %}
         

From 8a4edb45e71843b16e5bdb8fe6f1752e5c76b1c0 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 29 Jan 2021 09:08:03 -0500
Subject: [PATCH 122/151] also search url, timestamp, tags on public index

---
 archivebox/core/views.py                          | 3 ++-
 archivebox/themes/default/core/snapshot_list.html | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index b46e364e..810b4740 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -9,6 +9,7 @@ from django.http import HttpResponse
 from django.views import View, static
 from django.views.generic.list import ListView
 from django.views.generic import FormView
+from django.db.models import Q
 from django.contrib.auth.mixins import UserPassesTestMixin
 
 from core.models import Snapshot
@@ -107,7 +108,7 @@ class PublicArchiveView(ListView):
         qs = super().get_queryset(**kwargs) 
         query = self.request.GET.get('q')
         if query:
-            qs = qs.filter(title__icontains=query)
+            qs = qs.filter(Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query))
         for snapshot in qs:
             snapshot.icons = snapshot_icons(snapshot)
         return qs
diff --git a/archivebox/themes/default/core/snapshot_list.html b/archivebox/themes/default/core/snapshot_list.html
index 84abee7d..dd8ebf15 100644
--- a/archivebox/themes/default/core/snapshot_list.html
+++ b/archivebox/themes/default/core/snapshot_list.html
@@ -6,7 +6,7 @@
         
             

                 
-                
+                
                 
                 
Date: Fri, 29 Jan 2021 09:09:23 -0500
Subject: [PATCH 123/151] improve loading snapshots tooltips

---
 archivebox/themes/default/main_index_row.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/themes/default/main_index_row.html b/archivebox/themes/default/main_index_row.html
index bcc3e112..eae60ea9 100644
--- a/archivebox/themes/default/main_index_row.html
+++ b/archivebox/themes/default/main_index_row.html
@@ -9,7 +9,7 @@
             
         {% endif %}
 
-        
+        
             {{link.title|default:'Loading...'}}
             {% if link.tags_str %}
                 

From ff7d2ffa09e65cf36d2c1d26eb5a160cd7320a27 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Fri, 29 Jan 2021 09:18:38 -0500
Subject: [PATCH 124/151] fix version in legacy footer

---
 archivebox/themes/default/main_index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/themes/default/main_index.html b/archivebox/themes/default/main_index.html
index 95af1963..269c9825 100644
--- a/archivebox/themes/default/main_index.html
+++ b/archivebox/themes/default/main_index.html
@@ -243,7 +243,7 @@
             

                 
                     Archive created using ArchiveBox
-                    version v{{VERSION}}   |   
+                    version v{{version}}   |   
                     Download index as JSON
                     


                     {{FOOTER_INFO}}

From 4576b40ccb26ba489cbee86b916150888c6df9b3 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 01:23:09 -0500
Subject: [PATCH 125/151] use action to collect docker tags

---
 .github/workflows/docker.yml | 53 ++++++++++++++----------------------
 1 file changed, 20 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 2a85086a..277061d1 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -1,9 +1,7 @@
 name: Build Docker image
 
 on:
-  push:
-    branches:
-      - master
+  on: workflow_dispatch
   release:
     types:
       - created
@@ -16,12 +14,6 @@ jobs:
   buildx:
     runs-on: ubuntu-latest
     steps:
-      - name: Docker Login
-        uses: docker/login-action@v1
-        with:
-           username: ${{ secrets.DOCKER_USERNAME }}
-           password: ${{ secrets.DOCKER_PASSWORD }}
-      
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -51,28 +43,23 @@ jobs:
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-
-      
-      - name: Prepare tags to publish
-        id: prep
-        run: |
-          # Always publish to latest.
-          TAGS="${{ secrets.DOCKER_USERNAME }}/archivebox:latest,archivebox/archivebox:latest"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION="${GITHUB_REF#refs/tags/}"
-            MINOR=${VERSION%.*}
-            MAJOR=${MINOR%.*}
-            TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$VERSION,archivebox/archivebox:$VERSION"
-            TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$MINOR,archivebox/archivebox:$MINOR"
-            TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$MAJOR,archivebox/archivebox:$MAJOR"
-          else
-            VERSION=$GITHUB_SHA
-            TAGS="$TAGS,${{ secrets.DOCKER_USERNAME }}/archivebox:$VERSION,archivebox/archivebox:$VERSION"
-          fi
 
-          echo ::set-output name=tags::${TAGS}
-        env:
-          GITHUB_REF: ${{ github.ref }}
-          GITHUB_SHA: ${{ github.sha }}
+      - name: Docker Login
+        uses: docker/login-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+           username: ${{ secrets.DOCKER_USERNAME }}
+           password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Collect Docker tags
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: archivebox/archivebox,nikisweeting/archivebox
+          tag-sha: true
+          tag-semver: |
+            {{version}}
+            {{major}}.{{minor}}
       
       - name: Build and push
         id: docker_build
@@ -81,11 +68,11 @@ jobs:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

From c2aaa41c7660fe7424f2c3c64fb50507d7109864 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 01:25:08 -0500
Subject: [PATCH 126/151] fix missing str path

---
 archivebox/extractors/wget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index 04886575..33529e4c 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -182,7 +182,7 @@ def wget_output_path(link: Link) -> Optional[str]:
                 last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
                 for file_present in search_dir.iterdir():
                     if file_present == last_part_of_url:
-                        return search_dir / file_present
+                        return str(search_dir / file_present)
 
         # Move up one directory level
         search_dir = search_dir.parent

From cddbd8f63e7ce6d646e3c9f9a55047a6b1b434a3 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 02:38:44 -0500
Subject: [PATCH 127/151] Update README.md

---
 README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.md b/README.md
index 7e0363a8..19196b4f 100644
--- a/README.md
+++ b/README.md
@@ -639,6 +639,13 @@ archivebox config --set DEBUG=True
 archivebox server --debug ...
 ```
 
+### Build and run a Github branch
+
+```bash
+docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
+docker run -it -v $PWD:/data archivebox:dev ...
+```
+
 #### Run the linters
 
 ```bash
@@ -655,6 +662,7 @@ archivebox server --debug ...
 
 #### Make migrations or enter a django shell
 
+Make sure to run this whenever you change things in `models.py`.
 ```bash
 cd archivebox/
 ./manage.py makemigrations
@@ -666,6 +674,7 @@ archivebox shell
 
 #### Build the docs, pip package, and docker image
 
+(Normally CI takes care of this, but these scripts can be run to do it manually)
 ```bash
 ./bin/build.sh
 
@@ -679,6 +688,7 @@ archivebox shell
 
 #### Roll a release
 
+(Normally CI takes care of this, but these scripts can be run to do it manually)
 ```bash
 ./bin/release.sh
 

From ed13ec7655c3d262ef937d3d3a225a90f79e1150 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 05:34:19 -0500
Subject: [PATCH 128/151] remove active theme

---
 archivebox/config.py        | 6 ++----
 archivebox/config_stubs.py  | 1 -
 archivebox/core/settings.py | 8 +++-----
 archivebox/main.py          | 3 +--
 4 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/archivebox/config.py b/archivebox/config.py
index dc014ed5..7fd4b2fc 100644
--- a/archivebox/config.py
+++ b/archivebox/config.py
@@ -76,7 +76,6 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'PUBLIC_SNAPSHOTS':         {'type': bool,  'default': True},
         'PUBLIC_ADD_VIEW':          {'type': bool,  'default': False},
         'FOOTER_INFO':              {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
-        'ACTIVE_THEME':             {'type': str,   'default': 'default'},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -204,12 +203,11 @@ def get_real_name(key: str) -> str:
 ################################ Constants #####################################
 
 PACKAGE_DIR_NAME = 'archivebox'
-TEMPLATES_DIR_NAME = 'themes'
+TEMPLATES_DIR_NAME = 'templates'
 
 ARCHIVE_DIR_NAME = 'archive'
 SOURCES_DIR_NAME = 'sources'
 LOGS_DIR_NAME = 'logs'
-STATIC_DIR_NAME = 'static'
 SQL_INDEX_FILENAME = 'index.sqlite3'
 JSON_INDEX_FILENAME = 'index.json'
 HTML_INDEX_FILENAME = 'index.html'
@@ -702,7 +700,7 @@ def get_code_locations(config: ConfigDict) -> SimpleConfigValueDict:
         'TEMPLATES_DIR': {
             'path': (config['TEMPLATES_DIR']).resolve(),
             'enabled': True,
-            'is_valid': (config['TEMPLATES_DIR'] / config['ACTIVE_THEME'] / 'static').exists(),
+            'is_valid': (config['TEMPLATES_DIR'] / 'static').exists(),
         },
         # 'NODE_MODULES_DIR': {
         #     'path': ,
diff --git a/archivebox/config_stubs.py b/archivebox/config_stubs.py
index 988f58a1..f9c22a0c 100644
--- a/archivebox/config_stubs.py
+++ b/archivebox/config_stubs.py
@@ -50,7 +50,6 @@ class ConfigDict(BaseConfig, total=False):
     PUBLIC_INDEX: bool
     PUBLIC_SNAPSHOTS: bool
     FOOTER_INFO: str
-    ACTIVE_THEME: str
 
     SAVE_TITLE: bool
     SAVE_FAVICON: bool
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index bfc0cdc3..bcf9c073 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -11,7 +11,6 @@ from ..config import (
     SECRET_KEY,
     ALLOWED_HOSTS,
     PACKAGE_DIR,
-    ACTIVE_THEME,
     TEMPLATES_DIR_NAME,
     SQL_INDEX_FILENAME,
     OUTPUT_DIR,
@@ -69,13 +68,12 @@ AUTHENTICATION_BACKENDS = [
 STATIC_URL = '/static/'
 
 STATICFILES_DIRS = [
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / ACTIVE_THEME / 'static'),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'default' / 'static'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
 ]
 
 TEMPLATE_DIRS = [
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / ACTIVE_THEME),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'default'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
 ]
 
diff --git a/archivebox/main.py b/archivebox/main.py
index c666f5d6..c55a2c04 100644
--- a/archivebox/main.py
+++ b/archivebox/main.py
@@ -79,7 +79,6 @@ from .config import (
     ARCHIVE_DIR_NAME,
     SOURCES_DIR_NAME,
     LOGS_DIR_NAME,
-    STATIC_DIR_NAME,
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,
     SQL_INDEX_FILENAME,
@@ -125,10 +124,10 @@ ALLOWED_IN_OUTPUT_DIR = {
     '.virtualenv',
     'node_modules',
     'package-lock.json',
+    'static',
     ARCHIVE_DIR_NAME,
     SOURCES_DIR_NAME,
     LOGS_DIR_NAME,
-    STATIC_DIR_NAME,
     SQL_INDEX_FILENAME,
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,

From a98298103daf10f189f7c0547dee03b593ef0d9e Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 05:35:07 -0500
Subject: [PATCH 129/151] cleanup templates and views

---
 archivebox/core/admin.py                      |   4 +-
 archivebox/core/templatetags/core_tags.py     |   2 +-
 archivebox/core/urls.py                       |   8 +-
 archivebox/core/views.py                      |  16 +-
 archivebox/index/html.py                      |   6 +-
 .../admin/actions_as_select.html              |   0
 .../admin/app_index.html                      |   0
 .../{themes => templates}/admin/base.html     |   0
 .../{themes => templates}/admin/login.html    |   0
 archivebox/templates/admin/private_index.html | 150 ++++++++++++
 .../admin/private_index_grid.html}            |   2 +-
 .../admin/snapshots_grid.html                 |   2 +-
 .../core/add.html}                            |   2 +-
 .../default => templates/core}/base.html      |   0
 .../core/index_row.html}                      |   6 +-
 .../core/minimal_index.html}                  |   4 +-
 .../core/public_index.html}                   |   2 +-
 .../core/snapshot.html}                       |   0
 .../core/static_index.html}                   |   2 +-
 .../default => templates}/static/add.css      |   0
 .../default => templates}/static/admin.css    |   0
 .../default => templates}/static/archive.png  | Bin
 .../static/bootstrap.min.css                  |   0
 .../default => templates}/static/external.png | Bin
 .../static/jquery.dataTables.min.css          |   0
 .../static/jquery.dataTables.min.js           |   0
 .../static/jquery.min.js                      |   0
 .../default => templates}/static/sort_asc.png | Bin
 .../static/sort_both.png                      | Bin
 .../static/sort_desc.png                      | Bin
 .../default => templates}/static/spinner.gif  | Bin
 archivebox/themes/legacy/main_index.html      | 215 ------------------
 archivebox/themes/legacy/main_index_row.html  |  16 --
 etc/ArchiveBox.conf.default                   |   1 -
 34 files changed, 179 insertions(+), 259 deletions(-)
 rename archivebox/{themes => templates}/admin/actions_as_select.html (100%)
 rename archivebox/{themes => templates}/admin/app_index.html (100%)
 rename archivebox/{themes => templates}/admin/base.html (100%)
 rename archivebox/{themes => templates}/admin/login.html (100%)
 create mode 100644 archivebox/templates/admin/private_index.html
 rename archivebox/{themes/admin/grid_change_list.html => templates/admin/private_index_grid.html} (99%)
 rename archivebox/{themes => templates}/admin/snapshots_grid.html (99%)
 rename archivebox/{themes/default/add_links.html => templates/core/add.html} (98%)
 rename archivebox/{themes/default => templates/core}/base.html (100%)
 rename archivebox/{themes/default/main_index_row.html => templates/core/index_row.html} (82%)
 rename archivebox/{themes/default/main_index_minimal.html => templates/core/minimal_index.html} (90%)
 rename archivebox/{themes/default/core/snapshot_list.html => templates/core/public_index.html} (97%)
 rename archivebox/{themes/default/link_details.html => templates/core/snapshot.html} (100%)
 rename archivebox/{themes/default/main_index.html => templates/core/static_index.html} (99%)
 rename archivebox/{themes/default => templates}/static/add.css (100%)
 rename archivebox/{themes/default => templates}/static/admin.css (100%)
 rename archivebox/{themes/default => templates}/static/archive.png (100%)
 rename archivebox/{themes/default => templates}/static/bootstrap.min.css (100%)
 rename archivebox/{themes/default => templates}/static/external.png (100%)
 rename archivebox/{themes/default => templates}/static/jquery.dataTables.min.css (100%)
 rename archivebox/{themes/default => templates}/static/jquery.dataTables.min.js (100%)
 rename archivebox/{themes/default => templates}/static/jquery.min.js (100%)
 rename archivebox/{themes/default => templates}/static/sort_asc.png (100%)
 rename archivebox/{themes/default => templates}/static/sort_both.png (100%)
 rename archivebox/{themes/default => templates}/static/sort_desc.png (100%)
 rename archivebox/{themes/default => templates}/static/spinner.gif (100%)
 delete mode 100644 archivebox/themes/legacy/main_index.html
 delete mode 100644 archivebox/themes/legacy/main_index_row.html

diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index f641b177..518731f1 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -171,7 +171,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
         saved_list_max_show_all = self.list_max_show_all
 
         # Monkey patch here plus core_tags.py
-        self.change_list_template = 'admin/grid_change_list.html'
+        self.change_list_template = 'private_index_grid.html'
         self.list_per_page = 20
         self.list_max_show_all = self.list_per_page
 
@@ -249,7 +249,7 @@ class ArchiveBoxAdmin(admin.AdminSite):
             else:
                 context["form"] = form
 
-        return render(template_name='add_links.html', request=request, context=context)
+        return render(template_name='add.html', request=request, context=context)
 
 admin.site = ArchiveBoxAdmin()
 admin.site.register(get_user_model())
diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py
index 25f06852..9ac1ee27 100644
--- a/archivebox/core/templatetags/core_tags.py
+++ b/archivebox/core/templatetags/core_tags.py
@@ -14,7 +14,7 @@ register = template.Library()
 def snapshot_image(snapshot):
     result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first()
     if result:
-        return reverse('LinkAssets', args=[f'{str(snapshot.timestamp)}/{result.output}'])
+        return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
     
     return static('archive.png')
 
diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py
index b8e4bafb..4c7b429c 100644
--- a/archivebox/core/urls.py
+++ b/archivebox/core/urls.py
@@ -5,22 +5,24 @@ from django.views import static
 from django.conf import settings
 from django.views.generic.base import RedirectView
 
-from core.views import MainIndex, LinkDetails, PublicArchiveView, AddView
+from core.views import HomepageView, SnapshotView, PublicIndexView, AddView
 
 
 # print('DEBUG', settings.DEBUG)
 
 urlpatterns = [
+    path('public/', PublicIndexView.as_view(), name='public-index'),
+
     path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
     path('favicon.ico', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'favicon.ico'}),
 
     path('docs/', RedirectView.as_view(url='https://github.com/ArchiveBox/ArchiveBox/wiki'), name='Docs'),
 
     path('archive/', RedirectView.as_view(url='/')),
-    path('archive/', LinkDetails.as_view(), name='LinkAssets'),
+    path('archive/', SnapshotView.as_view(), name='Snapshot'),
 
     path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
-    path('add/', AddView.as_view()),
+    path('add/', AddView.as_view(), name='add'),
     
     path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
     path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index 810b4740..0e19fad6 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -28,20 +28,20 @@ from ..util import base_url, ansi_to_html
 from ..index.html import snapshot_icons
 
 
-class MainIndex(View):
-    template = 'main_index.html'
-
+class HomepageView(View):
     def get(self, request):
         if request.user.is_authenticated:
             return redirect('/admin/core/snapshot/')
 
         if PUBLIC_INDEX:
-            return redirect('public-index')
+            return redirect('/public')
         
         return redirect(f'/admin/login/?next={request.path}')
 
 
-class LinkDetails(View):
+class SnapshotView(View):
+    # render static html index from filesystem archive//index.html
+
     def get(self, request, path):
         # missing trailing slash -> redirect to index
         if '/' not in path:
@@ -91,8 +91,8 @@ class LinkDetails(View):
             status=404,
         )
 
-class PublicArchiveView(ListView):
-    template = 'snapshot_list.html'
+class PublicIndexView(ListView):
+    template_name = 'public_index.html'
     model = Snapshot
     paginate_by = 100
     ordering = ['title']
@@ -122,7 +122,7 @@ class PublicArchiveView(ListView):
 
 
 class AddView(UserPassesTestMixin, FormView):
-    template_name = "add_links.html"
+    template_name = "add.html"
     form_class = AddLinkForm
 
     def get_initial(self):
diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 28f25fde..3eca5f01 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -25,9 +25,9 @@ from ..config import (
     HTML_INDEX_FILENAME,
 )
 
-MAIN_INDEX_TEMPLATE = 'main_index.html'
-MINIMAL_INDEX_TEMPLATE = 'main_index_minimal.html'
-LINK_DETAILS_TEMPLATE = 'link_details.html'
+MAIN_INDEX_TEMPLATE = 'static_index.html'
+MINIMAL_INDEX_TEMPLATE = 'minimal_index.html'
+LINK_DETAILS_TEMPLATE = 'snapshot.html'
 TITLE_LOADING_MSG = 'Not yet archived...'
 
 
diff --git a/archivebox/themes/admin/actions_as_select.html b/archivebox/templates/admin/actions_as_select.html
similarity index 100%
rename from archivebox/themes/admin/actions_as_select.html
rename to archivebox/templates/admin/actions_as_select.html
diff --git a/archivebox/themes/admin/app_index.html b/archivebox/templates/admin/app_index.html
similarity index 100%
rename from archivebox/themes/admin/app_index.html
rename to archivebox/templates/admin/app_index.html
diff --git a/archivebox/themes/admin/base.html b/archivebox/templates/admin/base.html
similarity index 100%
rename from archivebox/themes/admin/base.html
rename to archivebox/templates/admin/base.html
diff --git a/archivebox/themes/admin/login.html b/archivebox/templates/admin/login.html
similarity index 100%
rename from archivebox/themes/admin/login.html
rename to archivebox/templates/admin/login.html
diff --git a/archivebox/templates/admin/private_index.html b/archivebox/templates/admin/private_index.html
new file mode 100644
index 00000000..7afb62c3
--- /dev/null
+++ b/archivebox/templates/admin/private_index.html
@@ -0,0 +1,150 @@
+{% extends "base.html" %}
+{% load static %}
+
+{% block body %}
+    

+        
+            

+                
+                
+                
+                
+                
+            

+        
+    

+    

         
             

             
     
     
         📄 
-            {% if link.icons  %} {{link.icons}} {% else %} {{ link.num_outputs}} {% endif %}
+            
+                {% if link.icons  %}
+                    {{link.icons}}
+                {% else %}
+                    {{link.num_outputs}} 
+                {% endif %}
+            
         
     
    {{link.url}}

         📄 

From 5c54bcc1f3ccacafcca554047127ea87c5a106a0 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Thu, 28 Jan 2021 22:57:12 -0500
Subject: [PATCH 117/151] fix files icons greying out on public index

---
 archivebox/index/html.py                      |  2 +-
 archivebox/themes/default/main_index.html     |  8 --------
 archivebox/themes/default/main_index_row.html | 18 +++++++++---------
 archivebox/themes/default/static/admin.css    |  7 ++++++-
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 0ba8e7c1..91ff83cd 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -121,7 +121,7 @@ def snapshot_icons(snapshot) -> str:
     path = link.archive_path
     canon = link.canonical_outputs()
     output = ""
-    output_template = '{} '
+    output_template = '{}  '
     icons = {
         "singlefile": "❶",
         "wget": "🆆",
diff --git a/archivebox/themes/default/main_index.html b/archivebox/themes/default/main_index.html
index 85753b31..95af1963 100644
--- a/archivebox/themes/default/main_index.html
+++ b/archivebox/themes/default/main_index.html
@@ -185,14 +185,6 @@
             .title-col a {
                 color: black;
             }
-            .tags {
-                float: right;
-                border-radius: 5px;
-                background-color: #bfdfff;
-                padding: 2px 5px;
-                margin-left: 4px;
-                margin-top: 1px;
-            }
         
         
         
diff --git a/archivebox/themes/default/main_index_row.html b/archivebox/themes/default/main_index_row.html
index cfbcbfe8..cb821f61 100644
--- a/archivebox/themes/default/main_index_row.html
+++ b/archivebox/themes/default/main_index_row.html
@@ -2,7 +2,7 @@
 
 

      {% if link.bookmarked_date  %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} 
+    
         {% if link.is_archived %}
             
         {% else %}
@@ -23,15 +23,15 @@
         
     
     
-        📄 
-            
-                {% if link.icons  %}
-                    {{link.icons}}
-                {% else %}
+        
+            {% if link.icons  %}
+                {{link.icons}} {{link.num_outputs}}
+            {% else %}
+                📄 
                     {{link.num_outputs}} 
-                {% endif %}
-            
-        
+                
+            {% endif %}
+        
     
    {{link.url}}
 

+        
+            
+                
+                
+                
+                
+            
+        
+            
+                {% for link in object_list %}
+                    {% include 'main_index_row.html' with link=link  %}
+                {% endfor %}
+            
+        
BookmarkedSnapshot ({{object_list|length}})FilesOriginal URL

+        

+            
+                {% if page_obj.has_previous %}
+                    « first
+                    previous
+                {% endif %}
+        
+                
+                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
+                
+        
+                {% if page_obj.has_next %}
+                    next 
+                    last »
+                {% endif %}
+            
+    
+            {% if page_obj.has_next %}
+                next 
+                last »
+            {% endif %}
+        
+        

+    

+{% endblock %}
+{% extends "admin/base_site.html" %}
+{% load i18n admin_urls static admin_list %}
+{% load core_tags %}
+
+{% block extrastyle %}
+  {{ block.super }}
+  
+  {% if cl.formset %}
+    
+  {% endif %}
+  {% if cl.formset or action_form %}
+    
+  {% endif %}
+  {{ media.css }}
+  {% if not actions_on_top and not actions_on_bottom %}
+    
+  {% endif %}
+{% endblock %}
+
+{% block extrahead %}
+{{ block.super }}
+{{ media.js }}
+{% endblock %}
+
+{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %}
+
+{% if not is_popup %}
+{% block breadcrumbs %}
+

+{% translate 'Home' %}
+› {{ cl.opts.app_config.verbose_name }}
+› {{ cl.opts.verbose_name_plural|capfirst }}
+

+{% endblock %}
+{% endif %}
+
+{% block coltype %}{% endblock %}
+
+{% block content %}
+  

+    {% block object-tools %}
+        
    
+          {% block object-tools-items %}
+            {% change_list_object_tools %}
+          {% endblock %}
+        

+    {% endblock %}
+    {% if cl.formset and cl.formset.errors %}
+        

+        {% if cl.formset.total_error_count == 1 %}{% translate "Please correct the error below." %}{% else %}{% translate "Please correct the errors below." %}{% endif %}
+        

+        {{ cl.formset.non_form_errors }}
+    {% endif %}
+    

+      

+        {% block search %}{% search_form cl %}{% endblock %}
+        {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %}
+
+        
{% csrf_token %}
+        {% if cl.formset %}
+          
{{ cl.formset.management_form }}

+        {% endif %}
+
+        {% block result_list %}
+          {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %}
+          {% comment %} 
+             Table grid
+            {% result_list cl %} 
+          {% endcomment %}
+          {% snapshots_grid cl %}
+          {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %}
+        {% endblock %}
+        {% block pagination %}{% pagination cl %}{% endblock %}
+        

+      

+      {% block filters %}
+        {% if cl.has_filters %}
+          

+            
{% translate 'Filter' %}

+            {% if cl.has_active_filters %}

+              ✖ {% translate "Clear all filters" %}
+            
{% endif %}
+            {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %}
+          

+        {% endif %}
+      {% endblock %}
+    

+  

+{% endblock %}
diff --git a/archivebox/themes/admin/grid_change_list.html b/archivebox/templates/admin/private_index_grid.html
similarity index 99%
rename from archivebox/themes/admin/grid_change_list.html
rename to archivebox/templates/admin/private_index_grid.html
index 6894efd7..b60f3a3e 100644
--- a/archivebox/themes/admin/grid_change_list.html
+++ b/archivebox/templates/admin/private_index_grid.html
@@ -88,4 +88,4 @@
       {% endblock %}
     

   

-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/archivebox/themes/admin/snapshots_grid.html b/archivebox/templates/admin/snapshots_grid.html
similarity index 99%
rename from archivebox/themes/admin/snapshots_grid.html
rename to archivebox/templates/admin/snapshots_grid.html
index a7a2d4f9..10788060 100644
--- a/archivebox/themes/admin/snapshots_grid.html
+++ b/archivebox/templates/admin/snapshots_grid.html
@@ -159,4 +159,4 @@ footer {
     {% endfor %}
     
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/archivebox/themes/default/add_links.html b/archivebox/templates/core/add.html
similarity index 98%
rename from archivebox/themes/default/add_links.html
rename to archivebox/templates/core/add.html
index fa8b441f..0f161885 100644
--- a/archivebox/themes/default/add_links.html
+++ b/archivebox/templates/core/add.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "core/base.html" %}
 
 {% load static %}
 {% load i18n %}
diff --git a/archivebox/themes/default/base.html b/archivebox/templates/core/base.html
similarity index 100%
rename from archivebox/themes/default/base.html
rename to archivebox/templates/core/base.html
diff --git a/archivebox/themes/default/main_index_row.html b/archivebox/templates/core/index_row.html
similarity index 82%
rename from archivebox/themes/default/main_index_row.html
rename to archivebox/templates/core/index_row.html
index eae60ea9..cba3ec39 100644
--- a/archivebox/themes/default/main_index_row.html
+++ b/archivebox/templates/core/index_row.html
@@ -4,9 +4,9 @@
      {% if link.bookmarked_date  %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} 
     
         {% if link.is_archived %}
-            
+            
         {% else %}
-            
+            
         {% endif %}
 
         
@@ -28,7 +28,7 @@
                 {{link.icons}} {{link.num_outputs}}
             {% else %}
                 📄 
-                    {{link.num_outputs}} 
+                    {{link.num_outputs}} 
                 
             {% endif %}
         
diff --git a/archivebox/themes/default/main_index_minimal.html b/archivebox/templates/core/minimal_index.html
similarity index 90%
rename from archivebox/themes/default/main_index_minimal.html
rename to archivebox/templates/core/minimal_index.html
index dcfaa23f..3c69a831 100644
--- a/archivebox/themes/default/main_index_minimal.html
+++ b/archivebox/templates/core/minimal_index.html
@@ -16,9 +16,9 @@
             
             
                 {% for link in links %}
-                    {% include "main_index_row.html" with link=link %} 
+                    {% include "index_row.html" with link=link %} 
                 {% endfor %}
             
         
     
-
\ No newline at end of file
+
diff --git a/archivebox/themes/default/core/snapshot_list.html b/archivebox/templates/core/public_index.html
similarity index 97%
rename from archivebox/themes/default/core/snapshot_list.html
rename to archivebox/templates/core/public_index.html
index dd8ebf15..327042ea 100644
--- a/archivebox/themes/default/core/snapshot_list.html
+++ b/archivebox/templates/core/public_index.html
@@ -28,7 +28,7 @@
         
             
                 {% for link in object_list %}
-                    {% include 'main_index_row.html' with link=link  %}
+                    {% include 'index_row.html' with link=link  %}
                 {% endfor %}
             
         
diff --git a/archivebox/themes/default/link_details.html b/archivebox/templates/core/snapshot.html
similarity index 100%
rename from archivebox/themes/default/link_details.html
rename to archivebox/templates/core/snapshot.html
diff --git a/archivebox/themes/default/main_index.html b/archivebox/templates/core/static_index.html
similarity index 99%
rename from archivebox/themes/default/main_index.html
rename to archivebox/templates/core/static_index.html
index 269c9825..07066e27 100644
--- a/archivebox/themes/default/main_index.html
+++ b/archivebox/templates/core/static_index.html
@@ -234,7 +234,7 @@
             
             
                 {% for link in links %}
-                   {% include 'main_index_row.html' with link=link %}
+                   {% include 'index_row.html' with link=link %}
                 {% endfor %}
             
         
diff --git a/archivebox/themes/default/static/add.css b/archivebox/templates/static/add.css
similarity index 100%
rename from archivebox/themes/default/static/add.css
rename to archivebox/templates/static/add.css
diff --git a/archivebox/themes/default/static/admin.css b/archivebox/templates/static/admin.css
similarity index 100%
rename from archivebox/themes/default/static/admin.css
rename to archivebox/templates/static/admin.css
diff --git a/archivebox/themes/default/static/archive.png b/archivebox/templates/static/archive.png
similarity index 100%
rename from archivebox/themes/default/static/archive.png
rename to archivebox/templates/static/archive.png
diff --git a/archivebox/themes/default/static/bootstrap.min.css b/archivebox/templates/static/bootstrap.min.css
similarity index 100%
rename from archivebox/themes/default/static/bootstrap.min.css
rename to archivebox/templates/static/bootstrap.min.css
diff --git a/archivebox/themes/default/static/external.png b/archivebox/templates/static/external.png
similarity index 100%
rename from archivebox/themes/default/static/external.png
rename to archivebox/templates/static/external.png
diff --git a/archivebox/themes/default/static/jquery.dataTables.min.css b/archivebox/templates/static/jquery.dataTables.min.css
similarity index 100%
rename from archivebox/themes/default/static/jquery.dataTables.min.css
rename to archivebox/templates/static/jquery.dataTables.min.css
diff --git a/archivebox/themes/default/static/jquery.dataTables.min.js b/archivebox/templates/static/jquery.dataTables.min.js
similarity index 100%
rename from archivebox/themes/default/static/jquery.dataTables.min.js
rename to archivebox/templates/static/jquery.dataTables.min.js
diff --git a/archivebox/themes/default/static/jquery.min.js b/archivebox/templates/static/jquery.min.js
similarity index 100%
rename from archivebox/themes/default/static/jquery.min.js
rename to archivebox/templates/static/jquery.min.js
diff --git a/archivebox/themes/default/static/sort_asc.png b/archivebox/templates/static/sort_asc.png
similarity index 100%
rename from archivebox/themes/default/static/sort_asc.png
rename to archivebox/templates/static/sort_asc.png
diff --git a/archivebox/themes/default/static/sort_both.png b/archivebox/templates/static/sort_both.png
similarity index 100%
rename from archivebox/themes/default/static/sort_both.png
rename to archivebox/templates/static/sort_both.png
diff --git a/archivebox/themes/default/static/sort_desc.png b/archivebox/templates/static/sort_desc.png
similarity index 100%
rename from archivebox/themes/default/static/sort_desc.png
rename to archivebox/templates/static/sort_desc.png
diff --git a/archivebox/themes/default/static/spinner.gif b/archivebox/templates/static/spinner.gif
similarity index 100%
rename from archivebox/themes/default/static/spinner.gif
rename to archivebox/templates/static/spinner.gif
diff --git a/archivebox/themes/legacy/main_index.html b/archivebox/themes/legacy/main_index.html
deleted file mode 100644
index 74e7bf65..00000000
--- a/archivebox/themes/legacy/main_index.html
+++ /dev/null
@@ -1,215 +0,0 @@
-
-
-    
-        Archived Sites
-        
-        
-        
-        
-        
-        
-        
-    
-    
-        

-            

-                

-                    

-                        
-                            Logo
-                            ArchiveBox
-                        
-                    

-                    

-                        Documentation   |   
-                        Source   |   
-                        Website
-                    

-                

-            

-        

-        
-            
-                
-                    
-                    
-                    
-                    
-                
-            
-            $rows
-        
BookmarkedSnapshot ($num_links)FilesOriginal URL

-        
-    
-
diff --git a/archivebox/themes/legacy/main_index_row.html b/archivebox/themes/legacy/main_index_row.html
deleted file mode 100644
index 9112eace..00000000
--- a/archivebox/themes/legacy/main_index_row.html
+++ /dev/null
@@ -1,16 +0,0 @@
-
-    $bookmarked_date
-    
-        
-        
-            $title
-            $tags
-        
-    
-    
-        📄 
-            $num_outputs
-        
-    
-    $url
-
diff --git a/etc/ArchiveBox.conf.default b/etc/ArchiveBox.conf.default
index fe3bcdde..982a1931 100644
--- a/etc/ArchiveBox.conf.default
+++ b/etc/ArchiveBox.conf.default
@@ -24,7 +24,6 @@
 # PUBLIC_INDEX = True
 # PUBLIC_SNAPSHOTS = True
 # FOOTER_INFO = Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.
-# ACTIVE_THEME = default
 
 
 [ARCHIVE_METHOD_TOGGLES]

From 6edae6a17f01edbe2644b10a5be3c58ce7b0fd34 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 05:35:17 -0500
Subject: [PATCH 130/151] add future api spec design

---
 archivebox/core/admin.py | 10 ++++++++++
 archivebox/core/urls.py  | 35 +++++++++++++++++++++++++++++++++--
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index 518731f1..8c3c3599 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -24,6 +24,16 @@ from main import add, remove
 from config import OUTPUT_DIR
 from extractors import archive_links
 
+# Admin URLs
+# /admin/
+# /admin/login/
+# /admin/core/
+# /admin/core/snapshot/
+# /admin/core/snapshot/:uuid/
+# /admin/core/tag/
+# /admin/core/tag/:uuid/
+
+
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
 def update_snapshots(modeladmin, request, queryset):
diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py
index 4c7b429c..182e4dca 100644
--- a/archivebox/core/urls.py
+++ b/archivebox/core/urls.py
@@ -33,6 +33,37 @@ urlpatterns = [
     
     path('index.html', RedirectView.as_view(url='/')),
     path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),
-    path('', MainIndex.as_view(), name='Home'),
-    path('public/', PublicArchiveView.as_view(), name='public-index'),
+    path('', HomepageView.as_view(), name='Home'),
 ]
+
+    # # Proposed UI URLs spec
+    # path('',                 HomepageView)
+    # path('/add',             AddView)
+    # path('/public',          PublicIndexView)
+    # path('/snapshot/:slug',  SnapshotView)
+    
+    # path('/admin',           admin.site.urls)
+    # path('/accounts',        django.contrib.auth.urls)
+
+    # # Prposed REST API spec
+    # # :slugs can be uuid, short_uuid, or any of the unique index_fields
+    # path('api/v1/'),
+    # path('api/v1/core/'                      [GET])
+    # path('api/v1/core/snapshot/',            [GET, POST, PUT]),
+    # path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
+    # path('api/v1/core/archiveresult',        [GET, POST, PUT]),
+    # path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
+    # path('api/v1/core/tag/',                 [GET, POST, PUT]),
+    # path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
+
+    # path('api/v1/cli/',                      [GET])
+    # path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
+
+    # path('api/v1/extractors/',                    [GET])
+    # path('api/v1/extractors/:extractor/',         [GET]),
+    # path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
+
+    # future, just an idea:
+    # path('api/v1/scheduler/',                [GET])
+    # path('api/v1/scheduler/task/',           [GET, POST, PUT]),
+    # path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),

From 1ce0eca2176a65a10161783ea12a89729bc4072d Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 05:35:29 -0500
Subject: [PATCH 131/151] add trailing slashes to canonical paths

---
 archivebox/index/schema.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py
index bc3a25da..5c5eb0f0 100644
--- a/archivebox/index/schema.py
+++ b/archivebox/index/schema.py
@@ -417,7 +417,7 @@ class Link:
             'favicon_path': 'favicon.ico',
             'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
             'wget_path': wget_output_path(self),
-            'warc_path': 'warc',
+            'warc_path': 'warc/',
             'singlefile_path': 'singlefile.html',
             'readability_path': 'readability/content.html',
             'mercury_path': 'mercury/content.html',
@@ -425,8 +425,8 @@ class Link:
             'screenshot_path': 'screenshot.png',
             'dom_path': 'output.html',
             'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
-            'git_path': 'git',
-            'media_path': 'media',
+            'git_path': 'git/',
+            'media_path': 'media/',
         }
         if self.is_static:
             # static binary files like PDF and images are handled slightly differently.

From cc80ceb0a27d1aa0564f43e4d21d069272eab3c0 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 05:47:33 -0500
Subject: [PATCH 132/151] fix icons in public index

---
 archivebox/index/html.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index 3eca5f01..cff50085 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -146,9 +146,15 @@ def snapshot_icons(snapshot) -> str:
 
     for extractor, _ in EXTRACTORS:
         if extractor not in exclude:
-            exists = extractor_items[extractor] is not None
+            exists = False
+            if extractor_items[extractor] is not None:
+                outpath = (Path(path) / canon[f"{extractor}_path"])
+                if outpath.is_dir():
+                    exists = any(outpath.glob('*.*'))
+                elif outpath.is_file():
+                    exists = outpath.stat().st_size > 100
             output += format_html(output_template, path, canon[f"{extractor}_path"], str(exists),
-                                             extractor, icons.get(extractor, "?"))
+                                         extractor, icons.get(extractor, "?"))
         if extractor == "wget":
             # warc isn't technically it's own extractor, so we have to add it after wget
             exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))

From d6de04a83ad0963c1b36209e124a66358d09aab6 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 06:07:35 -0500
Subject: [PATCH 133/151] fix lgtm errors

---
 archivebox/core/settings.py         | 2 ++
 archivebox/extractors/favicon.py    | 3 +--
 archivebox/index/__init__.py        | 2 +-
 archivebox/parsers/generic_txt.py   | 4 ++--
 archivebox/parsers/wallabag_atom.py | 2 +-
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index bcf9c073..918e15e9 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -33,6 +33,8 @@ LOGOUT_REDIRECT_URL = '/'
 PASSWORD_RESET_URL = '/accounts/password_reset/'
 APPEND_SLASH = True
 
+DEBUG = DEBUG or sys.environ.get('DEBUG', 'false').lower() != 'false' or '--debug' in sys.argv
+
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.contenttypes',
diff --git a/archivebox/extractors/favicon.py b/archivebox/extractors/favicon.py
index 3a4aeea7..b8831d0c 100644
--- a/archivebox/extractors/favicon.py
+++ b/archivebox/extractors/favicon.py
@@ -42,14 +42,13 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
         'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
     ]
-    status = 'pending'
+    status = 'failed'
     timer = TimedProgress(timeout, prefix='      ')
     try:
         run(cmd, cwd=str(out_dir), timeout=timeout)
         chmod_file(output, cwd=str(out_dir))
         status = 'succeeded'
     except Exception as err:
-        status = 'failed'
         output = err
     finally:
         timer.end()
diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py
index 8eab1d38..04ab0a8d 100644
--- a/archivebox/index/__init__.py
+++ b/archivebox/index/__init__.py
@@ -2,7 +2,6 @@ __package__ = 'archivebox.index'
 
 import os
 import shutil
-import json as pyjson
 from pathlib import Path
 
 from itertools import chain
@@ -42,6 +41,7 @@ from .html import (
     write_html_link_details,
 )
 from .json import (
+    pyjson,
     parse_json_link_details, 
     write_json_link_details,
 )
diff --git a/archivebox/parsers/generic_txt.py b/archivebox/parsers/generic_txt.py
index e296ec7e..94dd523c 100644
--- a/archivebox/parsers/generic_txt.py
+++ b/archivebox/parsers/generic_txt.py
@@ -51,9 +51,9 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
             # look inside the URL for any sub-urls, e.g. for archive.org links
             # https://web.archive.org/web/20200531203453/https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             # -> https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
-            for url in re.findall(URL_REGEX, line[1:]):
+            for sub_url in re.findall(URL_REGEX, line[1:]):
                 yield Link(
-                    url=htmldecode(url),
+                    url=htmldecode(sub_url),
                     timestamp=str(datetime.now().timestamp()),
                     title=None,
                     tags=None,
diff --git a/archivebox/parsers/wallabag_atom.py b/archivebox/parsers/wallabag_atom.py
index 0d77869f..7acfc2fc 100644
--- a/archivebox/parsers/wallabag_atom.py
+++ b/archivebox/parsers/wallabag_atom.py
@@ -45,7 +45,7 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
         try:
             tags = str_between(get_row('category'), 'label="', '" />')
-        except:
+        except Exception:
             tags = None
 
         yield Link(

From 326ce78496176f753e48d7142c199b750b3780d9 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 06:09:26 -0500
Subject: [PATCH 134/151] simplify debug

---
 archivebox/core/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index 918e15e9..e73c93d9 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -33,7 +33,7 @@ LOGOUT_REDIRECT_URL = '/'
 PASSWORD_RESET_URL = '/accounts/password_reset/'
 APPEND_SLASH = True
 
-DEBUG = DEBUG or sys.environ.get('DEBUG', 'false').lower() != 'false' or '--debug' in sys.argv
+DEBUG = DEBUG or ('--debug' in sys.argv)
 
 INSTALLED_APPS = [
     'django.contrib.auth',

From 8e493bf556c75d6560ab78e7f04556b290416178 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 06:16:24 -0500
Subject: [PATCH 135/151] heading fix

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 19196b4f..c1464eb8 100644
--- a/README.md
+++ b/README.md
@@ -639,7 +639,7 @@ archivebox config --set DEBUG=True
 archivebox server --debug ...
 ```
 
-### Build and run a Github branch
+#### Build and run a Github branch
 
 ```bash
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev

From c25853969d6996ca5200f411b0e96dee6ec6908c Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 08:25:34 -0500
Subject: [PATCH 136/151] add dbshell command examples for executing SQL

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c1464eb8..a83922a3 100644
--- a/README.md
+++ b/README.md
@@ -73,7 +73,7 @@ archivebox help
 - `archivebox add/remove/update/list` to manage Snapshots in the archive
 - `archivebox schedule` to pull in fresh URLs in regularly from [boorkmarks/history/Pocket/Pinboard/RSS/etc.](#input-formats)
 - `archivebox oneshot` archive single URLs without starting a whole collection
-- `archivebox shell` open a REPL to use the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
+- `archivebox shell/manage dbshell` open a REPL to use the [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha), or SQL API
 
 

 

@@ -669,6 +669,7 @@ cd archivebox/
 
 cd path/to/test/data/
 archivebox shell
+archivebox manage dbshell
 ```
 (uses `pytest -s`)
 

From 9d24bfd0dcef782a64d4b52117aa5ab5a67e9163 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 20:38:59 -0500
Subject: [PATCH 137/151] disable progress bars on mac again

---
 archivebox/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/archivebox/config.py b/archivebox/config.py
index 7fd4b2fc..23ec17d2 100644
--- a/archivebox/config.py
+++ b/archivebox/config.py
@@ -27,6 +27,7 @@ import re
 import sys
 import json
 import getpass
+import platform
 import shutil
 import django
 
@@ -51,7 +52,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
     'SHELL_CONFIG': {
         'IS_TTY':                   {'type': bool,  'default': lambda _: sys.stdout.isatty()},
         'USE_COLOR':                {'type': bool,  'default': lambda c: c['IS_TTY']},
-        'SHOW_PROGRESS':            {'type': bool,  'default': lambda c: c['IS_TTY']},
+        'SHOW_PROGRESS':            {'type': bool,  'default': lambda c: (c['IS_TTY'] and platform.system() != 'Darwin')},  # progress bars are buggy on mac, disable for now
         'IN_DOCKER':                {'type': bool,  'default': False},
         # TODO: 'SHOW_HINTS':       {'type:  bool,  'default': True},
     },

From d072f1d4136cb3cb0f07e413395f0e62dcb6f118 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 20:39:11 -0500
Subject: [PATCH 138/151] hide ssl warnings when checking SSL is disabled

---
 archivebox/config.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/archivebox/config.py b/archivebox/config.py
index 23ec17d2..f984d027 100644
--- a/archivebox/config.py
+++ b/archivebox/config.py
@@ -915,7 +915,11 @@ os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8))  # noqa: F821
 NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))
 sys.path.append(NODE_BIN_PATH)
 
-
+if not CHECK_SSL_VALIDITY:
+    import urllib3
+    import requests
+    requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
 ########################### Config Validity Checkers ###########################

From b9b1c3d9e8990ab3d603a78116be958a622b2a16 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 20:40:10 -0500
Subject: [PATCH 139/151] fix singlefile output path not relative

---
 archivebox/core/admin.py            | 2 +-
 archivebox/extractors/singlefile.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index 8c3c3599..ea51f668 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -99,7 +99,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     list_display = ('added', 'title_str', 'url_str', 'files', 'size')
     sort_fields = ('title_str', 'url_str', 'added')
     readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
-    search_fields = ['url', 'timestamp', 'title', 'tags__name']
+    search_fields = ['url__icontains', 'timestamp', 'title', 'tags__name']
     fields = (*readonly_fields, 'title', 'tags')
     list_filter = ('added', 'updated', 'tags')
     ordering = ['-added']
diff --git a/archivebox/extractors/singlefile.py b/archivebox/extractors/singlefile.py
index 8d9b36be..3279960e 100644
--- a/archivebox/extractors/singlefile.py
+++ b/archivebox/extractors/singlefile.py
@@ -39,7 +39,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
     """download full site using single-file"""
 
     out_dir = out_dir or Path(link.link_dir)
-    output = str(out_dir.absolute() / "singlefile.html")
+    output = "singlefile.html"
 
     browser_args = chrome_args(TIMEOUT=0)
 
@@ -50,7 +50,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
         '--browser-executable-path={}'.format(CHROME_BINARY),
         browser_args,
         link.url,
-        output
+        output,
     ]
 
     status = 'succeeded'
@@ -71,9 +71,9 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
         )
 
         # Check for common failure cases
-        if (result.returncode > 0):
+        if (result.returncode > 0) or not (out_dir / output).is_file():
             raise ArchiveError('SingleFile was not able to archive the page', hints)
-        chmod_file(output)
+        chmod_file(output, cwd=str(out_dir))
     except (Exception, OSError) as err:
         status = 'failed'
         # TODO: Make this prettier. This is necessary to run the command (escape JSON internal quotes).

From c089501073983b6d96d9ec08fcb66f49745e21db Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 20:41:39 -0500
Subject: [PATCH 140/151] add response status code to headers.json

---
 archivebox/util.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/archivebox/util.py b/archivebox/util.py
index 5530ab45..a96950bb 100644
--- a/archivebox/util.py
+++ b/archivebox/util.py
@@ -200,7 +200,13 @@ def get_headers(url: str, timeout: int=None) -> str:
             stream=True
         )
     
-    return pyjson.dumps(dict(response.headers), indent=4)
+    return pyjson.dumps(
+        {
+            'Status-Code': response.status_code,
+            **dict(response.headers),
+        },
+        indent=4,
+    )
 
 
 @enforce_types

From 24e24934f761ca488b0b51c21da1935df96ab244 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 21:58:38 -0500
Subject: [PATCH 141/151] add headers.json and fix relative singlefile path
 resolving for sonic

---
 archivebox/index/schema.py | 1 +
 archivebox/search/utils.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py
index 5c5eb0f0..7e2c784d 100644
--- a/archivebox/index/schema.py
+++ b/archivebox/index/schema.py
@@ -427,6 +427,7 @@ class Link:
             'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
             'git_path': 'git/',
             'media_path': 'media/',
+            'headers_path': 'headers.json',
         }
         if self.is_static:
             # static binary files like PDF and images are handled slightly differently.
diff --git a/archivebox/search/utils.py b/archivebox/search/utils.py
index 55c97e75..e6d15455 100644
--- a/archivebox/search/utils.py
+++ b/archivebox/search/utils.py
@@ -34,10 +34,11 @@ def get_indexable_content(results: QuerySet):
         return []
     # This should come from a plugin interface
 
+    # TODO: banish this duplication and get these from the extractor file
     if method == 'readability':
         return get_file_result_content(res, 'content.txt')
     elif method == 'singlefile':
-        return get_file_result_content(res, '')
+        return get_file_result_content(res,'',use_pwd=True)
     elif method == 'dom':
         return get_file_result_content(res,'',use_pwd=True)
     elif method == 'wget':

From 385daf9af8ad203ff03f50b5d9cb7d44c953522e Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 22:01:49 -0500
Subject: [PATCH 142/151] save the url as title for staticfiles or non html
 files

---
 archivebox/extractors/title.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/archivebox/extractors/title.py b/archivebox/extractors/title.py
index 816c0484..194c57ad 100644
--- a/archivebox/extractors/title.py
+++ b/archivebox/extractors/title.py
@@ -62,9 +62,6 @@ class TitleParser(HTMLParser):
 
 @enforce_types
 def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
-        return False
-
     # if link already has valid title, skip it
     if not overwrite and link.title and not link.title.lower().startswith('http'):
         return False
@@ -113,7 +110,11 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
                                         timestamp=link.timestamp)\
                                 .update(title=output)
         else:
-            raise ArchiveError('Unable to detect page title')
+            # if no content was returned, dont save a title (because it might be a temporary error)
+            if not html:
+                raise ArchiveError('Unable to detect page title')
+            # output = html[:128]       # use first bit of content as the title
+            output = link.base_url      # use the filename as the title (better UX)
     except Exception as err:
         status = 'failed'
         output = err

From e6fa16e13a24e0d6146398f3556133d97ce20156 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 22:02:11 -0500
Subject: [PATCH 143/151] only chmod wget output if it exists

---
 archivebox/extractors/wget.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index 33529e4c..54b631f9 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -105,7 +105,12 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
             if b'ERROR 500: Internal Server Error' in result.stderr:
                 raise ArchiveError('500 Internal Server Error', hints)
             raise ArchiveError('Wget failed or got an error from the server', hints)
-        chmod_file(output, cwd=str(out_dir))
+        
+        if (out_dir / output).exists():
+            chmod_file(output, cwd=str(out_dir))
+        else:
+            print(f'          {out_dir}/{output}')
+            raise ArchiveError('Failed to find wget output after running', hints)
     except Exception as err:
         status = 'failed'
         output = err

From 846c966c4d75929a5450e546d27e1e417a5e13de Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 22:02:39 -0500
Subject: [PATCH 144/151] use globbing to find wget output path

---
 archivebox/extractors/wget.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/archivebox/extractors/wget.py b/archivebox/extractors/wget.py
index 54b631f9..fac212c2 100644
--- a/archivebox/extractors/wget.py
+++ b/archivebox/extractors/wget.py
@@ -134,9 +134,7 @@ def wget_output_path(link: Link) -> Optional[str]:
 
     See docs on wget --adjust-extension (-E)
     """
-    if is_static_file(link.url):
-        return without_scheme(without_fragment(link.url))
-
+    
     # Wget downloads can save in a number of different ways depending on the url:
     #    https://example.com
     #       > example.com/index.html
@@ -187,7 +185,7 @@ def wget_output_path(link: Link) -> Optional[str]:
                 last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
                 for file_present in search_dir.iterdir():
                     if file_present == last_part_of_url:
-                        return str(search_dir / file_present)
+                        return str((search_dir / file_present).relative_to(link.link_dir))
 
         # Move up one directory level
         search_dir = search_dir.parent
@@ -195,10 +193,16 @@ def wget_output_path(link: Link) -> Optional[str]:
         if str(search_dir) == link.link_dir:
             break
 
-
+    # check for staticfiles
+    base_url = without_scheme(without_fragment(link.url))
+    domain_dir = Path(domain(link.url).replace(":", "+"))
+    files_within = list((Path(link.link_dir) / domain_dir).glob('**/*.*'))
+    if files_within:
+        return str((domain_dir / files_within[-1]).relative_to(link.link_dir))
     
-    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
-    if not search_dir.is_dir():
-        return str(search_dir.relative_to(link.link_dir))
+    # fallback to just the domain dir
+    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
+    if search_dir.is_dir():
+        return domain(link.url).replace(":", "+")
 
     return None

From 15e87353bd83fcc12e1086fbcce308a249a7b351 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 22:03:59 -0500
Subject: [PATCH 145/151] only show archive.org if enabled

---
 archivebox/index/html.py   | 2 ++
 archivebox/index/schema.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index cff50085..c8b9d07e 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -23,6 +23,7 @@ from ..config import (
     GIT_SHA,
     FOOTER_INFO,
     HTML_INDEX_FILENAME,
+    SAVE_ARCHIVE_DOT_ORG,
 )
 
 MAIN_INDEX_TEMPLATE = 'static_index.html'
@@ -103,6 +104,7 @@ def link_details_template(link: Link) -> str:
         'status': 'archived' if link.is_archived else 'not yet archived',
         'status_color': 'success' if link.is_archived else 'danger',
         'oldest_archive_date': ts_to_date(link.oldest_archive_date),
+        'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
     })
 
 @enforce_types
diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py
index 7e2c784d..7501da3a 100644
--- a/archivebox/index/schema.py
+++ b/archivebox/index/schema.py
@@ -412,6 +412,8 @@ class Link:
         """predict the expected output paths that should be present after archiving"""
 
         from ..extractors.wget import wget_output_path
+        # TODO: banish this awful duplication from the codebase and import these
+        # from their respective extractor files
         canonical = {
             'index_path': 'index.html',
             'favicon_path': 'favicon.ico',

From 54c53316939cfe6a1e6dbece64eff16f6061b5a5 Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 22:04:14 -0500
Subject: [PATCH 146/151] check for output existance when rendering files icons

---
 archivebox/index/html.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/archivebox/index/html.py b/archivebox/index/html.py
index c8b9d07e..5eba0959 100644
--- a/archivebox/index/html.py
+++ b/archivebox/index/html.py
@@ -140,22 +140,22 @@ def snapshot_icons(snapshot) -> str:
     exclude = ["favicon", "title", "headers", "archive_org"]
     # Missing specific entry for WARC
 
-    extractor_items = defaultdict(lambda: None)
+    extractor_outputs = defaultdict(lambda: None)
     for extractor, _ in EXTRACTORS:
         for result in archive_results:
-            if result.extractor == extractor:
-                extractor_items[extractor] = result
+            if result.extractor == extractor and result:
+                extractor_outputs[extractor] = result
 
     for extractor, _ in EXTRACTORS:
         if extractor not in exclude:
-            exists = False
-            if extractor_items[extractor] is not None:
-                outpath = (Path(path) / canon[f"{extractor}_path"])
-                if outpath.is_dir():
+            outpath = extractor_outputs[extractor] and extractor_outputs[extractor].output
+            if outpath:
+                outpath = (Path(path) / outpath)
+                if outpath.is_file():
+                    exists = True
+                elif outpath.is_dir():
                     exists = any(outpath.glob('*.*'))
-                elif outpath.is_file():
-                    exists = outpath.stat().st_size > 100
-            output += format_html(output_template, path, canon[f"{extractor}_path"], str(exists),
+            output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(outpath)),
                                          extractor, icons.get(extractor, "?"))
         if extractor == "wget":
             # warc isn't technically it's own extractor, so we have to add it after wget

From 560d3103a89b418dadced6e4f68eb37a3e674c4d Mon Sep 17 00:00:00 2001
From: Nick Sweeting 
Date: Sat, 30 Jan 2021 22:04:24 -0500
Subject: [PATCH 147/151] cleanup snapshot detail page UI

---
 archivebox/templates/core/snapshot.html | 211 +++++++++++++++---------
 1 file changed, 133 insertions(+), 78 deletions(-)

diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html
index b1edcfe0..ebf2385a 100644
--- a/archivebox/templates/core/snapshot.html
+++ b/archivebox/templates/core/snapshot.html
@@ -33,7 +33,7 @@
             }
             .nav > div {
                 min-height: 30px;
-                margin: 8px 0px;
+                line-height: 1.3;
             }
             .header-top a {
                 text-decoration: none;
@@ -68,6 +68,11 @@
                 vertical-align: -2px;
                 margin-right: 4px;
             }
+            .header-toggle {
+                line-height: 14px;
+                font-size: 70px;
+                vertical-align: -8px;
+            }
             
             .info-row {
                 margin-top: 2px;
@@ -76,24 +81,30 @@
             .info-row .alert {
                 margin-bottom: 0px;
             }
-            .card {
+            .header-bottom-frames .card {
                 overflow: hidden;
                 box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
                 margin-top: 10px;
+                border: 1px solid rgba(0,0,0,3);
+                border-radius: 14px;
+                background-color: black;
             }
             .card h4 {
                 font-size: 1.4vw;
             }
             .card-body {
-                font-size: 1vw;
-                padding-top: 1.2vw;
-                padding-left: 1vw;
-                padding-right: 1vw;
-                padding-bottom: 1vw;
+                font-size: 15px;
+                padding: 13px 10px;
+                padding-bottom: 6px;
+                /* padding-left: 3px; */
+                /* padding-right: 3px; */
+                /* padding-bottom: 3px; */
                 line-height: 1.1;
                 word-wrap: break-word;
                 max-height: 102px;
                 overflow: hidden;
+                background-color: #1a1a1a;
+                color: #d3d3d3;
             }
             .card-title {
                 margin-bottom: 4px;
@@ -126,7 +137,7 @@
                 border-top: 3px solid #aa1e55;
             }
             .card.selected-card {
-                border: 2px solid orange;
+                border: 1px solid orange;
                 box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
             }
             .iframe-large {
@@ -174,12 +185,13 @@
                 width: 98%;
                 border: 1px solid rgba(0,0,0,0.2);
                 box-shadow: 4px 4px 4px rgba(0,0,0,0.2);
-                margin-top: 5px;
+                margin-top: 0px;
             }
             .header-bottom-info {
                 color: #6f6f6f;
-                padding-top: 8px;
-                padding-bottom: 13px;
+                padding-top: 0px;
+                padding-bottom: 0px;
+                margin: 0px -15px;
             }
 
             .header-bottom-info > div {
@@ -203,12 +215,30 @@
                 margin-top: 5px;
             }
             .header-bottom-frames .card-title {
-                padding-bottom: 0px;
-                font-size: 1.2vw;
+                width: 100%;
+                text-align: center;
+                font-size: 18px;
                 margin-bottom: 5px;
+                display: inline-block;
+                color: #d3d3d3;
+                font-weight: 200;
+                vertical-align: 0px;
+                margin-top: -6px;
             }
             .header-bottom-frames .card-text {
+                width: 100%;
+                text-align: center;
                 font-size: 0.9em;
+                display: inline-block;
+                position: relative;
+                top: -11px;
+            }
+            .card-text code {
+                padding: .2rem .4rem;
+                font-size: 90%;
+                color: #bd4147;
+                background-color: #101010;
+                border-radius: .25rem;
             }
 
             @media(max-width: 1092px) {
@@ -247,7 +277,7 @@
                         
                     

                     

-                        Favicon
+                        Favicon
                           
                         {{title}}
                           
@@ -316,120 +346,145 @@
                     

                 

                 

-                    

+                    

                         

-                          
-                          

-                                
-                                    
-                                
-                                
Wget > WARC

-                                
archive/{{domain}}

-                            

-                        

-                    

-                    

-                        

                             
                             

-                                
-                                    
+                                
+                                    
./singlefile.html

                                 
                                 
Chrome > SingleFile

-                                
archive/singlefile.html

                           

                         

                     

-                    

-                        

-                            
-                            

-                                
-                                    
-                                
-                                
Archive.Org

-                                
web.archive.org/web/...

-                          

-                        

-                    

-                    

-                        

-                            
-                            

-                                
-                                    
-                                
-                                
Original

-                                
{{domain}}

-                          

-                        

-                    

-                    

-                    

+                    

                         

                             
                             

-                                
-                                    
+                                
+                                    
./output.pdf

                                 
                                 
Chrome > PDF

-                                
archive/output.pdf

                           

                         

                     

-                    

+                    

                         

-                            
+                            
                             

-                                
-                                    
+                                
+                                    
./screenshot.png

                                 
                                 
Chrome > Screenshot

-                                
archive/screenshot.png

                           

                         

                     

-                    

+                    

+                        

+                          
+                          

+                                
+                                    
./{{domain}}

+                                
+                                
Wget > HTML

+                            

+                        

+                    

+                    {% if SAVE_ARCHIVE_DOT_ORG %}
+                    

+                        

+                            
+                            

+                                
+                                    
🌐 web.archive.org/web/...

+                                
+                                
Archive.Org

+                          

+                        

+                    

+                    {% endif %}
+                    

+                        

+                            
+                            

+                                
+                                    
🌐 {{domain}}

+                                
+                                
Original

+                          

+                        

+                    

+                    

+                        

+                            
+                            

+                                
+                                    
./headers.json

+                                
+                                
Headers

+                          

+                        

+                    

+                    

                         

                             
                             

-                                
-                                    
+                                
+                                    
./output.html

                                 
                                 
Chrome > HTML

-                                
archive/output.html

                           

                         

                     

-                    

+                    

                         

                             
                             

-                                
-                                    
+                                
+                                    
./readability/content.html

                                 
                                 
Readability

-                                
archive/readability/...

                           

                         

                     

                     

-                    

+                    

                         

                             
                             

-                                
-                                    
+                                
+                                    
./mercury/content.html

                                 
-                                
mercury

-                                
archive/mercury/...

+                                
Mercury

+                          

+                        

+                    

+                    

+                        

+                            
+                            

+                                
+                                    
./media/*.mp4

+                                
+                                
Media

+                          

+                        

+                    

+                    

+                        

+                            
+                            

+                                
+                                    
./git/*.git

+                                
+                                
Git

                           

                         

                     

                 

             

         
-        
+