ArchiveBox/.github/workflows/test.yml

name: Run tests
on: [push]

env:
  DOCKER_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False

jobs:
  python_tests:
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ubuntu-22.04]
        # os: [ubuntu-22.04, macos-latest, windows-latest]
        python: [3.11]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
          node-version: 20.10.0

      - name: Setup PDM
        uses: pdm-project/setup-pdm@v3
        with:
          python-version: '3.11'
          cache: true

      ### Install Python & JS Dependencies
      - name: Get pip cache dir
        id: pip-cache
        run: |
          echo "::set-output name=dir::$(pip cache dir)"

      - name: Cache pip
        uses: actions/cache@v3
        id: cache-pip
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python }}-venv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

      - name: Install pip dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel pytest bottle build
          python -m pip install -r requirements.txt
          python -m pip install -e .[sonic,ldap]

      - name: Get npm cache dir
        id: npm-cache
        run: |
          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"

      - name: Cache npm
        uses: actions/cache@v3
        id: cache-npm
        with:
          path: ${{ steps.npm-cache.outputs.dir }}
          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node_modules

      - name: Install npm requirements
        run: |
          npm install
          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV

      ### Run the tests
      - name: Directory listing for debugging
        run: |
          pwd
          ls

      - name: Archivebox version
        run: |
          archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist

  docker_tests:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
      # - uses: satackey/action-docker-layer-caching@v0.0.8

      - name: Build image
        run: |
          docker build . -t "$DOCKER_IMAGE"

      - name: Init data dir
        run: |
          mkdir "${{ github.workspace }}/data"
          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init

      - name: Run test server
        run: |
          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
          docker run --name www-nginx -p 80:80 -d nginx

      - name: Add link
        run: |
          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local

      - name: Add stdin link
        run: |
          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add

      - name: List links
        run: |
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }

      - name: Start docker-compose stack
        run: |
          docker-compose run archivebox init
          docker-compose up -d
          sleep 5
          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'

      - name: Check added urls show up in index
        run: |
          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
          docker-compose down || true