name: Run tests on: [push] env: DOCKER_IMAGE: archivebox-ci PYTHONIOENCODING: utf-8 PYTHONLEGACYWINDOWSSTDIO: utf-8 USE_COLOR: False jobs: python_tests: runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-22.04] # os: [ubuntu-22.04, macos-latest, windows-latest] python: [3.11] steps: - uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 ### Setup Python & JS Languages - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} architecture: x64 - name: Set up Node JS uses: actions/setup-node@v4 with: node-version: 20.10.0 - name: Setup PDM uses: pdm-project/setup-pdm@v3 with: python-version: '3.11' cache: true ### Install Python & JS Dependencies - name: Get pip cache dir id: pip-cache run: | echo "::set-output name=dir::$(pip cache dir)" - name: Cache pip uses: actions/cache@v3 id: cache-pip with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }} restore-keys: | ${{ runner.os }}-${{ matrix.python }}-venv- - uses: awalsh128/cache-apt-pkgs-action@latest with: packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps version: 1.0 - name: Install pip dependencies run: | python -m pip install --upgrade pip setuptools wheel pytest bottle build python -m pip install -r requirements.txt python -m pip install -e .[sonic,ldap] - name: Get npm cache dir id: npm-cache run: | echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules" - name: Cache npm uses: actions/cache@v3 id: cache-npm with: path: ${{ steps.npm-cache.outputs.dir }} key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }} restore-keys: | ${{ runner.os }}-node_modules - name: Install npm requirements run: | npm install echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV ### Run the tests - name: Directory listing for debugging run: | pwd ls - name: Archivebox version run: | archivebox version - name: Test built package with pytest # TODO: remove this exception for windows once we get tests passing on that platform if: ${{ !contains(matrix.os, 'windows') }} run: | python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist docker_tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again # - uses: satackey/action-docker-layer-caching@v0.0.8 - name: Build image run: | docker build . -t "$DOCKER_IMAGE" - name: Init data dir run: | mkdir "${{ github.workspace }}/data" docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init - name: Run test server run: | sudo bash -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts' docker run --name www-nginx -p 80:80 -d nginx - name: Add link run: | docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local - name: Add stdin link run: | echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add - name: List links run: | docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; } docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; } - name: Start docker-compose stack run: | docker-compose run archivebox init docker-compose up -d sleep 5 curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox' curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django' - name: Check added urls show up in index run: | docker-compose run archivebox add 'http://example.com/#test_docker' --index-only curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker' docker-compose down || true