From 562fd5a3bb9b29a2d93347a17eeb56f61f621a69 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 31 Oct 2023 03:58:24 -0700 Subject: [PATCH] revert back to python-3.11 base image and add cache dirs for massive build speed increase --- Dockerfile | 55 ++++++++++++++++++++++++++++-------------------- pdm.lock | 6 +++--- requirements.txt | 2 +- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 553bf4c4..1213a7cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ # Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/ -FROM debian:bookworm-backports +FROM python:3.11-slim-bookworm LABEL name="archivebox" \ maintainer="Nick Sweeting " \ @@ -98,8 +98,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ # https://docs.linuxserver.io/general/understanding-puid-and-pgid # Install system apt dependencies (adding backports to access more recent apt updates) -RUN echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \ - # && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \ +RUN --mount=type=cache,target=/var/cache/apt \ + echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \ + && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \ && mkdir -p /etc/apt/keyrings \ && apt-get update -qq \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \ @@ -114,7 +115,8 @@ RUN echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \ ######### Language Environments #################################### # Install Node environment -RUN echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \ +RUN --mount=type=cache,target=/var/cache/apt \ + echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \ && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ && apt-get update -qq \ @@ -131,30 +133,34 @@ RUN echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \ ) | tee -a /VERSION.txt # Install Python environment -RUN echo "[+] Installing Python $PYTHON_VERSION environment in $GLOBAL_VENV and $APP_VENV..." \ +RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \ + echo "[+] Installing Python $PYTHON_VERSION environment in $GLOBAL_VENV and $APP_VENV..." \ && apt-get update -qq \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \ - python3 python3-pip python3-setuptools python3-wheel python3-venv python3-pdm python-dev-is-python3 \ + python3 python3-pip python3-setuptools python3-wheel python3-venv python-dev-is-python3 \ #python3-ldap libldap2-dev libsasl2-dev libssl-dev python3-msgpack \ && rm -rf /var/lib/apt/lists/* \ # tell PDM to allow using global system python site packages - && rm /usr/lib/python3*/EXTERNALLY-MANAGED \ + # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \ # create global virtual environment GLOBAL_VENV to use (better than using pip install --global) - && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \ + # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \ + # && python3 -m venv --system-site-packages $GLOBAL_VENV \ + # && python3 -m venv $GLOBAL_VENV \ # install global dependencies / python build dependencies in GLOBAL_VENV - && $GLOBAL_VENV/bin/pip install --break-system-packages --upgrade pip pdm setuptools wheel \ + && pip install --upgrade pip setuptools wheel \ # Save version info && ( \ which python3 && python3 --version | grep " $PYTHON_VERSION" \ - && which pip3 && pip3 --version \ - && which pdm && pdm --version \ + && which pip && pip --version \ + # && which pdm && pdm --version \ && echo -e '\n\n' \ ) | tee -a /VERSION.txt ######### Extractor Dependencies ################################## # Install apt dependencies -RUN echo "[+] Installing APT extractor dependencies globally using apt..." \ +RUN --mount=type=cache,target=/var/cache/apt \ + echo "[+] Installing APT extractor dependencies globally using apt..." \ && apt-get update -qq \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \ curl wget git yt-dlp ffmpeg ripgrep \ @@ -173,11 +179,12 @@ RUN echo "[+] Installing APT extractor dependencies globally using apt..." \ ) | tee -a /VERSION.txt # Install chromium browser using playwright -RUN echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \ +RUN --mount=type=cache,target=/var/cache/apt \ + echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \ && apt-get update -qq \ && if [[ "$TARGETPLATFORM" == "linux/amd64" || "$TARGETPLATFORM" == "linux/arm64" ]]; then \ # install Chromium using playwright - $GLOBAL_VENV/bin/pip install playwright \ + pip install playwright \ && $GLOBAL_VENV/bin/playwright install --with-deps chromium \ && export CHROME_BINARY="$($GLOBAL_VENV/bin/python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \ else \ @@ -199,8 +206,9 @@ RUN echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PAT # Install Node dependencies WORKDIR "$CODE_DIR" COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/" -RUN echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \ - && npm ci --prefer-offline --no-audit \ +RUN --mount=type=cache,target=/root/.npm \ + echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \ + && npm ci --prefer-offline --no-audit --cache /root/.npm \ && ( \ which node && node --version \ && which npm && npm version \ @@ -211,18 +219,18 @@ RUN echo "[+] Installing NPM extractor dependencies from package.json into $NODE # Install ArchiveBox Python dependencies WORKDIR "$CODE_DIR" -COPY --chown=root:root --chmod=755 "./pyproject.toml" "./pdm.lock" "requirements.txt" "$CODE_DIR/" -RUN --mount=type=cache,target=/root/.cache/pip \ - && echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt into $GLOBAL_VENV..." \ +COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR/" +RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \ + echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt into $GLOBAL_VENV..." \ && apt-get update -qq \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \ build-essential libssl-dev libldap2-dev libsasl2-dev \ - && ln -s "$GLOBAL_VENV" "$APP_VENV" \ + # && ln -s "$GLOBAL_VENV" "$APP_VENV" \ # && pdm use --venv in-project \ # && pdm run python -m ensurepip \ # && pdm sync --fail-fast --no-editable --group :all --no-self \ # && pdm export -o requirements.txt --without-hashes \ - && source $GLOBAL_VENV/bin/activate \ + # && source $GLOBAL_VENV/bin/activate \ && pip install -r requirements.txt \ && apt-get purge -y \ build-essential libssl-dev libldap2-dev libsasl2-dev \ @@ -232,7 +240,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Install ArchiveBox Python package from source COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" -RUN echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \ +RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \ + echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \ && apt-get update -qq \ # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi && if [[ "$TARGETPLATFORM" == "linux/arm/v7" ]]; then \ @@ -243,7 +252,7 @@ RUN echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \ procps; \ fi \ # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies - && $GLOBAL_VENV/bin/pip3 install --break-system-packages -e "$CODE_DIR"[sonic,ldap] \ + && pip install --break-system-packages -e "$CODE_DIR"[sonic,ldap] \ # save docker image size and always remove compilers / build tools after building is complete && apt-get purge -y build-essential \ && apt-get autoremove -y \ diff --git a/pdm.lock b/pdm.lock index 2a8610fa..503c9133 100644 --- a/pdm.lock +++ b/pdm.lock @@ -443,7 +443,7 @@ files = [ [[package]] name = "ipython" -version = "8.17.1" +version = "8.17.2" requires_python = ">=3.9" summary = "IPython: Productive Interactive Computing" dependencies = [ @@ -461,8 +461,8 @@ dependencies = [ "typing-extensions; python_version < \"3.10\"", ] files = [ - {file = "ipython-8.17.1-py3-none-any.whl", hash = "sha256:b4510d0a163c89c78cf81be68acc841dde4a8290b6ee3f3c9578ccdd2574553c"}, - {file = "ipython-8.17.1.tar.gz", hash = "sha256:9e12020f4bf74f08631c0f033bb580e074fbe36c64903195f3e63b9c0a986cbe"}, + {file = "ipython-8.17.2-py3-none-any.whl", hash = "sha256:1e4d1d666a023e3c93585ba0d8e962867f7a111af322efff6b9c58062b3e5444"}, + {file = "ipython-8.17.2.tar.gz", hash = "sha256:126bb57e1895594bb0d91ea3090bbd39384f6fe87c3d57fd558d0670f50339bb"}, ] [[package]] diff --git a/requirements.txt b/requirements.txt index 0270635f..c935bdd7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ django-extensions==3.1.5 executing==2.0.1 greenlet==3.0.0 idna==3.4 -ipython==8.17.1 +ipython==8.17.2 jedi==0.19.1 matplotlib-inline==0.1.6 mutagen==1.47.0