From 2ad776ac544e79a9dec926da2e33fad24fe51047 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 15:05:50 -0400 Subject: [PATCH 01/10] Explode compile step into multistage build for parallel compilation --- Dockerfile | 77 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b2224323..7f1f91fb3 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM composer:2.0 as step0 +FROM composer:2.0 as composer ARG TESTING=false ENV TESTING=$TESTING @@ -12,7 +12,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist \ `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi` -FROM php:8.0-cli-alpine as step1 +FROM php:8.0-cli-alpine as compile ARG DEBUG=false ENV DEBUG=$DEBUG @@ -40,43 +40,24 @@ RUN \ RUN docker-php-ext-install sockets +FROM compile AS redis RUN \ # Redis Extension git clone --depth 1 --branch $PHP_REDIS_VERSION https://github.com/phpredis/phpredis.git && \ cd phpredis && \ phpize && \ ./configure && \ - make && make install && \ - cd .. && \ - ## Swoole Extension + make && make install + +## Swoole Extension +FROM compile AS swoole +RUN \ git clone --depth 1 --branch $PHP_SWOOLE_VERSION https://github.com/swoole/swoole-src.git && \ cd swoole-src && \ phpize && \ ./configure --enable-http2 && \ make && make install && \ - cd .. && \ - ## Imagick Extension - git clone --depth 1 --branch $PHP_IMAGICK_VERSION https://github.com/imagick/imagick && \ - cd imagick && \ - phpize && \ - ./configure && \ - make && make install && \ - cd .. && \ - ## YAML Extension - git clone --depth 1 --branch $PHP_YAML_VERSION https://github.com/php/pecl-file_formats-yaml && \ - cd pecl-file_formats-yaml && \ - phpize && \ - ./configure && \ - make && make install && \ - cd .. && \ - ## Maxminddb extension - git clone --depth 1 --branch $PHP_MAXMINDDB_VERSION https://github.com/maxmind/MaxMind-DB-Reader-php.git && \ - cd MaxMind-DB-Reader-php && \ - cd ext && \ - phpize && \ - ./configure && \ - make && make install && \ - cd ../.. + cd .. ## Swoole Debugger setup RUN if [ "$DEBUG" == "true" ]; then \ @@ -90,6 +71,34 @@ RUN if [ "$DEBUG" == "true" ]; then \ cd ..;\ fi +## Imagick Extension +FROM compile AS imagick +RUN \ + git clone --depth 1 --branch $PHP_IMAGICK_VERSION https://github.com/imagick/imagick && \ + cd imagick && \ + phpize && \ + ./configure && \ + make && make install + +## YAML Extension +FROM compile AS yaml +RUN \ + git clone --depth 1 --branch $PHP_YAML_VERSION https://github.com/php/pecl-file_formats-yaml && \ + cd pecl-file_formats-yaml && \ + phpize && \ + ./configure && \ + make && make install + +## Maxminddb extension +FROM compile AS maxmind +RUN \ + git clone --depth 1 --branch $PHP_MAXMINDDB_VERSION https://github.com/maxmind/MaxMind-DB-Reader-php.git && \ + cd MaxMind-DB-Reader-php && \ + cd ext && \ + phpize && \ + ./configure && \ + make && make install + FROM php:8.0-cli-alpine as final LABEL maintainer="team@appwrite.io" @@ -184,12 +193,12 @@ RUN \ WORKDIR /usr/src/code -COPY --from=step0 /usr/local/src/vendor /usr/src/code/vendor -COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yasd.so* /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imagick.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor +COPY --from=swoole /usr/local/lib/php/extensions/no-debug-non-zts-20200930/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yasd.so* /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=redis /usr/local/lib/php/extensions/no-debug-non-zts-20200930/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imagick.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ # Add Source Code COPY ./app /usr/src/code/app From 87a9c97df67ae58ae3595ecccac39ca972b62851 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 15:06:21 -0400 Subject: [PATCH 02/10] Pull and build in parallel --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3381c3ecb..6df01b25e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,11 +26,15 @@ before_install: - docker buildx create --use - chmod -R u+x ./.travis-ci - export COMPOSE_INTERACTIVE_NO_CLI=1 +- export DOCKER_BUILDKIT=1 +- export COMPOSE_DOCKER_CLI_BUILD=1 # Only pass a single runtime for CI stability - echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env install: -- docker-compose up -d --build +- docker-compose pull +- docker-compose build +- docker-compose up -d - sleep 10 script: From 67148acab19cf22e521ca8ca5231de234ea2194f Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 15:24:10 -0400 Subject: [PATCH 03/10] Set envs inline to ensure compose uses buildkit --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6df01b25e..6274d66ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,14 +26,12 @@ before_install: - docker buildx create --use - chmod -R u+x ./.travis-ci - export COMPOSE_INTERACTIVE_NO_CLI=1 -- export DOCKER_BUILDKIT=1 -- export COMPOSE_DOCKER_CLI_BUILD=1 # Only pass a single runtime for CI stability - echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env install: - docker-compose pull -- docker-compose build +- DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain docker-compose build - docker-compose up -d - sleep 10 From 92cf111036d9fb60a8c20fc02210ec9dd353ee47 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 15:39:38 -0400 Subject: [PATCH 04/10] Define environment vars in env block --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6274d66ab..18e9d1aac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,12 @@ notifications: email: - team@appwrite.io +env: + - COMPOSE_INTERACTIVE_NO_CLI=1 + - DOCKER_BUILDKIT=1 + - COMPOSE_DOCKER_CLI_BUILD=1 + - BUILDKIT_PROGRESS=plain + before_install: - curl -fsSL https://get.docker.com | sh - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json @@ -25,13 +31,12 @@ before_install: - docker --version - docker buildx create --use - chmod -R u+x ./.travis-ci -- export COMPOSE_INTERACTIVE_NO_CLI=1 # Only pass a single runtime for CI stability - echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env install: - docker-compose pull -- DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 BUILDKIT_PROGRESS=plain docker-compose build +- docker-compose build - docker-compose up -d - sleep 10 From 395203d9e292f4b9a3ff89f0b6fa470097cd7aec Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 15:39:53 -0400 Subject: [PATCH 05/10] Update docker-compose to version compatible with buildkit --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 18e9d1aac..ede514848 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ notifications: - team@appwrite.io env: + - DOCKER_COMPOSE_VERSION=1.29.2 - COMPOSE_INTERACTIVE_NO_CLI=1 - DOCKER_BUILDKIT=1 - COMPOSE_DOCKER_CLI_BUILD=1 @@ -29,6 +30,11 @@ before_install: echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin fi - docker --version +- sudo rm /usr/local/bin/docker-compose +- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose +- chmod +x docker-compose +- sudo mv docker-compose /usr/local/bin +- docker-compose --version - docker buildx create --use - chmod -R u+x ./.travis-ci # Only pass a single runtime for CI stability From 484bda5c4d37f08f1f188502d80067652aee5fcd Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 15:44:27 -0400 Subject: [PATCH 06/10] Env block not for environment vars --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ede514848..f45b38023 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,13 +12,6 @@ notifications: email: - team@appwrite.io -env: - - DOCKER_COMPOSE_VERSION=1.29.2 - - COMPOSE_INTERACTIVE_NO_CLI=1 - - DOCKER_BUILDKIT=1 - - COMPOSE_DOCKER_CLI_BUILD=1 - - BUILDKIT_PROGRESS=plain - before_install: - curl -fsSL https://get.docker.com | sh - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json @@ -31,12 +24,16 @@ before_install: fi - docker --version - sudo rm /usr/local/bin/docker-compose -- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose +- curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - docker-compose --version - docker buildx create --use - chmod -R u+x ./.travis-ci +- export COMPOSE_INTERACTIVE_NO_CLI +- export DOCKER_BUILDKIT=1 +- export COMPOSE_DOCKER_CLI_BUILD=1 +- export BUILDKIT_PROGRESS=plain # Only pass a single runtime for CI stability - echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env From 2d622db37105d9be7f1d5c453f774dd9df285539 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 9 Sep 2021 12:46:40 -0400 Subject: [PATCH 07/10] Add comments to before_install instructions --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f45b38023..1ad6f1d6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,29 +13,35 @@ notifications: - team@appwrite.io before_install: +# Install latest Docker - curl -fsSL https://get.docker.com | sh +# Enable Buildkit in Docker config - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json - mkdir -p $HOME/.docker - echo '{"experimental":"enabled"}' | sudo tee $HOME/.docker/config.json - sudo service docker start +# Login to increase Docker Hub ratelimit - > if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin fi - docker --version +# Install latest Compose - sudo rm /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - docker-compose --version +# Enable Buildkit - docker buildx create --use -- chmod -R u+x ./.travis-ci - export COMPOSE_INTERACTIVE_NO_CLI - export DOCKER_BUILDKIT=1 - export COMPOSE_DOCKER_CLI_BUILD=1 - export BUILDKIT_PROGRESS=plain # Only pass a single runtime for CI stability - echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env +# Ensure Travis scripts are executable +- chmod -R u+x ./.travis-ci install: - docker-compose pull From ab3a84b32661773d000ba47920854ffe662556f4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 13:36:30 -0400 Subject: [PATCH 08/10] Remove builder after tests since travis reuses build nodes --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1ad6f1d6c..036f11aaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ before_install: - sudo mv docker-compose /usr/local/bin - docker-compose --version # Enable Buildkit -- docker buildx create --use +- docker buildx create --name travis_builder --use - export COMPOSE_INTERACTIVE_NO_CLI - export DOCKER_BUILDKIT=1 - export COMPOSE_DOCKER_CLI_BUILD=1 @@ -57,6 +57,10 @@ script: - docker-compose exec appwrite vars - docker-compose exec appwrite test --debug +after_script: +# travis re-uses their build nodes so clean them up +- docker buildx rm travis_builder + after_failure: - docker-compose logs appwrite From 5a1cb9a7e0dd5e7a9d3b131595f10f67d7b837af Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 17:15:37 -0400 Subject: [PATCH 09/10] Prefetch build images --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 036f11aaa..333b5af64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,10 @@ before_install: install: - docker-compose pull +# Upstream bug causes buildkit pulls to fail so prefetch base images +# https://github.com/moby/moby/issues/41864 +- docker pull composer:2.0 +- docker pull php:8.0-cli-alpine - docker-compose build - docker-compose up -d - sleep 10 From d1d16bd356785b798a13513a93ca65e11391f958 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 16 Sep 2021 08:45:53 -0400 Subject: [PATCH 10/10] Increase travis VM size --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 333b5af64..a4faea5a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: large + language: shell notifications: