Merge pull request #1562 from appwrite/feat-parallel-build

Feat - Parellel builds to reduce build time

.travis.yml

@@ -6,6 +6,9 @@ arch:
 
 os: linux
 
+vm:
+  size: large
+
 language: shell
 
 notifications:
@@ -13,24 +16,44 @@ notifications:
   - team@appwrite.io
 
 before_install:
+# Install latest Docker
 - curl -fsSL https://get.docker.com | sh
+# Enable Buildkit in Docker config
 - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json
 - mkdir -p $HOME/.docker
 - echo '{"experimental":"enabled"}' | sudo tee $HOME/.docker/config.json
 - sudo service docker start
+# Login to increase Docker Hub ratelimit
 - >
   if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then
     echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin
   fi
 - docker --version
-- docker buildx create --use
-- chmod -R u+x ./.travis-ci
-- export COMPOSE_INTERACTIVE_NO_CLI=1
+# Install latest Compose
+- sudo rm /usr/local/bin/docker-compose
+- curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > docker-compose
+- chmod +x docker-compose
+- sudo mv docker-compose /usr/local/bin
+- docker-compose --version
+# Enable Buildkit
+- docker buildx create --name travis_builder --use
+- export COMPOSE_INTERACTIVE_NO_CLI
+- export DOCKER_BUILDKIT=1
+- export COMPOSE_DOCKER_CLI_BUILD=1
+- export BUILDKIT_PROGRESS=plain
 # Only pass a single runtime for CI stability
 - echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env
+# Ensure Travis scripts are executable
+- chmod -R u+x ./.travis-ci
 
 install:
-- docker-compose up -d --build
+- docker-compose pull
+# Upstream bug causes buildkit pulls to fail so prefetch base images
+# https://github.com/moby/moby/issues/41864
+- docker pull composer:2.0
+- docker pull php:8.0-cli-alpine
+- docker-compose build
+- docker-compose up -d
 - sleep 10
 
 script:
@@ -41,6 +64,10 @@ script:
 - docker-compose exec appwrite vars
 - docker-compose exec appwrite test --debug
 
+after_script:
+# travis re-uses their build nodes so clean them up
+- docker buildx rm travis_builder
+
 after_failure:
 - docker-compose logs appwrite
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM composer:2.0 as step0
+FROM composer:2.0 as composer
 
 ARG TESTING=false
 ENV TESTING=$TESTING
@@ -12,7 +12,7 @@ RUN composer update --ignore-platform-reqs --optimize-autoloader \
     --no-plugins --no-scripts --prefer-dist \
     `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi`
 
-FROM php:8.0-cli-alpine as step1
+FROM php:8.0-cli-alpine as compile
 
 ARG DEBUG=false
 ENV DEBUG=$DEBUG
@@ -41,43 +41,24 @@ RUN \
 
 RUN docker-php-ext-install sockets
 
+FROM compile AS redis
 RUN \
   # Redis Extension
   git clone --depth 1 --branch $PHP_REDIS_VERSION https://github.com/phpredis/phpredis.git && \
   cd phpredis && \
   phpize && \
   ./configure && \
-  make && make install && \
-  cd .. && \
-  ## Swoole Extension
+  make && make install
+
+## Swoole Extension
+FROM compile AS swoole
+RUN \
   git clone --depth 1 --branch $PHP_SWOOLE_VERSION https://github.com/swoole/swoole-src.git && \
   cd swoole-src && \
   phpize && \
   ./configure --enable-sockets --enable-http2 --enable-openssl && \
   make && make install && \
-  cd .. && \
-  ## Imagick Extension
-  git clone --depth 1 --branch $PHP_IMAGICK_VERSION https://github.com/imagick/imagick && \
-  cd imagick && \
-  phpize && \
-  ./configure && \
-  make && make install && \
-  cd .. && \
-  ## YAML Extension
-  git clone --depth 1 --branch $PHP_YAML_VERSION https://github.com/php/pecl-file_formats-yaml && \
-  cd pecl-file_formats-yaml && \
-  phpize && \
-  ./configure && \
-  make && make install && \
-  cd .. && \
-  ## Maxminddb extension
-  git clone --depth 1 --branch $PHP_MAXMINDDB_VERSION https://github.com/maxmind/MaxMind-DB-Reader-php.git && \
-  cd MaxMind-DB-Reader-php && \
-  cd ext && \
-  phpize && \
-  ./configure && \
-  make && make install && \
-  cd ../..
+  cd ..
 
 ## Swoole Debugger setup
 RUN if [ "$DEBUG" == "true" ]; then \
@@ -91,6 +72,34 @@ RUN if [ "$DEBUG" == "true" ]; then \
     cd ..;\
   fi
 
+## Imagick Extension
+FROM compile AS imagick
+RUN \
+  git clone --depth 1 --branch $PHP_IMAGICK_VERSION https://github.com/imagick/imagick && \
+  cd imagick && \
+  phpize && \
+  ./configure && \
+  make && make install
+
+## YAML Extension
+FROM compile AS yaml
+RUN \
+  git clone --depth 1 --branch $PHP_YAML_VERSION https://github.com/php/pecl-file_formats-yaml && \
+  cd pecl-file_formats-yaml && \
+  phpize && \
+  ./configure && \
+  make && make install
+
+## Maxminddb extension
+FROM compile AS maxmind
+RUN \
+  git clone --depth 1 --branch $PHP_MAXMINDDB_VERSION https://github.com/maxmind/MaxMind-DB-Reader-php.git && \
+  cd MaxMind-DB-Reader-php && \
+  cd ext && \
+  phpize && \
+  ./configure && \
+  make && make install
+
 FROM php:8.0-cli-alpine as final
 
 LABEL maintainer="team@appwrite.io"
@@ -185,12 +194,12 @@ RUN \
 
 WORKDIR /usr/src/code
 
-COPY --from=step0 /usr/local/src/vendor /usr/src/code/vendor
-COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yasd.so* /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
-COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
-COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imagick.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
-COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
-COPY --from=step1 /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ 
+COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor
+COPY --from=swoole /usr/local/lib/php/extensions/no-debug-non-zts-20200930/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yasd.so* /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
+COPY --from=redis /usr/local/lib/php/extensions/no-debug-non-zts-20200930/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
+COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imagick.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
+COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
+COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
 
 # Add Source Code
 COPY ./app /usr/src/code/app