From 0b84c641ff4edc1b3701b697421e4ae74ccaa782 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 27 Jun 2022 12:13:33 +0100 Subject: [PATCH 01/16] WIP: Pipeline for Single Image deploy --- .github/workflows/deploy-omnibus-image.yml | 84 ++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/deploy-omnibus-image.yml diff --git a/.github/workflows/deploy-omnibus-image.yml b/.github/workflows/deploy-omnibus-image.yml new file mode 100644 index 0000000000..a7c0e4caf8 --- /dev/null +++ b/.github/workflows/deploy-omnibus-image.yml @@ -0,0 +1,84 @@ +name: Deploy Budibase Single Container Image to DockerHub +on: + push: + branches: + - "omnibus-action" + - "develop" + - "master" + - "main" +jobs: + build: + name: "build" + runs-on: ubuntu-latest + env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + BRANCH: ${{ github.event.pull_request.head.ref }} + PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + strategy: + matrix: + node-version: [14.x] + steps: + - name: "Checkout" + uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - name: Run Yarn + run: yarn + - name: Run Yarn Bootstrap + run: yarn bootstrap + - name: Runt Yarn Lint + run: yarn lint + - name: Run Yarn Build + run: yarn build + - run: yarn test + env: + CI: true + name: Budibase CI + test: + name: Budibase Single Image Tests + runs-on: ubuntu-latest + env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + BRANCH: ${{ github.event.pull_request.head.ref }} + PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + steps: + - name: Test Code Coverage + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ./packages/server/coverage/clover.xml + name: codecov-umbrella + verbose: true + - name: Test with Cypress + uses: cypress-io/github-action@v2 + with: + install: false + command: yarn test:e2e:ci + deploy: + name: Deploy the Budibase Single Container Image to Docker Hub + runs-on: ubuntu-latest + env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + BRANCH: ${{ github.event.pull_request.head.ref }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} + DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} + PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com + steps: + - name: Login to Docker Hub + run: | + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + - name: Get the latest budibase release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Proxy service docker image + run: | + yarn build:docker:single:image + docker tag budibase $REGISTRY_URL/budibase/single:$RELEASE_TAG + docker push $REGISTRY_URL/budibase/single:$RELEASE_TAG From 7b6fd352c035fa6bd31f7e9d8b21a839748447d9 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 27 Jun 2022 12:35:35 +0100 Subject: [PATCH 02/16] WIP: Pipeline for Single Image deploy --- .github/workflows/deploy-omnibus-image.yml | 24 ++++------------------ 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/workflows/deploy-omnibus-image.yml b/.github/workflows/deploy-omnibus-image.yml index a7c0e4caf8..af9f629eab 100644 --- a/.github/workflows/deploy-omnibus-image.yml +++ b/.github/workflows/deploy-omnibus-image.yml @@ -13,7 +13,10 @@ jobs: env: BASE_BRANCH: ${{ github.event.pull_request.base.ref}} BRANCH: ${{ github.event.pull_request.head.ref }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} + DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com strategy: matrix: node-version: [14.x] @@ -38,14 +41,6 @@ jobs: env: CI: true name: Budibase CI - test: - name: Budibase Single Image Tests - runs-on: ubuntu-latest - env: - BASE_BRANCH: ${{ github.event.pull_request.base.ref}} - BRANCH: ${{ github.event.pull_request.head.ref }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} - steps: - name: Test Code Coverage uses: codecov/codecov-action@v1 with: @@ -58,17 +53,6 @@ jobs: with: install: false command: yarn test:e2e:ci - deploy: - name: Deploy the Budibase Single Container Image to Docker Hub - runs-on: ubuntu-latest - env: - BASE_BRANCH: ${{ github.event.pull_request.base.ref}} - BRANCH: ${{ github.event.pull_request.head.ref }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com - steps: - name: Login to Docker Hub run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD @@ -81,4 +65,4 @@ jobs: run: | yarn build:docker:single:image docker tag budibase $REGISTRY_URL/budibase/single:$RELEASE_TAG - docker push $REGISTRY_URL/budibase/single:$RELEASE_TAG + docker push $REGISTRY_URL/budibase/single:$RELEASE_TAG From e54583f689ae6380d41ab2077a4645c0a8e0c86f Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 27 Jun 2022 13:43:54 +0100 Subject: [PATCH 03/16] WIP: Pipeline for Single Image deploy --- ...ibus-image.yml => deploy-single-image.yml} | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) rename .github/workflows/{deploy-omnibus-image.yml => deploy-single-image.yml} (71%) diff --git a/.github/workflows/deploy-omnibus-image.yml b/.github/workflows/deploy-single-image.yml similarity index 71% rename from .github/workflows/deploy-omnibus-image.yml rename to .github/workflows/deploy-single-image.yml index af9f629eab..8a66c686b7 100644 --- a/.github/workflows/deploy-omnibus-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -6,17 +6,17 @@ on: - "develop" - "master" - "main" +env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + BRANCH: ${{ github.event.pull_request.head.ref }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} + DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} + PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com jobs: build: name: "build" runs-on: ubuntu-latest - env: - BASE_BRANCH: ${{ github.event.pull_request.base.ref}} - BRANCH: ${{ github.event.pull_request.head.ref }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com strategy: matrix: node-version: [14.x] @@ -41,6 +41,11 @@ jobs: env: CI: true name: Budibase CI + test: + name: "test" + runs-on: ubuntu-latest + needs: build + steps: - name: Test Code Coverage uses: codecov/codecov-action@v1 with: @@ -53,16 +58,22 @@ jobs: with: install: false command: yarn test:e2e:ci + deploy: + name: "deploy" + runs-on: ubuntu-latest + needs: test + steps: - name: Login to Docker Hub run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - name: Get the latest budibase release version + - name: Get the latest release version id: version run: | release_version=$(cat lerna.json | jq -r '.version') + echo $release_version echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - name: Tag and release Proxy service docker image run: | yarn build:docker:single:image - docker tag budibase $REGISTRY_URL/budibase/single:$RELEASE_TAG - docker push $REGISTRY_URL/budibase/single:$RELEASE_TAG + docker tag budibase budibase/single:$RELEASE_TAG + docker push budibase/single:$RELEASE_TAG From 9ea1507bc889bcdab85f5d60b33da703edeecc08 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 27 Jun 2022 14:17:04 +0100 Subject: [PATCH 04/16] WIP: Pipeline for Single Image deploy --- .github/workflows/deploy-single-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index 8a66c686b7..2013b4b58f 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -45,6 +45,9 @@ jobs: name: "test" runs-on: ubuntu-latest needs: build + strategy: + matrix: + node-version: [14.x] steps: - name: Test Code Coverage uses: codecov/codecov-action@v1 From fa5a423a52a0a689c56e25508de21f22329df1ac Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 27 Jun 2022 14:37:46 +0100 Subject: [PATCH 05/16] WIP: Pipeline for Single Image deploy --- .github/workflows/deploy-single-image.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index 2013b4b58f..cbdc63cb3c 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -9,6 +9,7 @@ on: env: BASE_BRANCH: ${{ github.event.pull_request.base.ref}} BRANCH: ${{ github.event.pull_request.head.ref }} + CI: true DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} @@ -37,18 +38,8 @@ jobs: run: yarn lint - name: Run Yarn Build run: yarn build - - run: yarn test - env: - CI: true - name: Budibase CI - test: - name: "test" - runs-on: ubuntu-latest - needs: build - strategy: - matrix: - node-version: [14.x] - steps: + - name: Yarn Test + run: yarn test - name: Test Code Coverage uses: codecov/codecov-action@v1 with: @@ -61,11 +52,6 @@ jobs: with: install: false command: yarn test:e2e:ci - deploy: - name: "deploy" - runs-on: ubuntu-latest - needs: test - steps: - name: Login to Docker Hub run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD From 405e5919c4e552066b61df535b2ec2ad4092eec2 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 27 Jun 2022 16:27:19 +0100 Subject: [PATCH 06/16] WIP: Pipeline for Single Image deploy tag --- .github/workflows/deploy-single-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index cbdc63cb3c..16b3917a2f 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -64,5 +64,5 @@ jobs: - name: Tag and release Proxy service docker image run: | yarn build:docker:single:image - docker tag budibase budibase/single:$RELEASE_TAG - docker push budibase/single:$RELEASE_TAG + docker tag budibase budibase/budibase:$RELEASE_TAG + docker push budibase/budibase:$RELEASE_TAG From 6006d5eb170ba565919fa59634f62400f828a6f8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 5 Jul 2022 19:56:47 +0100 Subject: [PATCH 07/16] Adding correct release tag --- .github/workflows/deploy-single-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index 16b3917a2f..1ed739f049 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -61,8 +61,8 @@ jobs: release_version=$(cat lerna.json | jq -r '.version') echo $release_version echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Proxy service docker image + - name: Tag and release Budibase service docker image run: | yarn build:docker:single:image - docker tag budibase budibase/budibase:$RELEASE_TAG - docker push budibase/budibase:$RELEASE_TAG + docker tag budibase budibase/budibase:v${{ env.RELEASE_VERSION }} + docker push budibase/budibase:v${{ env.RELEASE_VERSION }} From 0da424b5c683a3aa0163f101c4a5c530e42ab19c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Jul 2022 16:33:40 +0100 Subject: [PATCH 08/16] Adding missing environment variable. --- hosting/single/Dockerfile | 3 ++- hosting/single/test.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 5e1b0b1374..b338201a20 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -50,7 +50,8 @@ ENV \ SELF_HOSTED=1 \ TARGETBUILD=$TARGETBUILD \ WORKER_PORT=4002 \ - WORKER_URL=http://localhost:4002 + WORKER_URL=http://localhost:4002 \ + APPS_URL=http://localhost:4001 # install base dependencies RUN apt-get update && \ diff --git a/hosting/single/test.sh b/hosting/single/test.sh index c7ef53f994..8830426a47 100755 --- a/hosting/single/test.sh +++ b/hosting/single/test.sh @@ -1,4 +1,4 @@ #!/bin/bash -id=$(docker run -t -d -p 80:80 budibase:latest) +id=$(docker run -t -d -p 8080:80 budibase:latest) docker exec -it $id bash docker kill $id From 216298c74b1c407211d7837a004e8139fdd8fa67 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Jul 2022 16:42:38 +0100 Subject: [PATCH 09/16] Fixing issue which can block accessing platform - if error scenario occurs but admin user partially created. --- packages/worker/src/api/controllers/global/users.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index f84a44098a..5e3b3accb4 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -60,11 +60,13 @@ export const adminUser = async (ctx: any) => { tenantId, } try { + // always bust checklist beforehand, if an error occurs but can proceed, don't get + // stuck in a cycle + await cache.bustCache(cache.CacheKeys.CHECKLIST) const finalUser = await users.save(user, { hashPassword, requirePassword, }) - await cache.bustCache(cache.CacheKeys.CHECKLIST) // events let account: CloudAccount | undefined From 7bedf77f6fabdc5ea258921a95ec9de8e7d53ecb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Jul 2022 16:53:25 +0100 Subject: [PATCH 10/16] Adding QEMU builder for ARM builds. --- .github/workflows/deploy-single-image.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index 1ed739f049..be24caa228 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -28,6 +28,11 @@ jobs: uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 - name: Install Pro run: yarn install:pro $BRANCH $BASE_BRANCH - name: Run Yarn @@ -62,7 +67,9 @@ jobs: echo $release_version echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - name: Tag and release Budibase service docker image - run: | - yarn build:docker:single:image - docker tag budibase budibase/budibase:v${{ env.RELEASE_VERSION }} - docker push budibase/budibase:v${{ env.RELEASE_VERSION }} + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/arm64 + tags: budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} + file: ./hosting/single/Dockerfile From bd3f405ba46d7743fb7ca3ae0eefbcf6c0ff5a6e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 6 Jul 2022 16:56:05 +0100 Subject: [PATCH 11/16] Removing testing from single image deployment as its already carried out by CI workflow. --- .github/workflows/deploy-single-image.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index be24caa228..e0c0c1f9df 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -43,20 +43,6 @@ jobs: run: yarn lint - name: Run Yarn Build run: yarn build - - name: Yarn Test - run: yarn test - - name: Test Code Coverage - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: ./packages/server/coverage/clover.xml - name: codecov-umbrella - verbose: true - - name: Test with Cypress - uses: cypress-io/github-action@v2 - with: - install: false - command: yarn test:e2e:ci - name: Login to Docker Hub run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD From 555226ece1d62b1d819bdb6f1a5ddfb68312dece Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 7 Jul 2022 14:32:48 +0100 Subject: [PATCH 12/16] Adding push to deploy image. --- .github/workflows/deploy-single-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index e0c0c1f9df..aa1a7ec1f0 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -56,6 +56,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . + push: true platforms: linux/amd64,linux/arm64 tags: budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile From 7fe7dcf8a634fb4a4cdef4ca4601a438d0b5b378 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 7 Jul 2022 16:38:48 +0100 Subject: [PATCH 13/16] Adding login action. --- .github/workflows/deploy-single-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index aa1a7ec1f0..09deeacd2d 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -10,8 +10,6 @@ env: BASE_BRANCH: ${{ github.event.pull_request.base.ref}} BRANCH: ${{ github.event.pull_request.head.ref }} CI: true - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} REGISTRY_URL: registry.hub.docker.com jobs: @@ -44,8 +42,10 @@ jobs: - name: Run Yarn Build run: yarn build - name: Login to Docker Hub - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} - name: Get the latest release version id: version run: | From e1b9f16a55ae05f7db149974bdca41bd54105420 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 7 Jul 2022 18:13:46 +0100 Subject: [PATCH 14/16] Updating tags to sufficient access level. --- .github/workflows/deploy-single-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index 09deeacd2d..4a04bf3f98 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -58,5 +58,5 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 - tags: budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} + tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile From 187f1965e22f6f34b8018fd6710f28f045ee1171 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 8 Jul 2022 14:28:09 +0100 Subject: [PATCH 15/16] Moving files around to make single image build easier to follow, getting all data directories into one place for easy volume management. --- hosting/single/Dockerfile | 12 +++++++----- hosting/single/{ => clouseau}/clouseau | 0 hosting/single/{ => clouseau}/clouseau.ini | 2 +- hosting/single/{ => clouseau}/log4j.properties | 0 hosting/single/couch/local.ini | 5 +++++ hosting/single/{ => couch}/vm.args | 0 hosting/single/{ => nginx}/nginx-default-site.conf | 3 --- hosting/single/{ => nginx}/nginx.conf | 0 hosting/single/runner.sh | 2 +- 9 files changed, 14 insertions(+), 10 deletions(-) rename hosting/single/{ => clouseau}/clouseau (100%) rename hosting/single/{ => clouseau}/clouseau.ini (92%) rename hosting/single/{ => clouseau}/log4j.properties (100%) create mode 100644 hosting/single/couch/local.ini rename hosting/single/{ => couch}/vm.args (100%) rename hosting/single/{ => nginx}/nginx-default-site.conf (99%) rename hosting/single/{ => nginx}/nginx.conf (100%) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index b338201a20..1b48de2849 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -67,8 +67,8 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh & npm install --global yarn pm2 # setup nginx -ADD hosting/single/nginx.conf /etc/nginx -ADD hosting/single/nginx-default-site.conf /etc/nginx/sites-enabled/default +ADD hosting/single/nginx/nginx.conf /etc/nginx +ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default RUN mkdir -p /var/log/nginx && \ touch /var/log/nginx/error.log && \ touch /var/run/nginx.pid @@ -87,13 +87,15 @@ RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clou WORKDIR /opt/clouseau RUN mkdir ./bin -ADD hosting/single/clouseau ./bin/ -ADD hosting/single/log4j.properties hosting/single/clouseau.ini ./ +ADD hosting/single/clouseau/clouseau ./bin/ +ADD hosting/single/clouseau/log4j.properties hosting/single/clouseau/clouseau.ini ./ RUN chmod +x ./bin/clouseau # setup CouchDB WORKDIR /opt/couchdb -ADD hosting/single/vm.args ./etc/ +ADD hosting/single/couch/vm.args hosting/single/couch/local.ini ./etc/ +RUN mkdir -p /data/couch/dbs /data/couch/views +RUN chown couchdb:couchdb /data/couch /data/couch/dbs /data/couch/views # setup minio WORKDIR /minio diff --git a/hosting/single/clouseau b/hosting/single/clouseau/clouseau similarity index 100% rename from hosting/single/clouseau rename to hosting/single/clouseau/clouseau diff --git a/hosting/single/clouseau.ini b/hosting/single/clouseau/clouseau.ini similarity index 92% rename from hosting/single/clouseau.ini rename to hosting/single/clouseau/clouseau.ini index f086cf0398..78e43744e5 100644 --- a/hosting/single/clouseau.ini +++ b/hosting/single/clouseau/clouseau.ini @@ -7,7 +7,7 @@ name=clouseau@127.0.0.1 cookie=monster ; the path where you would like to store the search index files -dir=/opt/couchdb/data/search +dir=/data/search ; the number of search indexes that can be open simultaneously max_indexes_open=500 diff --git a/hosting/single/log4j.properties b/hosting/single/clouseau/log4j.properties similarity index 100% rename from hosting/single/log4j.properties rename to hosting/single/clouseau/log4j.properties diff --git a/hosting/single/couch/local.ini b/hosting/single/couch/local.ini new file mode 100644 index 0000000000..72872a60e1 --- /dev/null +++ b/hosting/single/couch/local.ini @@ -0,0 +1,5 @@ +; CouchDB Configuration Settings + +[couchdb] +database_dir = /data/couch/dbs +view_index_dir = /data/couch/views diff --git a/hosting/single/vm.args b/hosting/single/couch/vm.args similarity index 100% rename from hosting/single/vm.args rename to hosting/single/couch/vm.args diff --git a/hosting/single/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf similarity index 99% rename from hosting/single/nginx-default-site.conf rename to hosting/single/nginx/nginx-default-site.conf index 964313fa73..c0d80a0185 100644 --- a/hosting/single/nginx-default-site.conf +++ b/hosting/single/nginx/nginx-default-site.conf @@ -88,7 +88,4 @@ server { gzip_proxied any; gzip_comp_level 6; gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; - - - } diff --git a/hosting/single/nginx.conf b/hosting/single/nginx/nginx.conf similarity index 100% rename from hosting/single/nginx.conf rename to hosting/single/nginx/nginx.conf diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 6f3d247842..8dc3174993 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,6 +1,6 @@ redis-server --requirepass $REDIS_PASSWORD & /opt/clouseau/bin/clouseau & -/minio/minio server /minio & +/minio/minio server /data/minio & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then From 33e8600d625cdaee8794610d87d70efa989bd0a1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 8 Jul 2022 18:33:07 +0100 Subject: [PATCH 16/16] Updating dockerfile to be friendly to volume usage, as well as pre-filling with uuids rather than static secrets. --- .yarnrc | 1 + hosting/single/Dockerfile | 24 +++++++++++++----------- hosting/single/runner.sh | 28 ++++++++++++++++++++++++++++ packages/server/.yarnrc | 1 + packages/worker/.yarnrc | 1 + 5 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 .yarnrc create mode 100644 packages/server/.yarnrc create mode 100644 packages/worker/.yarnrc diff --git a/.yarnrc b/.yarnrc new file mode 100644 index 0000000000..21fa517e23 --- /dev/null +++ b/.yarnrc @@ -0,0 +1 @@ +network-timeout 100000 diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 1b48de2849..772ae2a8ab 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -34,18 +34,10 @@ ENV \ ARCHITECTURE=amd \ BUDIBASE_ENVIRONMENT=PRODUCTION \ CLUSTER_PORT=80 \ - COUCHDB_PASSWORD=budibase \ - COUCHDB_USER=budibase \ - COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ # CUSTOM_DOMAIN=budi001.custom.com \ DEPLOYMENT_ENVIRONMENT=docker \ - INTERNAL_API_KEY=budibase \ - JWT_SECRET=testsecret \ - MINIO_ACCESS_KEY=budibase \ - MINIO_SECRET_KEY=budibase \ MINIO_URL=http://localhost:9000 \ POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \ - REDIS_PASSWORD=budibase \ REDIS_URL=localhost:6379 \ SELF_HOSTED=1 \ TARGETBUILD=$TARGETBUILD \ @@ -53,9 +45,21 @@ ENV \ WORKER_URL=http://localhost:4002 \ APPS_URL=http://localhost:4001 +# These secret env variables are generated by the runner at startup +# their values can be overriden by the user, they will be written +# to the .env file in the /data directory for use later on +# REDIS_PASSWORD=budibase \ +# COUCHDB_PASSWORD=budibase \ +# COUCHDB_USER=budibase \ +# COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ +# INTERNAL_API_KEY=budibase \ +# JWT_SECRET=testsecret \ +# MINIO_ACCESS_KEY=budibase \ +# MINIO_SECRET_KEY=budibase \ + # install base dependencies RUN apt-get update && \ - apt-get install -y software-properties-common wget nginx && \ + apt-get install -y software-properties-common wget nginx uuid-runtime && \ apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \ apt-get update @@ -94,8 +98,6 @@ RUN chmod +x ./bin/clouseau # setup CouchDB WORKDIR /opt/couchdb ADD hosting/single/couch/vm.args hosting/single/couch/local.ini ./etc/ -RUN mkdir -p /data/couch/dbs /data/couch/views -RUN chown couchdb:couchdb /data/couch /data/couch/dbs /data/couch/views # setup minio WORKDIR /minio diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 8dc3174993..f8c1fc5e56 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,3 +1,31 @@ +#!/bin/bash +declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD") +if [ -f "/data/.env" ]; then + export $(cat /data/.env | xargs) +fi +# first randomise any unset environment variables +for ENV_VAR in "${ENV_VARS[@]}" +do + temp=$(eval "echo \$$ENV_VAR") + if [[ -z "${temp}" ]]; then + eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" + fi +done +if [[ -z "${COUCH_DB_URL}" ]]; then + export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984 +fi +if [ ! -f "/data/.env" ]; then + touch /data/.env + for ENV_VAR in "${ENV_VARS[@]}" + do + temp=$(eval "echo \$$ENV_VAR") + echo "$ENV_VAR=$temp" >> /data/.env + done +fi + +# make these directories in runner, incase of mount +mkdir -p /data/couch/dbs /data/couch/views +chown couchdb:couchdb /data/couch /data/couch/dbs /data/couch/views redis-server --requirepass $REDIS_PASSWORD & /opt/clouseau/bin/clouseau & /minio/minio server /data/minio & diff --git a/packages/server/.yarnrc b/packages/server/.yarnrc new file mode 100644 index 0000000000..21fa517e23 --- /dev/null +++ b/packages/server/.yarnrc @@ -0,0 +1 @@ +network-timeout 100000 diff --git a/packages/worker/.yarnrc b/packages/worker/.yarnrc new file mode 100644 index 0000000000..21fa517e23 --- /dev/null +++ b/packages/worker/.yarnrc @@ -0,0 +1 @@ +network-timeout 100000