From df98d6bb5d6d0edf31090550426bd435739709fd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 2 Jul 2024 12:14:01 +0100 Subject: [PATCH] Updating CouchDB directory to be SQS by default, this image has both Lucene and SQS capabilities so it is backwards compatiable. --- hosting/couchdb/Dockerfile | 12 ++- hosting/couchdb/Dockerfile.v2 | 139 ---------------------------------- hosting/couchdb/runner.sh | 9 ++- hosting/couchdb/runner.v2.sh | 88 --------------------- package.json | 1 - 5 files changed, 16 insertions(+), 233 deletions(-) delete mode 100644 hosting/couchdb/Dockerfile.v2 delete mode 100644 hosting/couchdb/runner.v2.sh diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index ca72153e78..b95fa348f8 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -96,10 +96,13 @@ EXPOSE 5984 4369 9100 CMD ["/opt/couchdb/bin/couchdb"] FROM base as runner +ARG TARGETARCH +ENV TARGETARCH $TARGETARCH ENV COUCHDB_USER admin ENV COUCHDB_PASSWORD admin EXPOSE 5984 +EXPOSE 4984 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ @@ -125,7 +128,12 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./ WORKDIR /opt/couchdb ADD couch/vm.args couch/local.ini ./etc/ +# setup SQS +WORKDIR /opt/sqs +ADD sqs ./ +RUN chmod +x ./install.sh && ./install.sh + WORKDIR / ADD runner.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau -CMD ["./bbcouch-runner.sh"] \ No newline at end of file +RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs +CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/Dockerfile.v2 b/hosting/couchdb/Dockerfile.v2 deleted file mode 100644 index 126742cadb..0000000000 --- a/hosting/couchdb/Dockerfile.v2 +++ /dev/null @@ -1,139 +0,0 @@ -# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile -# -# Everything in this `base` image is adapted from the official `couchdb` image's -# Dockerfile. Only modifications related to upgrading from Debian bullseye to -# bookworm have been included. The `runner` image contains Budibase's -# customisations to the image, e.g. adding Clouseau. -FROM node:20-slim AS base - -# Add CouchDB user account to make sure the IDs are assigned consistently -RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb - -# be sure GPG and apt-transport-https are available and functional -RUN set -ex; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - dirmngr \ - gnupg \ - ; \ - rm -rf /var/lib/apt/lists/* - -# grab tini for signal handling and zombie reaping -# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407 -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends tini; \ - rm -rf /var/lib/apt/lists/*; \ - tini --version - -# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages -ENV GPG_COUCH_KEY \ -# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) - 390EF70BB1EA12B2773962950EE62FB37A00258D -RUN set -eux; \ - apt-get update; \ - apt-get install -y curl; \ - export GNUPGHOME="$(mktemp -d)"; \ - curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \ - gpg --batch --import keys.asc; \ - gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \ - command -v gpgconf && gpgconf --kill all || :; \ - rm -rf "$GNUPGHOME"; \ - apt-key list; \ - apt purge -y --autoremove curl; \ - rm -rf /var/lib/apt/lists/* - -ENV COUCHDB_VERSION 3.3.3 - -RUN . /etc/os-release; \ - echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \ - tee /etc/apt/sources.list.d/couchdb.list >/dev/null - -# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian -RUN set -eux; \ - apt-get update; \ - \ - echo "couchdb couchdb/mode select none" | debconf-set-selections; \ -# we DO want recommends this time - DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ - couchdb="$COUCHDB_VERSION"~bookworm \ - ; \ -# Undo symlinks to /var/log and /var/lib - rmdir /var/lib/couchdb /var/log/couchdb; \ - rm /opt/couchdb/data /opt/couchdb/var/log; \ - mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \ - chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \ - chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \ -# Remove file that sets logging to a file - rm /opt/couchdb/etc/default.d/10-filelog.ini; \ -# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh - find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \ -# Setup directories and permissions for config. Technically these could be 555 and 444 respectively -# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh. - find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \ - find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \ -# only local.d needs to be writable for the docker_entrypoint.sh - chmod -f 0777 /opt/couchdb/etc/local.d; \ -# apt clean-up - rm -rf /var/lib/apt/lists/*; - -# Add configuration -COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/ -# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ - -COPY docker-entrypoint.sh /usr/local/bin -RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat -ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"] - -VOLUME /opt/couchdb/data - -# 5984: Main CouchDB endpoint -# 4369: Erlang portmap daemon (epmd) -# 9100: CouchDB cluster communication port -EXPOSE 5984 4369 9100 -CMD ["/opt/couchdb/bin/couchdb"] - -FROM base as runner -ARG TARGETARCH -ENV TARGETARCH $TARGETARCH - -ENV COUCHDB_USER admin -ENV COUCHDB_PASSWORD admin -EXPOSE 5984 -EXPOSE 4984 - -RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ - wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ - apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \ - apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \ - apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \ - apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \ - rm -rf /var/lib/apt/lists/ - -# setup clouseau -WORKDIR / -RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \ - unzip clouseau-2.21.0-dist.zip && \ - mv clouseau-2.21.0 /opt/clouseau && \ - rm clouseau-2.21.0-dist.zip - -WORKDIR /opt/clouseau -RUN mkdir ./bin -ADD clouseau/clouseau ./bin/ -ADD clouseau/log4j.properties clouseau/clouseau.ini ./ - -# setup CouchDB -WORKDIR /opt/couchdb -ADD couch/vm.args couch/local.ini ./etc/ - -# setup SQS -WORKDIR /opt/sqs -ADD sqs ./ -RUN chmod +x ./install.sh && ./install.sh - -WORKDIR / -ADD runner.v2.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs -CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index aaadee6b43..f8cbe49b8f 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -70,9 +70,12 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & # Start CouchDB. -/docker-entrypoint.sh /opt/couchdb/bin/couchdb & +/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 & -# Wati for CouchDB to start up. +# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues. +/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 & + +# Wait for CouchDB to start up. while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do echo 'Waiting for CouchDB to start...'; sleep 5; @@ -82,4 +85,4 @@ done # function correctly, so we create them here. curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator -sleep infinity \ No newline at end of file +sleep infinity diff --git a/hosting/couchdb/runner.v2.sh b/hosting/couchdb/runner.v2.sh deleted file mode 100644 index f8cbe49b8f..0000000000 --- a/hosting/couchdb/runner.v2.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -DATA_DIR=${DATA_DIR:-/data} -COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7} - -mkdir -p ${DATA_DIR} -mkdir -p ${DATA_DIR}/couch/{dbs,views} -mkdir -p ${DATA_DIR}/search -chown -R couchdb:couchdb ${DATA_DIR}/couch - -echo ${TARGETBUILD} > /buildtarget.txt -if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persistent data & SSH on port 2222 - DATA_DIR="${DATA_DIR:-/home}" - WEBSITES_ENABLE_APP_SERVICE_STORAGE=true - mkdir -p $DATA_DIR/{search,minio,couch} - mkdir -p $DATA_DIR/couch/{dbs,views} - chown -R couchdb:couchdb $DATA_DIR/couch/ - apt update - apt-get install -y openssh-server - echo "root:Docker!" | chpasswd - mkdir -p /tmp - chmod +x /tmp/ssh_setup.sh \ - && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null) - cp /etc/sshd_config /etc/ssh/sshd_config - /etc/init.d/ssh restart - sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini -elif [[ "${TARGETBUILD}" = "single" ]]; then - # In the single image build, the Dockerfile specifies /data as a volume - # mount, so we use that for all persistent data. - sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini -elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then - # We remove the database_dir and view_index_dir settings from the local.ini - # in docker-compose because it will default to /opt/couchdb/data which is what - # our docker-compose was using prior to us switching to using our own CouchDB - # image. - sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini - sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini - sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini -elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then - # In Kubernetes the directory /opt/couchdb/data has a persistent volume - # mount for storing database data. - sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini - - # We remove the database_dir and view_index_dir settings from the local.ini - # in Kubernetes because it will default to /opt/couchdb/data which is what - # our Helm chart was using prior to us switching to using our own CouchDB - # image. - sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini - sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini - - # We remove the -name setting from the vm.args file in Kubernetes because - # it will default to the pod FQDN, which is what's required for clustering - # to work. - sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args -else - # For all other builds, we use /data for persistent data. - sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini -fi - -sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args -sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini - -# Start Clouseau. Budibase won't function correctly without Clouseau running, it -# powers the search API endpoints which are used to do all sorts, including -# populating app grids. -/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & - -# Start CouchDB. -/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 & - -# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues. -/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 & - -# Wait for CouchDB to start up. -while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do - echo 'Waiting for CouchDB to start...'; - sleep 5; -done - -# CouchDB needs the `_users` and `_replicator` databases to exist before it will -# function correctly, so we create them here. -curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users -curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator -sleep infinity diff --git a/package.json b/package.json index 7e5ce25923..6416397c94 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "build:docker:single:sqs": "./scripts/build-single-image-sqs.sh", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 --push ./hosting/couchdb", - "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.3.3-sqs --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run --stream env:multi:enable",