From 4331e1ef03189adb244fb7d462b7e67121573d45 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 1 Nov 2022 04:55:13 +0000 Subject: [PATCH 1/6] feat: update PR --- app/config/collections.php | 6 +++--- docker-compose.yml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 0214b608e..6fc7ec1e3 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2314,7 +2314,7 @@ $collections = [ '$id' => ID::custom('entrypoint'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 16384, + 'size' => 2048, 'signed' => true, 'required' => false, 'default' => null, @@ -2324,7 +2324,7 @@ $collections = [ '$id' => ID::custom('path'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 16384, + 'size' => 2048, 'signed' => true, 'required' => false, 'default' => null, @@ -2896,7 +2896,7 @@ $collections = [ '$id' => 'compression', 'type' => Database::VAR_STRING, 'signed' => true, - 'size' => 128, + 'size' => 10, 'format' => '', 'filters' => [], 'required' => true, diff --git a/docker-compose.yml b/docker-compose.yml index 17754b6fe..ea9241a9d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/local/dev - - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis From 8ced55bf46764e8c8c735cb28b6c7b661576691f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 2 Nov 2022 20:26:03 +0530 Subject: [PATCH 2/6] feat: add new rsync worker --- Dockerfile | 2 ++ app/cli.php | 1 + app/preload.php | 1 + app/tasks/rsync.php | 42 +++++++++++++++++++++++++++++++++++++ bin/certificate-sync | 3 +++ docker-compose.yml | 49 +++++++++++++++++++++++++++++--------------- 6 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 app/tasks/rsync.php create mode 100644 bin/certificate-sync diff --git a/Dockerfile b/Dockerfile index a7cae3850..c44c9359e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -271,6 +271,7 @@ RUN \ && apk add --no-cache \ libstdc++ \ certbot \ + rsync \ brotli-dev \ yaml-dev \ imagemagick \ @@ -332,6 +333,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/certificate-sync && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ diff --git a/app/cli.php b/app/cli.php index 09b3dc941..34f4f999f 100644 --- a/app/cli.php +++ b/app/cli.php @@ -14,6 +14,7 @@ $cli = new CLI(); include 'tasks/doctor.php'; include 'tasks/maintenance.php'; +include 'tasks/rsync.php'; include 'tasks/install.php'; include 'tasks/migrate.php'; include 'tasks/sdks.php'; diff --git a/app/preload.php b/app/preload.php index bf8b0bfd1..ec81244ba 100644 --- a/app/preload.php +++ b/app/preload.php @@ -35,6 +35,7 @@ foreach ( realpath(__DIR__ . '/../vendor/symfony'), realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload + realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove workerman autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/tasks/rsync.php b/app/tasks/rsync.php new file mode 100644 index 000000000..3ca5dfbe6 --- /dev/null +++ b/app/tasks/rsync.php @@ -0,0 +1,42 @@ +task('rsync') + ->desc('Runs rsync to sync certificates between the storage mount and traefik.') + ->param('source', null, new Text(255), 'Source path to sync from.', false) + ->param('destination', null, new Text(255), 'Destination path to sync to.', false) + ->param('interval', null, new Integer(true), 'Interval to run rsync', false) + ->action(function ($source, $destination, $interval) { + + Console::title('RSync V1'); + Console::success(APP_NAME . ' rsync process v1 has started'); + + Console::loop(function () use ($interval, $source, $destination) { + $time = DateTime::now(); + + Console::info("[{$time}] Executing rsync every {$interval} seconds"); + Console::info("Syncing between $source and $destination"); + + if (!file_exists($source)) { + Console::error('Source directory does not exist. Skipping ... '); + return; + } + + $stdin = ""; + $stdout = ""; + $stderr = ""; + + Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); + Console::success($stdout); + Console::error($stderr); + + }, $interval); + }); diff --git a/bin/certificate-sync b/bin/certificate-sync new file mode 100644 index 000000000..0d4b4718b --- /dev/null +++ b/bin/certificate-sync @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php rsync $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f143b437f..06df148ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,7 +85,7 @@ services: - ./dev:/usr/local/dev - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - - mariadb + - mysql - redis # - clamav entrypoint: @@ -204,7 +204,7 @@ services: - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: - - mariadb + - mysql - redis environment: - _APP_ENV @@ -235,7 +235,7 @@ services: - ./src:/usr/src/code/src depends_on: - redis - - mariadb + - mysql environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 @@ -264,7 +264,7 @@ services: - ./src:/usr/src/code/src depends_on: - redis - - mariadb + - mysql - request-catcher environment: - _APP_ENV @@ -287,7 +287,7 @@ services: - appwrite depends_on: - redis - - mariadb + - mysql volumes: - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw @@ -348,7 +348,7 @@ services: #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis - - mariadb + - mysql environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 @@ -377,7 +377,7 @@ services: - ./src:/usr/src/code/src depends_on: - redis - - mariadb + - mysql environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 @@ -405,7 +405,7 @@ services: - appwrite depends_on: - redis - - mariadb + - mysql volumes: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw @@ -442,7 +442,7 @@ services: - ./src:/usr/src/code/src depends_on: - redis - - mariadb + - mysql - appwrite-executor environment: - _APP_ENV @@ -488,7 +488,7 @@ services: - /tmp:/tmp:rw depends_on: - redis - - mariadb + - mysql - appwrite environment: - _APP_ENV @@ -619,6 +619,21 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-rsync: + entrypoint: certificate-sync + <<: *x-logging + container_name: appwrite-rsync + build: + context: . + restart: unless-stopped + command: + - --source=/data/src/ --destination=/data/dest/ --interval=10 + networks: + - appwrite + # volumes: # Mount the rsync source and destination directories + # - /nfs/config:/data/src + # - /storage/config:/data/dest + appwrite-usage-timeseries: entrypoint: - usage @@ -637,7 +652,7 @@ services: - ./dev:/usr/local/dev depends_on: - influxdb - - mariadb + - mysql environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 @@ -675,7 +690,7 @@ services: - ./dev:/usr/local/dev depends_on: - influxdb - - mariadb + - mysql environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 @@ -715,14 +730,14 @@ services: - _APP_REDIS_USER - _APP_REDIS_PASS - mariadb: - image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p - container_name: appwrite-mariadb + mysql: + image: mysql:8.0.28-oracle # fix issues when upgrading using: mysql_upgrade -u root -p + container_name: appwrite-mysql <<: *x-logging networks: - appwrite volumes: - - appwrite-mariadb:/var/lib/mysql:rw + - appwrite-mysql:/var/lib/mysql:rw ports: - "3306:3306" environment: @@ -886,7 +901,7 @@ networks: runtimes: volumes: - appwrite-mariadb: + appwrite-mysql: appwrite-redis: appwrite-cache: appwrite-uploads: From 0aac303d51bb88639cfa163c8c8ab11061c366e8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 2 Nov 2022 20:38:13 +0530 Subject: [PATCH 3/6] feat: rename rsync to volume-sync worker --- Dockerfile | 2 +- app/cli.php | 2 +- app/tasks/{rsync.php => volume-sync.php} | 3 +-- bin/certificate-sync | 3 --- bin/volume-sync | 3 +++ docker-compose.yml | 9 ++++----- 6 files changed, 10 insertions(+), 12 deletions(-) rename app/tasks/{rsync.php => volume-sync.php} (97%) delete mode 100644 bin/certificate-sync create mode 100644 bin/volume-sync diff --git a/Dockerfile b/Dockerfile index c44c9359e..d284e2a90 100755 --- a/Dockerfile +++ b/Dockerfile @@ -333,7 +333,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/certificate-sync && \ + chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ diff --git a/app/cli.php b/app/cli.php index 34f4f999f..55704e47e 100644 --- a/app/cli.php +++ b/app/cli.php @@ -14,7 +14,7 @@ $cli = new CLI(); include 'tasks/doctor.php'; include 'tasks/maintenance.php'; -include 'tasks/rsync.php'; +include 'tasks/volume-sync.php'; include 'tasks/install.php'; include 'tasks/migrate.php'; include 'tasks/sdks.php'; diff --git a/app/tasks/rsync.php b/app/tasks/volume-sync.php similarity index 97% rename from app/tasks/rsync.php rename to app/tasks/volume-sync.php index 3ca5dfbe6..2114348b1 100644 --- a/app/tasks/rsync.php +++ b/app/tasks/volume-sync.php @@ -2,14 +2,13 @@ global $cli; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\DateTime; use Utopia\Validator\Integer; use Utopia\Validator\Text; $cli - ->task('rsync') + ->task('volume-sync') ->desc('Runs rsync to sync certificates between the storage mount and traefik.') ->param('source', null, new Text(255), 'Source path to sync from.', false) ->param('destination', null, new Text(255), 'Destination path to sync to.', false) diff --git a/bin/certificate-sync b/bin/certificate-sync deleted file mode 100644 index 0d4b4718b..000000000 --- a/bin/certificate-sync +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php rsync $@ \ No newline at end of file diff --git a/bin/volume-sync b/bin/volume-sync new file mode 100644 index 000000000..5190750a2 --- /dev/null +++ b/bin/volume-sync @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php volume-sync $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8b2faf8e1..895c2a5e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -603,12 +603,11 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT - appwrite-rsync: - entrypoint: certificate-sync + appwrite-volume-sync: + entrypoint: volume-sync <<: *x-logging - container_name: appwrite-rsync - build: - context: . + container_name: appwrite-volume-sync + image: appwrite-dev restart: unless-stopped command: - --source=/data/src/ --destination=/data/dest/ --interval=10 From 0c6107990dbc8242aa5bd277f0f3b6a9db7188b0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 7 Nov 2022 17:17:51 +0530 Subject: [PATCH 4/6] feat: update volume sync --- .env | 6 +++--- app/tasks/volume-sync.php | 5 +++++ docker-compose.yml | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 950fd4482..b28a75a9d 100644 --- a/.env +++ b/.env @@ -17,14 +17,14 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -_APP_DB_HOST=mariadb +_APP_DB_HOST=mysql _APP_DB_PORT=3306 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite -_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite +_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mysql:3306/appwrite +_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mysql:3306/appwrite _APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 _APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 diff --git a/app/tasks/volume-sync.php b/app/tasks/volume-sync.php index 2114348b1..cede9fe23 100644 --- a/app/tasks/volume-sync.php +++ b/app/tasks/volume-sync.php @@ -18,6 +18,11 @@ $cli Console::title('RSync V1'); Console::success(APP_NAME . ' rsync process v1 has started'); + if (!file_exists($source)) { + Console::error('Source directory does not exist. Exiting ... '); + Console::exit(1); + } + Console::loop(function () use ($interval, $source, $destination) { $time = DateTime::now(); diff --git a/docker-compose.yml b/docker-compose.yml index 012b95f0e..b3fee3fc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -647,7 +647,7 @@ services: <<: *x-logging container_name: appwrite-volume-sync image: appwrite-dev - restart: unless-stopped + restart: on-failure:4 command: - --source=/data/src/ --destination=/data/dest/ --interval=10 networks: From f5bf48baa7c433afbd1a44b07a062c1eb45466d6 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 7 Nov 2022 17:36:42 +0530 Subject: [PATCH 5/6] feat: update volume sync --- app/tasks/volume-sync.php | 2 +- docker-compose.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/tasks/volume-sync.php b/app/tasks/volume-sync.php index cede9fe23..efd8618ee 100644 --- a/app/tasks/volume-sync.php +++ b/app/tasks/volume-sync.php @@ -20,7 +20,7 @@ $cli if (!file_exists($source)) { Console::error('Source directory does not exist. Exiting ... '); - Console::exit(1); + Console::exit(0); } Console::loop(function () use ($interval, $source, $destination) { diff --git a/docker-compose.yml b/docker-compose.yml index b3fee3fc4..f6687493a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -647,7 +647,6 @@ services: <<: *x-logging container_name: appwrite-volume-sync image: appwrite-dev - restart: on-failure:4 command: - --source=/data/src/ --destination=/data/dest/ --interval=10 networks: From 682790272fa75604d25a764256c2758caaa9da52 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 7 Nov 2022 18:10:56 +0530 Subject: [PATCH 6/6] feat: update volume sync --- app/tasks/volume-sync.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/tasks/volume-sync.php b/app/tasks/volume-sync.php index efd8618ee..8d5fec201 100644 --- a/app/tasks/volume-sync.php +++ b/app/tasks/volume-sync.php @@ -41,6 +41,5 @@ $cli Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr); Console::success($stdout); Console::error($stderr); - }, $interval); });