chore: remove resque library and update health check endpoints

2024-06-14 00:34:51 +12:00 · 2023-10-18 19:44:06 +00:00 · 2023-10-18 19:44:06 +00:00 · be26e4b729
parent 80086e9edc
commit be26e4b729
10 changed files with 93 additions and 250 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -206,7 +206,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi
 │       ├── Network
 │       ├── OpenSSL
 │       ├── Promises
-│       ├── Resque
 │       ├── Specification
 │       ├── Task
 │       ├── Template
@ -251,7 +250,6 @@ Appwrite stack is a combination of a variety of open-source technologies and too
 - Imagemagick - for manipulating and managing image media files.
 - Webp - for better compression of images on supporting clients.
 - SMTP - for sending email messages and alerts.
- Resque - for managing data queues and scheduled tasks over a Redis server.

 ## Package Managers

--- a/app/controllers/api/health.php
+++ b/app/controllers/api/health.php
@ -14,6 +14,7 @@ use Utopia\Registry\Registry;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
+use Utopia\Validator\Text;

 App::get('/v1/health')
    ->desc('Get HTTP')
@ -347,10 +348,11 @@ App::get('/v1/health/queue/webhooks')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::WEBHOOK_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/logs')
@ -364,10 +366,11 @@ App::get('/v1/health/queue/logs')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::AUDITS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::AUDITS_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/certificates')
@ -381,10 +384,11 @@ App::get('/v1/health/queue/certificates')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::CERTIFICATES_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/builds')
@ -398,13 +402,14 @@ App::get('/v1/health/queue/builds')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::BUILDS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::BUILDS_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

-App::get('/v1/health/queue/databases')
+App::get('/v1/health/queue/databases/:databaseId')
    ->desc('Get databases queue')
    ->groups(['api', 'health'])
    ->label('scope', 'health.read')
@ -415,10 +420,13 @@ App::get('/v1/health/queue/databases')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('databaseId', 'database_db_main', new Text(256), 'Database for which to check the queue size', true)
+    ->inject('databaseId')
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::DATABASE_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (string $databaseId, Connection $queue, Response $response) {
+        $client = new Client($databaseId, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/deletes')
@ -432,10 +440,11 @@ App::get('/v1/health/queue/deletes')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::DELETE_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::DELETE_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/mails')
@ -449,10 +458,11 @@ App::get('/v1/health/queue/mails')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::MAILS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::MAILS_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/messaging')
@ -466,10 +476,11 @@ App::get('/v1/health/queue/messaging')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::MESSAGING_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/migrations')
@ -483,10 +494,11 @@ App::get('/v1/health/queue/migrations')
    ->label('sdk.response.code', Response::STATUS_CODE_OK)
    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('queue')
    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->dynamic(new Document([ 'size' => Resque::size(Event::MIGRATIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
+    ->action(function (Connection $queue, Response $response) {
+        $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue);
+        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
    }, ['response']);

 App::get('/v1/health/queue/functions')
--- a/app/init.php
+++ b/app/init.php
@ -262,14 +262,6 @@ Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php');
 Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php');
 Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php');

-$user = App::getEnv('_APP_REDIS_USER', '');
-$pass = App::getEnv('_APP_REDIS_PASS', '');
-if (!empty($user) || !empty($pass)) {
-    Resque::setBackend('redis://' . $user . ':' . $pass . '@' . App::getEnv('_APP_REDIS_HOST', '') . ':' . App::getEnv('_APP_REDIS_PORT', ''));
-} else {
-    Resque::setBackend(App::getEnv('_APP_REDIS_HOST', '') . ':' . App::getEnv('_APP_REDIS_PORT', ''));
-}
-
 /**
 * New DB Filters
 */
--- a/composer.json
+++ b/composer.json
@ -68,7 +68,6 @@
        "utopia-php/swoole": "0.5.*",
        "utopia-php/vcs": "0.5.*",
        "utopia-php/websocket": "0.1.*",
-        "resque/php-resque": "1.3.6",
        "matomo/device-detector": "6.1.*",
        "dragonmantank/cron-expression": "3.3.2",
        "influxdb/influxdb-php": "1.15.2",
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "5060c891e467051fedd25f3853cd37b0",
+    "content-hash": "3a0624bf1df70e602233efa5916aa6ce",
    "packages": [
        {
            "name": "adhocore/jwt",
@ -339,53 +339,6 @@
            ],
            "time": "2022-07-05T22:32:14+00:00"
        },
-        {
-            "name": "colinmollenhour/credis",
-            "version": "v1.15.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/colinmollenhour/credis.git",
-                "reference": "28810439de1d9597b7ba11794ed9479fb6f3de7c"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/28810439de1d9597b7ba11794ed9479fb6f3de7c",
-                "reference": "28810439de1d9597b7ba11794ed9479fb6f3de7c",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=5.6.0"
-            },
-            "suggest": {
-                "ext-redis": "Improved performance for communicating with redis"
-            },
-            "type": "library",
-            "autoload": {
-                "classmap": [
-                    "Client.php",
-                    "Cluster.php",
-                    "Sentinel.php",
-                    "Module.php"
-                ]
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Colin Mollenhour",
-                    "email": "colin@mollenhour.com"
-                }
-            ],
-            "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.",
-            "homepage": "https://github.com/colinmollenhour/credis",
-            "support": {
-                "issues": "https://github.com/colinmollenhour/credis/issues",
-                "source": "https://github.com/colinmollenhour/credis/tree/v1.15.0"
-            },
-            "time": "2023-04-18T15:34:23+00:00"
-        },
        {
            "name": "dragonmantank/cron-expression",
            "version": "v3.3.2",
@ -1476,56 +1429,6 @@
            },
            "time": "2023-04-04T09:54:51+00:00"
        },
-        {
-            "name": "psr/log",
-            "version": "1.1.4",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/php-fig/log.git",
-                "reference": "d49695b909c3b7628b6289db5479a1c204601f11"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/php-fig/log/zipball/d49695b909c3b7628b6289db5479a1c204601f11",
-                "reference": "d49695b909c3b7628b6289db5479a1c204601f11",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=5.3.0"
-            },
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-master": "1.1.x-dev"
-                }
-            },
-            "autoload": {
-                "psr-4": {
-                    "Psr\\Log\\": "Psr/Log/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "PHP-FIG",
-                    "homepage": "https://www.php-fig.org/"
-                }
-            ],
-            "description": "Common interface for logging libraries",
-            "homepage": "https://github.com/php-fig/log",
-            "keywords": [
-                "log",
-                "psr",
-                "psr-3"
-            ],
-            "support": {
-                "source": "https://github.com/php-fig/log/tree/1.1.4"
-            },
-            "time": "2021-05-03T11:20:27+00:00"
-        },
        {
            "name": "ralouphie/getallheaders",
            "version": "3.0.3",
@ -1570,89 +1473,6 @@
            },
            "time": "2019-03-08T08:55:37+00:00"
        },
-        {
-            "name": "resque/php-resque",
-            "version": "v1.3.6",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/resque/php-resque.git",
-                "reference": "fe41c04763699b1318d97ed14cc78583e9380161"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/resque/php-resque/zipball/fe41c04763699b1318d97ed14cc78583e9380161",
-                "reference": "fe41c04763699b1318d97ed14cc78583e9380161",
-                "shasum": ""
-            },
-            "require": {
-                "colinmollenhour/credis": "~1.7",
-                "php": ">=5.6.0",
-                "psr/log": "~1.0"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "^5.7"
-            },
-            "suggest": {
-                "ext-pcntl": "REQUIRED for forking processes on platforms that support it (so anything but Windows).",
-                "ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.",
-                "ext-redis": "Native PHP extension for Redis connectivity. Credis will automatically utilize when available."
-            },
-            "bin": [
-                "bin/resque",
-                "bin/resque-scheduler"
-            ],
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-master": "1.0-dev"
-                }
-            },
-            "autoload": {
-                "psr-0": {
-                    "Resque": "lib",
-                    "ResqueScheduler": "lib"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Dan Hunsaker",
-                    "email": "danhunsaker+resque@gmail.com",
-                    "role": "Maintainer"
-                },
-                {
-                    "name": "Rajib Ahmed",
-                    "homepage": "https://github.com/rajibahmed",
-                    "role": "Maintainer"
-                },
-                {
-                    "name": "Steve Klabnik",
-                    "email": "steve@steveklabnik.com",
-                    "role": "Maintainer"
-                },
-                {
-                    "name": "Chris Boulton",
-                    "email": "chris@bigcommerce.com",
-                    "role": "Creator"
-                }
-            ],
-            "description": "Redis backed library for creating background jobs and processing them later. Based on resque for Ruby.",
-            "homepage": "http://www.github.com/resque/php-resque/",
-            "keywords": [
-                "background",
-                "job",
-                "redis",
-                "resque"
-            ],
-            "support": {
-                "issues": "https://github.com/resque/php-resque/issues",
-                "source": "https://github.com/resque/php-resque/tree/v1.3.6"
-            },
-            "time": "2020-04-16T16:39:50+00:00"
-        },
        {
            "name": "slickdeals/statsd",
            "version": "3.1.0",
@ -4605,6 +4425,56 @@
            ],
            "time": "2022-04-01T12:37:26+00:00"
        },
+        {
+            "name": "psr/log",
+            "version": "3.0.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/php-fig/log.git",
+                "reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/php-fig/log/zipball/fe5ea303b0887d5caefd3d431c3e61ad47037001",
+                "reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.0.0"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "3.x-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Psr\\Log\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "PHP-FIG",
+                    "homepage": "https://www.php-fig.org/"
+                }
+            ],
+            "description": "Common interface for logging libraries",
+            "homepage": "https://github.com/php-fig/log",
+            "keywords": [
+                "log",
+                "psr",
+                "psr-3"
+            ],
+            "support": {
+                "source": "https://github.com/php-fig/log/tree/3.0.0"
+            },
+            "time": "2021-07-14T16:46:02+00:00"
+        },
        {
            "name": "sebastian/cli-parser",
            "version": "1.0.1",
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -872,7 +872,6 @@ services:
  # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI.
  # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
  # RedisCommander - A nice UI for exploring Redis data
-  # Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
  # Chronograf - A nice UI for exploring InfluxDB data
  # Webgrind - A nice UI for exploring and debugging code-level stuff

@ -914,19 +913,6 @@ services:
  #   ports:
  #   - "8081:8081"

-  # resque:
-  #   image: appwrite/resque-web:1.1.0
-  #   restart: unless-stopped
-  #   networks:
-  #     - appwrite
-  #   ports:
-  #     - "5678:5678"
-  #   environment:
-  #     - RESQUE_WEB_HOST=redis
-  #     - RESQUE_WEB_PORT=6379
-  #     - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
-  #     - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
-
  # chronograf:
  #   image: chronograf:1.6
  #   container_name: appwrite-chronograf
--- a/src/Appwrite/Event/Database.php
+++ b/src/Appwrite/Event/Database.php
@ -107,7 +107,6 @@ class Database extends Event
     */
    public function trigger(): string|bool
    {
-
        $this->setQueue($this->getProject()->getAttribute('database'));

        $client = new Client($this->queue, $this->connection);
--- a/src/Appwrite/Platform/Tasks/Maintenance.php
+++ b/src/Appwrite/Platform/Tasks/Maintenance.php
@ -22,7 +22,7 @@ class Maintenance extends Action
    public function __construct()
    {
        $this
-            ->desc('Schedules maintenance tasks and publishes them to resque')
+            ->desc('Schedules maintenance tasks and publishes them to our queues')
            ->inject('dbForConsole')
            ->inject('queueForCertificates')
            ->inject('queueForDeletes')
--- a/src/Appwrite/Platform/Workers/Databases.php
+++ b/src/Appwrite/Platform/Workers/Databases.php
@ -104,6 +104,7 @@ class Databases extends Action
            'attributeId' => $attribute->getId()
        ]);
        /**
+         * TODO @christyjacob4 verify if this is still the case
         * Fetch attribute from the database, since with Resque float values are loosing informations.
         */
        $attribute = $dbForProject->getDocument('attributes', $attribute->getId());
--- a/tests/resources/docker/docker-compose.yml
+++ b/tests/resources/docker/docker-compose.yml
@ -376,20 +376,6 @@ services:
  #   ports:
  #   - "8081:8081"

-  # resque:
-  #   image: registry.gitlab.com/appwrite/appwrite/resque-web:v1.0.2
-  #   restart: unless-stopped
-  #   networks:
-  #     - appwrite
-  #   ports:
-  #     - "5678:5678"
-  #   environment:
-  #     - RESQUE_WEB_HOST=redis
-  #     - RESQUE_WEB_PORT=6379
-  #     - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
-  #     - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
-
-
  # webgrind:
  #   image: 'jokkedk/webgrind:latest'
  #   volumes: