From b8b81a9bd1a1ffc8cd6600a1dd214324a8620a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Jun 2024 19:05:29 +0000 Subject: [PATCH 01/22] WIP: Schedulded executions --- app/controllers/api/functions.php | 59 +++++++++++++------ app/controllers/api/messaging.php | 6 +- .../Platform/Tasks/ScheduleFunctions.php | 2 +- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 392cf034f9..900147398a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -32,6 +32,7 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Datetime as DatetimeValidator; use Utopia\Database\Validator\Roles; use Utopia\Database\Validator\UID; use Utopia\Storage\Device; @@ -1511,16 +1512,21 @@ App::post('/v1/functions/:functionId/executions') ->param('path', '/', new Text(2048), 'HTTP path of execution. Path can include query params. Default value is /', true) ->param('method', 'POST', new Whitelist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], true), 'HTTP method of execution. Default value is GET.', true) ->param('headers', [], new Assoc(), 'HTTP headers of execution. Defaults to empty.', true) + ->param('scheduledAt', null, new DatetimeValidator(requireDateInFuture: true), 'Scheduled execution time in [ISO 8601](https://www.iso.org/iso-8601-date-and-time-format.html) format. DateTime value must be in future.', true) ->inject('response') ->inject('project') ->inject('dbForProject') + ->inject('dbForConsole') ->inject('user') ->inject('queueForEvents') ->inject('queueForUsage') - ->inject('mode') ->inject('queueForFunctions') ->inject('geodb') - ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, ?string $scheduledAt, Response $response, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { + + if(!$async && !is_null($scheduledAt)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, 'Scheduled executions must run asynchronously. Don\'t set scheduledAt to execute immediately, or set async to true.'); + } $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1625,6 +1631,12 @@ App::post('/v1/functions/:functionId/executions') $executionId = ID::unique(); + $status = $async ? 'waiting' : 'processing'; + + if(!is_null($scheduledAt)) { + $status = 'scheduled'; + } + $execution = new Document([ '$id' => $executionId, '$permissions' => !$user->isEmpty() ? [Permission::read(Role::user($user->getId()))] : [], @@ -1633,7 +1645,7 @@ App::post('/v1/functions/:functionId/executions') 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'trigger' => 'http', // http / schedule / event - 'status' => $async ? 'waiting' : 'processing', // waiting / processing / completed / failed + 'status' => $status, // waiting / processing / completed / failed 'responseStatusCode' => 0, 'responseHeaders' => [], 'requestPath' => $path, @@ -1656,20 +1668,33 @@ App::post('/v1/functions/:functionId/executions') $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); } - $queueForFunctions - ->setType('http') - ->setExecution($execution) - ->setFunction($function) - ->setBody($body) - ->setHeaders($headers) - ->setPath($path) - ->setMethod($method) - ->setJWT($jwt) - ->setProject($project) - ->setUser($user) - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->trigger(); + if(is_null($scheduledAt)) { + $queueForFunctions + ->setType('http') + ->setExecution($execution) + ->setFunction($function) + ->setBody($body) + ->setHeaders($headers) + ->setPath($path) + ->setMethod($method) + ->setJWT($jwt) + ->setProject($project) + ->setUser($user) + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) + ->trigger(); + } else { + $dbForConsole->createDocument('schedules', new Document([ + 'region' => System::getEnv('_APP_REGION', 'default'), + 'resourceType' => 'function', + 'resourceId' => $function->getId(), + 'resourceInternalId' => $function->getInternalId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $project->getId(), + 'schedule' => $scheduledAt, + 'active' => true, + ])); + } return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index e3696cc7e0..7da0348a8f 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2697,7 +2697,7 @@ App::post('/v1/messaging/messages/email') 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $scheduledAt, + 'schedule' => $scheduledAt, 'active' => true, ])); @@ -2813,7 +2813,7 @@ App::post('/v1/messaging/messages/sms') 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $scheduledAt, + 'schedule' => $scheduledAt, 'active' => true, ])); @@ -2989,7 +2989,7 @@ App::post('/v1/messaging/messages/push') 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $scheduledAt, + 'schedule' => $scheduledAt, 'active' => true, ])); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index e2c278714f..e8941c2ffa 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -41,7 +41,7 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine foreach ($this->schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); + $cron = new CronExpression($schedule['schedule']); // TODO: Allow schedule to be DateTime, like ScheduleMessaging.php $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); From 7e8f72d267ba8cf8bf80f7a607dfd87a787fe09e Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:39:38 +0100 Subject: [PATCH 02/22] feat: implement scheduledAt in schedule --- .../Platform/Tasks/ScheduleFunctions.php | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index e8941c2ffa..c8e68e0d1e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -40,32 +40,39 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine - foreach ($this->schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); // TODO: Allow schedule to be DateTime, like ScheduleMessaging.php - $nextDate = $cron->getNextRunDate(); - $next = DateTime::format($nextDate); + foreach ($this->schedules as $scheduleKey => $schedule) { + if (CronExpression::isValidExpression($schedule['schedule'])) { + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); + } else { + try { + $nextDate = new \DateTime($schedule['schedule']); + $schedule['delete'] = true; + } catch (\Exception) { + Console::error('Failed to parse schedule: ' . $schedule['schedule']); + continue; + } + } + $next = DateTime::format($nextDate); $currentTick = $next < $timeFrame; if (!$currentTick) { continue; } - $total++; - - $promiseStart = \time(); // in seconds - $executionStart = $nextDate->getTimestamp(); // in seconds - $delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + $total += 1; + $delay = $nextDate->getTimestamp() - \time(); // Time to wait from now until execution needs to be queued if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; } - $delayedExecutions[$delay][] = $key; + $delayedExecutions[$delay][] = $scheduleKey; } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools) { + \go(function () use ($delay, $scheduleKeys, $pools, $dbForConsole) { \sleep($delay); // in seconds $queue = $pools->get('queue')->pop(); @@ -76,7 +83,6 @@ class ScheduleFunctions extends ScheduleBase if (!\array_key_exists($scheduleKey, $this->schedules)) { return; } - $schedule = $this->schedules[$scheduleKey]; $queueForFunctions = new Func($connection); @@ -88,6 +94,13 @@ class ScheduleFunctions extends ScheduleBase ->setPath('/') ->setProject($schedule['project']) ->trigger(); + + if ($schedule['delete']) { + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); + } } $queue->reclaim(); From 1e9ced2878af41f08502f18f51d8b259eb1c513c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:57:03 +0100 Subject: [PATCH 03/22] test: `scheduled` status --- .../Platform/Tasks/ScheduleFunctions.php | 2 +- .../Functions/FunctionsCustomClientTest.php | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index c8e68e0d1e..afde51f23b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -51,7 +51,7 @@ class ScheduleFunctions extends ScheduleBase } catch (\Exception) { Console::error('Failed to parse schedule: ' . $schedule['schedule']); continue; - } + } } $next = DateTime::format($nextDate); diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 12dd7cda59..db4421dd3e 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -175,6 +175,100 @@ class FunctionsCustomClientTest extends Scope return []; } + public function testCreateScheduledExecution(): void + { + /** + * Test for SUCCESS + */ + $function = $this->client->call(Client::METHOD_POST, '/functions', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'execute' => [Role::user($this->getUser()['$id'])->toString()], + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'events' => [ + 'users.*.create', + 'users.*.delete', + ], + 'timeout' => 10, + ]); + + $this->assertEquals(201, $function['headers']['status-code']); + + $folder = 'php'; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz"; + $this->packageCode($folder); + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/deployments', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', \basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); + + $function = $this->client->call(Client::METHOD_PATCH, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(200, $function['headers']['status-code']); + + // Schedule execution for the future + $futureTime = (new \DateTime())->add(new \DateInterval('PT1M'))->format('Y-m-d H:i:s'); + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'async' => true, + 'scheduledAt' => $futureTime, + ]); + + $this->assertEquals(202, $execution['headers']['status-code']); + $this->assertEquals('scheduled', $execution['body']['status'], \json_encode($execution['body'])); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $function['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } + + public function testCreateCustomExecution(): array { /** From 865b12ba9dac5e16a0e6ee460b93b2f0d36bfdd3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:24:51 +0100 Subject: [PATCH 04/22] chore: exception type --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 900147398a..5edfcc3069 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1525,7 +1525,7 @@ App::post('/v1/functions/:functionId/executions') ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, ?string $scheduledAt, Response $response, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { if(!$async && !is_null($scheduledAt)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, 'Scheduled executions must run asynchronously. Don\'t set scheduledAt to execute immediately, or set async to true.'); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Scheduled executions must run asynchronously. Set scheduledAt to a future date, or set async to true.'); } $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); From cfdd40dbeb90ea2d40ef289cc605320b6bf554dc Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:36:01 +0100 Subject: [PATCH 05/22] test: execution is completed --- .../Functions/FunctionsCustomClientTest.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index db4421dd3e..7ee209af03 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -246,7 +246,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(200, $function['headers']['status-code']); // Schedule execution for the future - $futureTime = (new \DateTime())->add(new \DateInterval('PT1M'))->format('Y-m-d H:i:s'); + $futureTime = (new \DateTime())->add(new \DateInterval('PT10S'))->format('Y-m-d H:i:s'); $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -258,6 +258,17 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(202, $execution['headers']['status-code']); $this->assertEquals('scheduled', $execution['body']['status'], \json_encode($execution['body'])); + $executionId = $execution['body']['$id']; + + \sleep(12); + + $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); + // Cleanup : Delete function $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $function['body']['$id'], [ 'content-type' => 'application/json', @@ -269,6 +280,7 @@ class FunctionsCustomClientTest extends Scope } + public function testCreateCustomExecution(): array { /** From a83d125f54544f399209aacda447a7b66b4b1fd5 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:34:10 +0100 Subject: [PATCH 06/22] chore: adjust timers --- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index afde51f23b..fd417ee274 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -11,8 +11,8 @@ use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase { - public const UPDATE_TIMER = 10; // seconds - public const ENQUEUE_TIMER = 60; // seconds + public const UPDATE_TIMER = 3; // seconds + public const ENQUEUE_TIMER = 4; // seconds private ?float $lastEnqueueUpdate = null; From dff9bed882155c78eb0db917c346c5aa759cfe09 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:41:37 +0100 Subject: [PATCH 07/22] test: increase delay --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 7ee209af03..b72443ed6a 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -260,7 +260,7 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - \sleep(12); + \sleep(20); $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ 'content-type' => 'application/json', From 6f38ef3a18da21fafb0402389c26ff96b6f969f4 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:22:38 +0100 Subject: [PATCH 08/22] test: poll scheduled function --- .../Functions/FunctionsCustomClientTest.php | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index b72443ed6a..0ca17e6061 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -260,13 +260,24 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - \sleep(20); + while (true) { + + $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $execution['headers']['status-code'] >= 400 + || \in_array($execution['body']['status'], ['completed', 'failed']) + ) { + break; + } + + \sleep(1); + } - $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); // Cleanup : Delete function From 10e37bb8ca52a3a6ead24a076d43a9c442e33ef8 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:01:04 +0100 Subject: [PATCH 09/22] test: fix --- .../Functions/FunctionsCustomClientTest.php | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0ca17e6061..0e70794879 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -260,24 +260,15 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - while (true) { + sleep(12); - $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); - - if ( - $execution['headers']['status-code'] >= 400 - || \in_array($execution['body']['status'], ['completed', 'failed']) - ) { - break; - } - - \sleep(1); - } + $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals(200, $execution['headers']['status-code']); $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); // Cleanup : Delete function From 2f0f7bf9c764725d98ce3b6ef3412a3b0ac09ad9 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:44:12 +0100 Subject: [PATCH 10/22] fix: test --- Dockerfile | 1 + app/controllers/api/functions.php | 8 +-- app/views/install/compose.phtml | 25 +++++++ bin/schedule-executions | 3 + docker-compose.yml | 29 +++++++- src/Appwrite/Event/Func.php | 24 +++++++ src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/ScheduleBase.php | 6 +- .../Platform/Tasks/ScheduleExecutions.php | 67 +++++++++++++++++++ .../Platform/Tasks/ScheduleFunctions.php | 42 +++++------- .../Platform/Tasks/ScheduleMessages.php | 2 +- src/Appwrite/Platform/Workers/Functions.php | 9 ++- .../Functions/FunctionsCustomClientTest.php | 6 +- 13 files changed, 185 insertions(+), 39 deletions(-) create mode 100644 bin/schedule-executions create mode 100644 src/Appwrite/Platform/Tasks/ScheduleExecutions.php diff --git a/Dockerfile b/Dockerfile index 1d2ac91ae0..d7e9849b1b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ + chmod +x /usr/local/bin/schedule-executions && \ chmod +x /usr/local/bin/schedule-messages && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index f8690ebce6..5aa3083f3c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1722,7 +1722,7 @@ App::post('/v1/functions/:functionId/executions') 'functionId' => $function->getId(), 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), - 'trigger' => 'http', // http / schedule / event + 'trigger' => (!is_null($scheduledAt)) ? 'schedule' : 'http', 'status' => $status, // waiting / processing / completed / failed 'responseStatusCode' => 0, 'responseHeaders' => [], @@ -1764,9 +1764,9 @@ App::post('/v1/functions/:functionId/executions') } else { $dbForConsole->createDocument('schedules', new Document([ 'region' => System::getEnv('_APP_REGION', 'default'), - 'resourceType' => 'function', - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getInternalId(), + 'resourceType' => 'execution', + 'resourceId' => $execution->getId(), + 'resourceInternalId' => $execution->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), 'schedule' => $scheduledAt, diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f265f53c19..9643440f5e 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -676,6 +676,31 @@ services: - _APP_DB_USER - _APP_DB_PASS + appwrite-task-scheduler-executions: + image: /: + entrypoint: schedule-executions + container_name: appwrite-task-scheduler-executions + <<: *x-logging + restart: unless-stopped + networks: + - appwrite + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + appwrite-task-scheduler-messages: image: /: entrypoint: schedule-messages diff --git a/bin/schedule-executions b/bin/schedule-executions new file mode 100644 index 0000000000..f239cad206 --- /dev/null +++ b/bin/schedule-executions @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php schedule-executions $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 250eb8b7aa..6dd2109c6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,6 @@ x-logging: &x-logging max-file: "5" max-size: "10m" -version: "3" - services: traefik: image: traefik:2.11 @@ -742,6 +740,33 @@ services: - _APP_DB_USER - _APP_DB_PASS + appwrite-task-scheduler-executions: + entrypoint: schedule-executions + <<: *x-logging + container_name: appwrite-task-scheduler-executions + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + appwrite-task-scheduler-messages: entrypoint: schedule-messages <<: *x-logging diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 11c9e980ed..67c28575bd 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -14,6 +14,7 @@ class Func extends Event protected string $path = ''; protected string $method = ''; protected array $headers = []; + protected ?string $functionId = null; protected ?Document $function = null; protected ?Document $execution = null; @@ -49,6 +50,28 @@ class Func extends Event return $this->function; } + /** + * Sets function id for the function event. + * + * @param string $functionId + */ + public function setFunctionId(string $functionId): self + { + $this->functionId = $functionId; + + return $this; + } + + /** + * Returns set function id for the function event. + * + * @return string|null + */ + public function getFunctionId(): ?string + { + return $this->functionId; + } + /** * Sets execution for the function event. * @@ -200,6 +223,7 @@ class Func extends Event 'project' => $this->project, 'user' => $this->user, 'function' => $this->function, + 'functionId' => $this->functionId, 'execution' => $this->execution, 'type' => $this->type, 'jwt' => $this->jwt, diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index ac1f99eec3..b7b333b2c6 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -9,6 +9,7 @@ use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueCount; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleFunctions; +use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; @@ -33,6 +34,7 @@ class Tasks extends Service ->addAction(SDKs::getName(), new SDKs()) ->addAction(SSL::getName(), new SSL()) ->addAction(ScheduleFunctions::getName(), new ScheduleFunctions()) + ->addAction(ScheduleExecutions::getName(), new ScheduleExecutions()) ->addAction(ScheduleMessages::getName(), new ScheduleMessages()) ->addAction(Specs::getName(), new Specs()) ->addAction(Upgrade::getName(), new Upgrade()) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index a50fbb2403..be0abc4b66 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -64,7 +64,8 @@ abstract class ScheduleBase extends Action $collectionId = match ($schedule->getAttribute('resourceType')) { 'function' => 'functions', - 'message' => 'messages' + 'message' => 'messages', + 'execution' => 'executions' }; $resource = $getProjectDB($project)->getDocument( @@ -113,7 +114,8 @@ abstract class ScheduleBase extends Action } catch (\Throwable $th) { $collectionId = match ($document->getAttribute('resourceType')) { 'function' => 'functions', - 'message' => 'messages' + 'message' => 'messages', + 'execution' => 'executions' }; Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php new file mode 100644 index 0000000000..01dde1e88b --- /dev/null +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -0,0 +1,67 @@ +schedules as $schedule) { + if (!$schedule['active'] || CronExpression::isValidExpression($schedule['schedule'])) { + unset($this->schedules[$schedule['resourceId']]); + continue; + } + + $now = new \DateTime(); + $scheduledAt = new \DateTime($schedule['schedule']); + + if ($scheduledAt > $now) { + continue; + } + + \go(function () use ($schedule, $pools, $dbForConsole) { + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + + $queueForFunctions = new Func($connection); + + $queueForFunctions + ->setType('schedule') + ->setFunctionId($schedule['resource']['functionId']) + ->setExecution($schedule['resource']) + ->setMethod('POST') + ->setPath('/') + ->setProject($schedule['project']) + ->trigger(); + + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); + + $queue->reclaim(); + + unset($this->schedules[$schedule['resourceId']]); + }); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index fd417ee274..6ea972af1e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -11,8 +11,8 @@ use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase { - public const UPDATE_TIMER = 3; // seconds - public const ENQUEUE_TIMER = 4; // seconds + public const UPDATE_TIMER = 10; // seconds + public const ENQUEUE_TIMER = 60; // seconds private ?float $lastEnqueueUpdate = null; @@ -40,39 +40,37 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine - foreach ($this->schedules as $scheduleKey => $schedule) { - if (CronExpression::isValidExpression($schedule['schedule'])) { - $cron = new CronExpression($schedule['schedule']); - $nextDate = $cron->getNextRunDate(); - } else { - try { - $nextDate = new \DateTime($schedule['schedule']); - $schedule['delete'] = true; - } catch (\Exception) { - Console::error('Failed to parse schedule: ' . $schedule['schedule']); - continue; - } + foreach ($this->schedules as $key => $schedule) { + if (!$schedule['active'] || !CronExpression::isValidExpression($schedule['schedule'])) { + unset($this->schedules[$schedule['resourceId']]); + continue; } + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); + $currentTick = $next < $timeFrame; if (!$currentTick) { continue; } - $total += 1; - $delay = $nextDate->getTimestamp() - \time(); // Time to wait from now until execution needs to be queued + $total++; + + $promiseStart = \time(); // in seconds + $executionStart = $nextDate->getTimestamp(); // in seconds + $delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; } - $delayedExecutions[$delay][] = $scheduleKey; + $delayedExecutions[$delay][] = $key; } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools, $dbForConsole) { + \go(function () use ($delay, $scheduleKeys, $pools) { \sleep($delay); // in seconds $queue = $pools->get('queue')->pop(); @@ -83,6 +81,7 @@ class ScheduleFunctions extends ScheduleBase if (!\array_key_exists($scheduleKey, $this->schedules)) { return; } + $schedule = $this->schedules[$scheduleKey]; $queueForFunctions = new Func($connection); @@ -94,13 +93,6 @@ class ScheduleFunctions extends ScheduleBase ->setPath('/') ->setProject($schedule['project']) ->trigger(); - - if ($schedule['delete']) { - $dbForConsole->deleteDocument( - 'schedules', - $schedule['$id'], - ); - } } $queue->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 8e52973a0c..145b6ee976 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -35,7 +35,7 @@ class ScheduleMessages extends ScheduleBase continue; } - \go(function () use ($now, $schedule, $pools, $dbForConsole) { + \go(function () use ($schedule, $pools, $dbForConsole) { $queue = $pools->get('queue')->pop(); $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index cbba9657ad..734fbab602 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -83,6 +83,7 @@ class Functions extends Action $eventData = $payload['payload'] ?? ''; $project = new Document($payload['project'] ?? []); $function = new Document($payload['function'] ?? []); + $functionId = $payload['functionId'] ?? ''; $user = new Document($payload['user'] ?? []); $method = $payload['method'] ?? 'POST'; $headers = $payload['headers'] ?? []; @@ -92,6 +93,10 @@ class Functions extends Action return; } + if ($function->isEmpty() && !empty($functionId)) { + $function = $dbForProject->getDocument('functions', $functionId); + } + $log->addTag('functionId', $function->getId()); $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); @@ -176,6 +181,7 @@ class Functions extends Action ); break; case 'schedule': + $execution = new Document($payload['execution'] ?? []); $this->execute( log: $log, dbForProject: $dbForProject, @@ -193,7 +199,7 @@ class Functions extends Action jwt: null, event: null, eventData: null, - executionId: null, + executionId: $execution->getId() ?? null ); break; } @@ -296,7 +302,6 @@ class Functions extends Action $headers['x-appwrite-user-id'] = $user->getId() ?? ''; $headers['x-appwrite-user-jwt'] = $jwt ?? ''; - /** Create execution or update execution status */ /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0e70794879..246bed8c51 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -57,6 +57,7 @@ class FunctionsCustomClientTest extends Scope 'execute' => [Role::user($this->getUser()['$id'])->toString()], 'runtime' => 'php-8.0', 'entrypoint' => 'index.php', + 'logging' => true, 'events' => [ 'users.*.create', 'users.*.delete', @@ -246,6 +247,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(200, $function['headers']['status-code']); // Schedule execution for the future + \date_default_timezone_set('UTC'); $futureTime = (new \DateTime())->add(new \DateInterval('PT10S'))->format('Y-m-d H:i:s'); $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ 'content-type' => 'application/json', @@ -260,7 +262,7 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - sleep(12); + sleep(20); $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ 'content-type' => 'application/json', @@ -281,8 +283,6 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); } - - public function testCreateCustomExecution(): array { /** From 83ffc41d92ea438bec099bfe6dbeb743c7826092 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:46:18 +0100 Subject: [PATCH 11/22] chore: fmt --- src/Appwrite/Event/Func.php | 4 ++-- src/Appwrite/Platform/Services/Tasks.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 67c28575bd..451df2b6c1 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -52,7 +52,7 @@ class Func extends Event /** * Sets function id for the function event. - * + * * @param string $functionId */ public function setFunctionId(string $functionId): self @@ -64,7 +64,7 @@ class Func extends Event /** * Returns set function id for the function event. - * + * * @return string|null */ public function getFunctionId(): ?string diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index b7b333b2c6..999270d2dc 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,8 +8,8 @@ use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueCount; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleExecutions; +use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 01dde1e88b..14d5632000 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -4,7 +4,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Cron\CronExpression; -use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; @@ -41,9 +40,9 @@ class ScheduleExecutions extends ScheduleBase \go(function () use ($schedule, $pools, $dbForConsole) { $queue = $pools->get('queue')->pop(); $connection = $queue->getResource(); - + $queueForFunctions = new Func($connection); - + $queueForFunctions ->setType('schedule') ->setFunctionId($schedule['resource']['functionId']) From 5f144f91adbb51f21f3dabf21def612e7b6834f3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:12:02 +0100 Subject: [PATCH 12/22] chore: matej review --- .../Platform/Tasks/ScheduleExecutions.php | 42 ++++++++----------- .../Platform/Tasks/ScheduleFunctions.php | 7 ---- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 14d5632000..b0f03a35ea 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; -use Cron\CronExpression; use Utopia\Database\Database; use Utopia\Pools\Group; @@ -24,8 +23,12 @@ class ScheduleExecutions extends ScheduleBase protected function enqueueResources(Group $pools, Database $dbForConsole): void { + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + $queueForFunctions = new Func($connection); + foreach ($this->schedules as $schedule) { - if (!$schedule['active'] || CronExpression::isValidExpression($schedule['schedule'])) { + if (!$schedule['active']) { unset($this->schedules[$schedule['resourceId']]); continue; } @@ -37,30 +40,21 @@ class ScheduleExecutions extends ScheduleBase continue; } - \go(function () use ($schedule, $pools, $dbForConsole) { - $queue = $pools->get('queue')->pop(); - $connection = $queue->getResource(); + $queueForFunctions + ->setType('schedule') + ->setFunctionId($schedule['resource']['functionId']) + ->setExecution($schedule['resource']) + ->setProject($schedule['project']) + ->trigger(); - $queueForFunctions = new Func($connection); + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); - $queueForFunctions - ->setType('schedule') - ->setFunctionId($schedule['resource']['functionId']) - ->setExecution($schedule['resource']) - ->setMethod('POST') - ->setPath('/') - ->setProject($schedule['project']) - ->trigger(); - - $dbForConsole->deleteDocument( - 'schedules', - $schedule['$id'], - ); - - $queue->reclaim(); - - unset($this->schedules[$schedule['resourceId']]); - }); + unset($this->schedules[$schedule['resourceId']]); } + + $queue->reclaim(); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 6ea972af1e..e4832d7435 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -41,11 +41,6 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine foreach ($this->schedules as $key => $schedule) { - if (!$schedule['active'] || !CronExpression::isValidExpression($schedule['schedule'])) { - unset($this->schedules[$schedule['resourceId']]); - continue; - } - $cron = new CronExpression($schedule['schedule']); $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); @@ -89,8 +84,6 @@ class ScheduleFunctions extends ScheduleBase $queueForFunctions ->setType('schedule') ->setFunction($schedule['resource']) - ->setMethod('POST') - ->setPath('/') ->setProject($schedule['project']) ->trigger(); } From 67fddeb4faba852b3a94974b09629dc3e2f606b6 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:23:14 +0100 Subject: [PATCH 13/22] fix: path and method --- src/Appwrite/Event/Func.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 451df2b6c1..86080593af 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -11,8 +11,8 @@ class Func extends Event protected string $jwt = ''; protected string $type = ''; protected string $body = ''; - protected string $path = ''; - protected string $method = ''; + protected ?string $path = null; + protected ?string $method = null; protected array $headers = []; protected ?string $functionId = null; protected ?Document $function = null; From 878f6c86df4b0f6796824ca5a76f8b2066d95738 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:01:43 +0100 Subject: [PATCH 14/22] chore: matej review p1 --- .../Platform/Tasks/ScheduleExecutions.php | 2 ++ .../Functions/FunctionsCustomClientTest.php | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index b0f03a35ea..ea1339511d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -42,6 +42,8 @@ class ScheduleExecutions extends ScheduleBase $queueForFunctions ->setType('schedule') + // Set functionId rather than function as we don't have access to $dbForProject + // TODO: Refactor to use function instead of functionId ->setFunctionId($schedule['resource']['functionId']) ->setExecution($schedule['resource']) ->setProject($schedule['project']) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index a187ee43ae..0c06b42d2a 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -59,7 +59,6 @@ class FunctionsCustomClientTest extends Scope 'execute' => [Role::user($this->getUser()['$id'])->toString()], 'runtime' => 'php-8.0', 'entrypoint' => 'index.php', - 'logging' => true, 'events' => [ 'users.*.create', 'users.*.delete', @@ -256,7 +255,7 @@ class FunctionsCustomClientTest extends Scope \sleep(1); } - $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); + $this->assertEquals('ready', $deployment['body']['status']); $function = $this->client->call(Client::METHOD_PATCH, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', @@ -278,7 +277,7 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(202, $execution['headers']['status-code']); - $this->assertEquals('scheduled', $execution['body']['status'], \json_encode($execution['body'])); + $this->assertEquals('scheduled', $execution['body']['status']); $executionId = $execution['body']['$id']; @@ -291,7 +290,21 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(200, $execution['headers']['status-code']); - $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); + $this->assertEquals('completed', $execution['body']['status']); + + /* Test for FAILURE */ + + // Schedule synchronous execution + + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'async' => false, + 'scheduledAt' => $futureTime, + ]); + + $this->assertEquals(400, $execution['headers']['status-code']); // Cleanup : Delete function $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $function['body']['$id'], [ From 303ce498becd949e43f916f8b861fc523520ba28 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:30:23 +0100 Subject: [PATCH 15/22] feat: allow custom path method body --- app/config/collections.php | 11 +++++++++++ app/controllers/api/functions.php | 8 ++++++++ src/Appwrite/Event/Func.php | 4 ++-- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 6 +++++- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 2 ++ .../Services/Functions/FunctionsCustomClientTest.php | 11 +++++++++++ 6 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 66bb2606cc..2b7777e85b 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4550,6 +4550,17 @@ $consoleCollections = array_merge([ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('metadata'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65535, + 'signed' => true, + 'required' => false, + 'default' => new \stdClass(), + 'array' => false, + 'filters' => ['json'], + ], [ '$id' => ID::custom('active'), 'type' => Database::VAR_BOOLEAN, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7137fbf9d3..2fcdfa92de 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1764,6 +1764,13 @@ App::post('/v1/functions/:functionId/executions') ->setParam('executionId', $execution->getId()) ->trigger(); } else { + $metadata = [ + 'headers' => $headers, + 'path' => $path, + 'method' => $method, + 'body' => $body, + ]; + $dbForConsole->createDocument('schedules', new Document([ 'region' => System::getEnv('_APP_REGION', 'default'), 'resourceType' => 'execution', @@ -1772,6 +1779,7 @@ App::post('/v1/functions/:functionId/executions') 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), 'schedule' => $scheduledAt, + 'metadata' => $metadata, 'active' => true, ])); } diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 86080593af..451df2b6c1 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -11,8 +11,8 @@ class Func extends Event protected string $jwt = ''; protected string $type = ''; protected string $body = ''; - protected ?string $path = null; - protected ?string $method = null; + protected string $path = ''; + protected string $method = ''; protected array $headers = []; protected ?string $functionId = null; protected ?Document $function = null; diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index ea1339511d..55cefb2a44 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -42,10 +42,14 @@ class ScheduleExecutions extends ScheduleBase $queueForFunctions ->setType('schedule') - // Set functionId rather than function as we don't have access to $dbForProject + // Set functionId instead of function as we don't have $dbForProject // TODO: Refactor to use function instead of functionId ->setFunctionId($schedule['resource']['functionId']) ->setExecution($schedule['resource']) + ->setMethod($schedule['metadata']['method'] ?? 'POST') + ->setPath($schedule['metadata']['path'] ?? '/') + ->setHeaders($schedule['metadata']['headers'] ?? []) + ->setBody($schedule['metadata']['body'] ?? '') ->setProject($schedule['project']) ->trigger(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index e4832d7435..e2c278714f 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -84,6 +84,8 @@ class ScheduleFunctions extends ScheduleBase $queueForFunctions ->setType('schedule') ->setFunction($schedule['resource']) + ->setMethod('POST') + ->setPath('/') ->setProject($schedule['project']) ->trigger(); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0c06b42d2a..38ed9c9564 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -268,16 +268,27 @@ class FunctionsCustomClientTest extends Scope // Schedule execution for the future \date_default_timezone_set('UTC'); $futureTime = (new \DateTime())->add(new \DateInterval('PT10S'))->format('Y-m-d H:i:s'); + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'async' => true, 'scheduledAt' => $futureTime, + 'path' => '/custom', + 'method' => 'GET', + 'body' => 'hello', + 'headers' => [ + 'content-type' => 'application/plain', + ], ]); $this->assertEquals(202, $execution['headers']['status-code']); + $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('scheduled', $execution['body']['status']); + $this->assertEquals('/custom', $execution['requestPath']); + $this->assertEquals('GET', $execution['requestMethod']); + $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); $executionId = $execution['body']['$id']; From 9eb8f02b5367b08c11566928810ddbd4d95543ea Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:42:35 +0100 Subject: [PATCH 16/22] fix: test --- .../e2e/Services/Functions/FunctionsCustomClientTest.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 38ed9c9564..50767a4c1e 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -284,11 +284,7 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(202, $execution['headers']['status-code']); - $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('scheduled', $execution['body']['status']); - $this->assertEquals('/custom', $execution['requestPath']); - $this->assertEquals('GET', $execution['requestMethod']); - $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); $executionId = $execution['body']['$id']; @@ -301,7 +297,11 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(200, $execution['headers']['status-code']); + $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('completed', $execution['body']['status']); + $this->assertEquals('/custom', $execution['requestPath']); + $this->assertEquals('GET', $execution['requestMethod']); + $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); /* Test for FAILURE */ From e8840090756f120eb37335d764c44d84d5038b02 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:14:07 +0100 Subject: [PATCH 17/22] fix: test --- app/controllers/api/functions.php | 1 + tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2fcdfa92de..b4ef380d8f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1769,6 +1769,7 @@ App::post('/v1/functions/:functionId/executions') 'path' => $path, 'method' => $method, 'body' => $body, + 'jwt' => $jwt, ]; $dbForConsole->createDocument('schedules', new Document([ diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 50767a4c1e..a8e5a1ed3c 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -299,9 +299,9 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(200, $execution['headers']['status-code']); $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('completed', $execution['body']['status']); - $this->assertEquals('/custom', $execution['requestPath']); - $this->assertEquals('GET', $execution['requestMethod']); - $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); + $this->assertEquals('/custom', $execution['body']['requestPath']); + $this->assertEquals('GET', $execution['body']['requestMethod']); + $this->assertEquals(['content-type' => 'application/plain'], $execution['body']['requestHeaders']); /* Test for FAILURE */ From fd12449cc3a3abb4e80ebeab7c8be150a14e14e7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:01:53 +0100 Subject: [PATCH 18/22] test: fix --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index a8e5a1ed3c..f824f47131 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -301,7 +301,6 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('completed', $execution['body']['status']); $this->assertEquals('/custom', $execution['body']['requestPath']); $this->assertEquals('GET', $execution['body']['requestMethod']); - $this->assertEquals(['content-type' => 'application/plain'], $execution['body']['requestHeaders']); /* Test for FAILURE */ From bb3ee810654e172ef48c625c53b9e9412726b1b1 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 28 Jun 2024 22:42:55 +0100 Subject: [PATCH 19/22] chore: rename metadata to data --- app/config/collections.php | 4 ++-- app/controllers/api/functions.php | 4 ++-- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 2b7777e85b..1b7036c587 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4551,7 +4551,7 @@ $consoleCollections = array_merge([ 'filters' => [], ], [ - '$id' => ID::custom('metadata'), + '$id' => ID::custom('data'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 65535, @@ -4559,7 +4559,7 @@ $consoleCollections = array_merge([ 'required' => false, 'default' => new \stdClass(), 'array' => false, - 'filters' => ['json'], + 'filters' => ['json', 'encrypt'], ], [ '$id' => ID::custom('active'), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b4ef380d8f..c6ab7d95d2 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1764,7 +1764,7 @@ App::post('/v1/functions/:functionId/executions') ->setParam('executionId', $execution->getId()) ->trigger(); } else { - $metadata = [ + $data = [ 'headers' => $headers, 'path' => $path, 'method' => $method, @@ -1780,7 +1780,7 @@ App::post('/v1/functions/:functionId/executions') 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), 'schedule' => $scheduledAt, - 'metadata' => $metadata, + 'data' => $data, 'active' => true, ])); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 55cefb2a44..a47e7b120e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -46,10 +46,10 @@ class ScheduleExecutions extends ScheduleBase // TODO: Refactor to use function instead of functionId ->setFunctionId($schedule['resource']['functionId']) ->setExecution($schedule['resource']) - ->setMethod($schedule['metadata']['method'] ?? 'POST') - ->setPath($schedule['metadata']['path'] ?? '/') - ->setHeaders($schedule['metadata']['headers'] ?? []) - ->setBody($schedule['metadata']['body'] ?? '') + ->setMethod($schedule['data']['method'] ?? 'POST') + ->setPath($schedule['data']['path'] ?? '/') + ->setHeaders($schedule['data']['headers'] ?? []) + ->setBody($schedule['data']['body'] ?? '') ->setProject($schedule['project']) ->trigger(); From fef22825fb0d384490197aa9aaae3ec8da40dad0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:35:37 +0100 Subject: [PATCH 20/22] chore: delete schedule if not active --- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index a47e7b120e..2fdbd98da3 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -29,6 +29,11 @@ class ScheduleExecutions extends ScheduleBase foreach ($this->schedules as $schedule) { if (!$schedule['active']) { + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); + unset($this->schedules[$schedule['resourceId']]); continue; } From f1603ecb6a233ff07a9646087ede113ed5c81649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 Jul 2024 17:15:20 +0000 Subject: [PATCH 21/22] Upgrade executor --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b68584d685..91826b4c68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -850,7 +850,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.5.5 + image: openruntimes/executor:0.6.0 restart: unless-stopped networks: - appwrite From 7dec12698796f129a32db2a9e5472362b206d433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 Jul 2024 17:16:53 +0000 Subject: [PATCH 22/22] Revert unwanted change --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 91826b4c68..b68584d685 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -850,7 +850,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.6.0 + image: openruntimes/executor:0.5.5 restart: unless-stopped networks: - appwrite