From b8b81a9bd1a1ffc8cd6600a1dd214324a8620a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Jun 2024 19:05:29 +0000 Subject: [PATCH 01/38] WIP: Schedulded executions --- app/controllers/api/functions.php | 59 +++++++++++++------ app/controllers/api/messaging.php | 6 +- .../Platform/Tasks/ScheduleFunctions.php | 2 +- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 392cf034f9..900147398a 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -32,6 +32,7 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Datetime as DatetimeValidator; use Utopia\Database\Validator\Roles; use Utopia\Database\Validator\UID; use Utopia\Storage\Device; @@ -1511,16 +1512,21 @@ App::post('/v1/functions/:functionId/executions') ->param('path', '/', new Text(2048), 'HTTP path of execution. Path can include query params. Default value is /', true) ->param('method', 'POST', new Whitelist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], true), 'HTTP method of execution. Default value is GET.', true) ->param('headers', [], new Assoc(), 'HTTP headers of execution. Defaults to empty.', true) + ->param('scheduledAt', null, new DatetimeValidator(requireDateInFuture: true), 'Scheduled execution time in [ISO 8601](https://www.iso.org/iso-8601-date-and-time-format.html) format. DateTime value must be in future.', true) ->inject('response') ->inject('project') ->inject('dbForProject') + ->inject('dbForConsole') ->inject('user') ->inject('queueForEvents') ->inject('queueForUsage') - ->inject('mode') ->inject('queueForFunctions') ->inject('geodb') - ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, ?string $scheduledAt, Response $response, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { + + if(!$async && !is_null($scheduledAt)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, 'Scheduled executions must run asynchronously. Don\'t set scheduledAt to execute immediately, or set async to true.'); + } $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1625,6 +1631,12 @@ App::post('/v1/functions/:functionId/executions') $executionId = ID::unique(); + $status = $async ? 'waiting' : 'processing'; + + if(!is_null($scheduledAt)) { + $status = 'scheduled'; + } + $execution = new Document([ '$id' => $executionId, '$permissions' => !$user->isEmpty() ? [Permission::read(Role::user($user->getId()))] : [], @@ -1633,7 +1645,7 @@ App::post('/v1/functions/:functionId/executions') 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'trigger' => 'http', // http / schedule / event - 'status' => $async ? 'waiting' : 'processing', // waiting / processing / completed / failed + 'status' => $status, // waiting / processing / completed / failed 'responseStatusCode' => 0, 'responseHeaders' => [], 'requestPath' => $path, @@ -1656,20 +1668,33 @@ App::post('/v1/functions/:functionId/executions') $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); } - $queueForFunctions - ->setType('http') - ->setExecution($execution) - ->setFunction($function) - ->setBody($body) - ->setHeaders($headers) - ->setPath($path) - ->setMethod($method) - ->setJWT($jwt) - ->setProject($project) - ->setUser($user) - ->setParam('functionId', $function->getId()) - ->setParam('executionId', $execution->getId()) - ->trigger(); + if(is_null($scheduledAt)) { + $queueForFunctions + ->setType('http') + ->setExecution($execution) + ->setFunction($function) + ->setBody($body) + ->setHeaders($headers) + ->setPath($path) + ->setMethod($method) + ->setJWT($jwt) + ->setProject($project) + ->setUser($user) + ->setParam('functionId', $function->getId()) + ->setParam('executionId', $execution->getId()) + ->trigger(); + } else { + $dbForConsole->createDocument('schedules', new Document([ + 'region' => System::getEnv('_APP_REGION', 'default'), + 'resourceType' => 'function', + 'resourceId' => $function->getId(), + 'resourceInternalId' => $function->getInternalId(), + 'resourceUpdatedAt' => DateTime::now(), + 'projectId' => $project->getId(), + 'schedule' => $scheduledAt, + 'active' => true, + ])); + } return $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index e3696cc7e0..7da0348a8f 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2697,7 +2697,7 @@ App::post('/v1/messaging/messages/email') 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $scheduledAt, + 'schedule' => $scheduledAt, 'active' => true, ])); @@ -2813,7 +2813,7 @@ App::post('/v1/messaging/messages/sms') 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $scheduledAt, + 'schedule' => $scheduledAt, 'active' => true, ])); @@ -2989,7 +2989,7 @@ App::post('/v1/messaging/messages/push') 'resourceInternalId' => $message->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), - 'schedule' => $scheduledAt, + 'schedule' => $scheduledAt, 'active' => true, ])); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index e2c278714f..e8941c2ffa 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -41,7 +41,7 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine foreach ($this->schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); + $cron = new CronExpression($schedule['schedule']); // TODO: Allow schedule to be DateTime, like ScheduleMessaging.php $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); From 7e8f72d267ba8cf8bf80f7a607dfd87a787fe09e Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:39:38 +0100 Subject: [PATCH 02/38] feat: implement scheduledAt in schedule --- .../Platform/Tasks/ScheduleFunctions.php | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index e8941c2ffa..c8e68e0d1e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -40,32 +40,39 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine - foreach ($this->schedules as $key => $schedule) { - $cron = new CronExpression($schedule['schedule']); // TODO: Allow schedule to be DateTime, like ScheduleMessaging.php - $nextDate = $cron->getNextRunDate(); - $next = DateTime::format($nextDate); + foreach ($this->schedules as $scheduleKey => $schedule) { + if (CronExpression::isValidExpression($schedule['schedule'])) { + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); + } else { + try { + $nextDate = new \DateTime($schedule['schedule']); + $schedule['delete'] = true; + } catch (\Exception) { + Console::error('Failed to parse schedule: ' . $schedule['schedule']); + continue; + } + } + $next = DateTime::format($nextDate); $currentTick = $next < $timeFrame; if (!$currentTick) { continue; } - $total++; - - $promiseStart = \time(); // in seconds - $executionStart = $nextDate->getTimestamp(); // in seconds - $delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued + $total += 1; + $delay = $nextDate->getTimestamp() - \time(); // Time to wait from now until execution needs to be queued if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; } - $delayedExecutions[$delay][] = $key; + $delayedExecutions[$delay][] = $scheduleKey; } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools) { + \go(function () use ($delay, $scheduleKeys, $pools, $dbForConsole) { \sleep($delay); // in seconds $queue = $pools->get('queue')->pop(); @@ -76,7 +83,6 @@ class ScheduleFunctions extends ScheduleBase if (!\array_key_exists($scheduleKey, $this->schedules)) { return; } - $schedule = $this->schedules[$scheduleKey]; $queueForFunctions = new Func($connection); @@ -88,6 +94,13 @@ class ScheduleFunctions extends ScheduleBase ->setPath('/') ->setProject($schedule['project']) ->trigger(); + + if ($schedule['delete']) { + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); + } } $queue->reclaim(); From 1e9ced2878af41f08502f18f51d8b259eb1c513c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:57:03 +0100 Subject: [PATCH 03/38] test: `scheduled` status --- .../Platform/Tasks/ScheduleFunctions.php | 2 +- .../Functions/FunctionsCustomClientTest.php | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index c8e68e0d1e..afde51f23b 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -51,7 +51,7 @@ class ScheduleFunctions extends ScheduleBase } catch (\Exception) { Console::error('Failed to parse schedule: ' . $schedule['schedule']); continue; - } + } } $next = DateTime::format($nextDate); diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 12dd7cda59..db4421dd3e 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -175,6 +175,100 @@ class FunctionsCustomClientTest extends Scope return []; } + public function testCreateScheduledExecution(): void + { + /** + * Test for SUCCESS + */ + $function = $this->client->call(Client::METHOD_POST, '/functions', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'execute' => [Role::user($this->getUser()['$id'])->toString()], + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'events' => [ + 'users.*.create', + 'users.*.delete', + ], + 'timeout' => 10, + ]); + + $this->assertEquals(201, $function['headers']['status-code']); + + $folder = 'php'; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz"; + $this->packageCode($folder); + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/deployments', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', \basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); + + $function = $this->client->call(Client::METHOD_PATCH, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(200, $function['headers']['status-code']); + + // Schedule execution for the future + $futureTime = (new \DateTime())->add(new \DateInterval('PT1M'))->format('Y-m-d H:i:s'); + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'async' => true, + 'scheduledAt' => $futureTime, + ]); + + $this->assertEquals(202, $execution['headers']['status-code']); + $this->assertEquals('scheduled', $execution['body']['status'], \json_encode($execution['body'])); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $function['body']['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } + + public function testCreateCustomExecution(): array { /** From 865b12ba9dac5e16a0e6ee460b93b2f0d36bfdd3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:24:51 +0100 Subject: [PATCH 04/38] chore: exception type --- app/controllers/api/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 900147398a..5edfcc3069 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1525,7 +1525,7 @@ App::post('/v1/functions/:functionId/executions') ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, ?string $scheduledAt, Response $response, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) { if(!$async && !is_null($scheduledAt)) { - throw new Exception(Exception::GENERAL_QUERY_INVALID, 'Scheduled executions must run asynchronously. Don\'t set scheduledAt to execute immediately, or set async to true.'); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Scheduled executions must run asynchronously. Set scheduledAt to a future date, or set async to true.'); } $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); From cfdd40dbeb90ea2d40ef289cc605320b6bf554dc Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:36:01 +0100 Subject: [PATCH 05/38] test: execution is completed --- .../Functions/FunctionsCustomClientTest.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index db4421dd3e..7ee209af03 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -246,7 +246,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(200, $function['headers']['status-code']); // Schedule execution for the future - $futureTime = (new \DateTime())->add(new \DateInterval('PT1M'))->format('Y-m-d H:i:s'); + $futureTime = (new \DateTime())->add(new \DateInterval('PT10S'))->format('Y-m-d H:i:s'); $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -258,6 +258,17 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(202, $execution['headers']['status-code']); $this->assertEquals('scheduled', $execution['body']['status'], \json_encode($execution['body'])); + $executionId = $execution['body']['$id']; + + \sleep(12); + + $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); + // Cleanup : Delete function $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $function['body']['$id'], [ 'content-type' => 'application/json', @@ -269,6 +280,7 @@ class FunctionsCustomClientTest extends Scope } + public function testCreateCustomExecution(): array { /** From a83d125f54544f399209aacda447a7b66b4b1fd5 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:34:10 +0100 Subject: [PATCH 06/38] chore: adjust timers --- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index afde51f23b..fd417ee274 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -11,8 +11,8 @@ use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase { - public const UPDATE_TIMER = 10; // seconds - public const ENQUEUE_TIMER = 60; // seconds + public const UPDATE_TIMER = 3; // seconds + public const ENQUEUE_TIMER = 4; // seconds private ?float $lastEnqueueUpdate = null; From dff9bed882155c78eb0db917c346c5aa759cfe09 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:41:37 +0100 Subject: [PATCH 07/38] test: increase delay --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 7ee209af03..b72443ed6a 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -260,7 +260,7 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - \sleep(12); + \sleep(20); $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ 'content-type' => 'application/json', From 6f38ef3a18da21fafb0402389c26ff96b6f969f4 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:22:38 +0100 Subject: [PATCH 08/38] test: poll scheduled function --- .../Functions/FunctionsCustomClientTest.php | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index b72443ed6a..0ca17e6061 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -260,13 +260,24 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - \sleep(20); + while (true) { + + $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $execution['headers']['status-code'] >= 400 + || \in_array($execution['body']['status'], ['completed', 'failed']) + ) { + break; + } + + \sleep(1); + } - $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); // Cleanup : Delete function From 10e37bb8ca52a3a6ead24a076d43a9c442e33ef8 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:01:04 +0100 Subject: [PATCH 09/38] test: fix --- .../Functions/FunctionsCustomClientTest.php | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0ca17e6061..0e70794879 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -260,24 +260,15 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - while (true) { + sleep(12); - $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); - - if ( - $execution['headers']['status-code'] >= 400 - || \in_array($execution['body']['status'], ['completed', 'failed']) - ) { - break; - } - - \sleep(1); - } + $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals(200, $execution['headers']['status-code']); $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); // Cleanup : Delete function From 2f0f7bf9c764725d98ce3b6ef3412a3b0ac09ad9 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:44:12 +0100 Subject: [PATCH 10/38] fix: test --- Dockerfile | 1 + app/controllers/api/functions.php | 8 +-- app/views/install/compose.phtml | 25 +++++++ bin/schedule-executions | 3 + docker-compose.yml | 29 +++++++- src/Appwrite/Event/Func.php | 24 +++++++ src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/ScheduleBase.php | 6 +- .../Platform/Tasks/ScheduleExecutions.php | 67 +++++++++++++++++++ .../Platform/Tasks/ScheduleFunctions.php | 42 +++++------- .../Platform/Tasks/ScheduleMessages.php | 2 +- src/Appwrite/Platform/Workers/Functions.php | 9 ++- .../Functions/FunctionsCustomClientTest.php | 6 +- 13 files changed, 185 insertions(+), 39 deletions(-) create mode 100644 bin/schedule-executions create mode 100644 src/Appwrite/Platform/Tasks/ScheduleExecutions.php diff --git a/Dockerfile b/Dockerfile index 1d2ac91ae0..d7e9849b1b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ chmod +x /usr/local/bin/schedule-functions && \ + chmod +x /usr/local/bin/schedule-executions && \ chmod +x /usr/local/bin/schedule-messages && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index f8690ebce6..5aa3083f3c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1722,7 +1722,7 @@ App::post('/v1/functions/:functionId/executions') 'functionId' => $function->getId(), 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), - 'trigger' => 'http', // http / schedule / event + 'trigger' => (!is_null($scheduledAt)) ? 'schedule' : 'http', 'status' => $status, // waiting / processing / completed / failed 'responseStatusCode' => 0, 'responseHeaders' => [], @@ -1764,9 +1764,9 @@ App::post('/v1/functions/:functionId/executions') } else { $dbForConsole->createDocument('schedules', new Document([ 'region' => System::getEnv('_APP_REGION', 'default'), - 'resourceType' => 'function', - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getInternalId(), + 'resourceType' => 'execution', + 'resourceId' => $execution->getId(), + 'resourceInternalId' => $execution->getInternalId(), 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), 'schedule' => $scheduledAt, diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f265f53c19..9643440f5e 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -676,6 +676,31 @@ services: - _APP_DB_USER - _APP_DB_PASS + appwrite-task-scheduler-executions: + image: /: + entrypoint: schedule-executions + container_name: appwrite-task-scheduler-executions + <<: *x-logging + restart: unless-stopped + networks: + - appwrite + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + appwrite-task-scheduler-messages: image: /: entrypoint: schedule-messages diff --git a/bin/schedule-executions b/bin/schedule-executions new file mode 100644 index 0000000000..f239cad206 --- /dev/null +++ b/bin/schedule-executions @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php schedule-executions $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 250eb8b7aa..6dd2109c6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,6 @@ x-logging: &x-logging max-file: "5" max-size: "10m" -version: "3" - services: traefik: image: traefik:2.11 @@ -742,6 +740,33 @@ services: - _APP_DB_USER - _APP_DB_PASS + appwrite-task-scheduler-executions: + entrypoint: schedule-executions + <<: *x-logging + container_name: appwrite-task-scheduler-executions + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + appwrite-task-scheduler-messages: entrypoint: schedule-messages <<: *x-logging diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 11c9e980ed..67c28575bd 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -14,6 +14,7 @@ class Func extends Event protected string $path = ''; protected string $method = ''; protected array $headers = []; + protected ?string $functionId = null; protected ?Document $function = null; protected ?Document $execution = null; @@ -49,6 +50,28 @@ class Func extends Event return $this->function; } + /** + * Sets function id for the function event. + * + * @param string $functionId + */ + public function setFunctionId(string $functionId): self + { + $this->functionId = $functionId; + + return $this; + } + + /** + * Returns set function id for the function event. + * + * @return string|null + */ + public function getFunctionId(): ?string + { + return $this->functionId; + } + /** * Sets execution for the function event. * @@ -200,6 +223,7 @@ class Func extends Event 'project' => $this->project, 'user' => $this->user, 'function' => $this->function, + 'functionId' => $this->functionId, 'execution' => $this->execution, 'type' => $this->type, 'jwt' => $this->jwt, diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index ac1f99eec3..b7b333b2c6 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -9,6 +9,7 @@ use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueCount; use Appwrite\Platform\Tasks\QueueRetry; use Appwrite\Platform\Tasks\ScheduleFunctions; +use Appwrite\Platform\Tasks\ScheduleExecutions; use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; @@ -33,6 +34,7 @@ class Tasks extends Service ->addAction(SDKs::getName(), new SDKs()) ->addAction(SSL::getName(), new SSL()) ->addAction(ScheduleFunctions::getName(), new ScheduleFunctions()) + ->addAction(ScheduleExecutions::getName(), new ScheduleExecutions()) ->addAction(ScheduleMessages::getName(), new ScheduleMessages()) ->addAction(Specs::getName(), new Specs()) ->addAction(Upgrade::getName(), new Upgrade()) diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index a50fbb2403..be0abc4b66 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -64,7 +64,8 @@ abstract class ScheduleBase extends Action $collectionId = match ($schedule->getAttribute('resourceType')) { 'function' => 'functions', - 'message' => 'messages' + 'message' => 'messages', + 'execution' => 'executions' }; $resource = $getProjectDB($project)->getDocument( @@ -113,7 +114,8 @@ abstract class ScheduleBase extends Action } catch (\Throwable $th) { $collectionId = match ($document->getAttribute('resourceType')) { 'function' => 'functions', - 'message' => 'messages' + 'message' => 'messages', + 'execution' => 'executions' }; Console::error("Failed to load schedule for project {$document['projectId']} {$collectionId} {$document['resourceId']}"); diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php new file mode 100644 index 0000000000..01dde1e88b --- /dev/null +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -0,0 +1,67 @@ +schedules as $schedule) { + if (!$schedule['active'] || CronExpression::isValidExpression($schedule['schedule'])) { + unset($this->schedules[$schedule['resourceId']]); + continue; + } + + $now = new \DateTime(); + $scheduledAt = new \DateTime($schedule['schedule']); + + if ($scheduledAt > $now) { + continue; + } + + \go(function () use ($schedule, $pools, $dbForConsole) { + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + + $queueForFunctions = new Func($connection); + + $queueForFunctions + ->setType('schedule') + ->setFunctionId($schedule['resource']['functionId']) + ->setExecution($schedule['resource']) + ->setMethod('POST') + ->setPath('/') + ->setProject($schedule['project']) + ->trigger(); + + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); + + $queue->reclaim(); + + unset($this->schedules[$schedule['resourceId']]); + }); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index fd417ee274..6ea972af1e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -11,8 +11,8 @@ use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase { - public const UPDATE_TIMER = 3; // seconds - public const ENQUEUE_TIMER = 4; // seconds + public const UPDATE_TIMER = 10; // seconds + public const ENQUEUE_TIMER = 60; // seconds private ?float $lastEnqueueUpdate = null; @@ -40,39 +40,37 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine - foreach ($this->schedules as $scheduleKey => $schedule) { - if (CronExpression::isValidExpression($schedule['schedule'])) { - $cron = new CronExpression($schedule['schedule']); - $nextDate = $cron->getNextRunDate(); - } else { - try { - $nextDate = new \DateTime($schedule['schedule']); - $schedule['delete'] = true; - } catch (\Exception) { - Console::error('Failed to parse schedule: ' . $schedule['schedule']); - continue; - } + foreach ($this->schedules as $key => $schedule) { + if (!$schedule['active'] || !CronExpression::isValidExpression($schedule['schedule'])) { + unset($this->schedules[$schedule['resourceId']]); + continue; } + $cron = new CronExpression($schedule['schedule']); + $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); + $currentTick = $next < $timeFrame; if (!$currentTick) { continue; } - $total += 1; - $delay = $nextDate->getTimestamp() - \time(); // Time to wait from now until execution needs to be queued + $total++; + + $promiseStart = \time(); // in seconds + $executionStart = $nextDate->getTimestamp(); // in seconds + $delay = $executionStart - $promiseStart; // Time to wait from now until execution needs to be queued if (!isset($delayedExecutions[$delay])) { $delayedExecutions[$delay] = []; } - $delayedExecutions[$delay][] = $scheduleKey; + $delayedExecutions[$delay][] = $key; } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools, $dbForConsole) { + \go(function () use ($delay, $scheduleKeys, $pools) { \sleep($delay); // in seconds $queue = $pools->get('queue')->pop(); @@ -83,6 +81,7 @@ class ScheduleFunctions extends ScheduleBase if (!\array_key_exists($scheduleKey, $this->schedules)) { return; } + $schedule = $this->schedules[$scheduleKey]; $queueForFunctions = new Func($connection); @@ -94,13 +93,6 @@ class ScheduleFunctions extends ScheduleBase ->setPath('/') ->setProject($schedule['project']) ->trigger(); - - if ($schedule['delete']) { - $dbForConsole->deleteDocument( - 'schedules', - $schedule['$id'], - ); - } } $queue->reclaim(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 8e52973a0c..145b6ee976 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -35,7 +35,7 @@ class ScheduleMessages extends ScheduleBase continue; } - \go(function () use ($now, $schedule, $pools, $dbForConsole) { + \go(function () use ($schedule, $pools, $dbForConsole) { $queue = $pools->get('queue')->pop(); $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index cbba9657ad..734fbab602 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -83,6 +83,7 @@ class Functions extends Action $eventData = $payload['payload'] ?? ''; $project = new Document($payload['project'] ?? []); $function = new Document($payload['function'] ?? []); + $functionId = $payload['functionId'] ?? ''; $user = new Document($payload['user'] ?? []); $method = $payload['method'] ?? 'POST'; $headers = $payload['headers'] ?? []; @@ -92,6 +93,10 @@ class Functions extends Action return; } + if ($function->isEmpty() && !empty($functionId)) { + $function = $dbForProject->getDocument('functions', $functionId); + } + $log->addTag('functionId', $function->getId()); $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); @@ -176,6 +181,7 @@ class Functions extends Action ); break; case 'schedule': + $execution = new Document($payload['execution'] ?? []); $this->execute( log: $log, dbForProject: $dbForProject, @@ -193,7 +199,7 @@ class Functions extends Action jwt: null, event: null, eventData: null, - executionId: null, + executionId: $execution->getId() ?? null ); break; } @@ -296,7 +302,6 @@ class Functions extends Action $headers['x-appwrite-user-id'] = $user->getId() ?? ''; $headers['x-appwrite-user-jwt'] = $jwt ?? ''; - /** Create execution or update execution status */ /** Create execution or update execution status */ $execution = $dbForProject->getDocument('executions', $executionId ?? ''); if ($execution->isEmpty()) { diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0e70794879..246bed8c51 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -57,6 +57,7 @@ class FunctionsCustomClientTest extends Scope 'execute' => [Role::user($this->getUser()['$id'])->toString()], 'runtime' => 'php-8.0', 'entrypoint' => 'index.php', + 'logging' => true, 'events' => [ 'users.*.create', 'users.*.delete', @@ -246,6 +247,7 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(200, $function['headers']['status-code']); // Schedule execution for the future + \date_default_timezone_set('UTC'); $futureTime = (new \DateTime())->add(new \DateInterval('PT10S'))->format('Y-m-d H:i:s'); $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ 'content-type' => 'application/json', @@ -260,7 +262,7 @@ class FunctionsCustomClientTest extends Scope $executionId = $execution['body']['$id']; - sleep(12); + sleep(20); $execution = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/executions/' . $executionId, [ 'content-type' => 'application/json', @@ -281,8 +283,6 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); } - - public function testCreateCustomExecution(): array { /** From 83ffc41d92ea438bec099bfe6dbeb743c7826092 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:46:18 +0100 Subject: [PATCH 11/38] chore: fmt --- src/Appwrite/Event/Func.php | 4 ++-- src/Appwrite/Platform/Services/Tasks.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 67c28575bd..451df2b6c1 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -52,7 +52,7 @@ class Func extends Event /** * Sets function id for the function event. - * + * * @param string $functionId */ public function setFunctionId(string $functionId): self @@ -64,7 +64,7 @@ class Func extends Event /** * Returns set function id for the function event. - * + * * @return string|null */ public function getFunctionId(): ?string diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index b7b333b2c6..999270d2dc 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,8 +8,8 @@ use Appwrite\Platform\Tasks\Maintenance; use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueCount; use Appwrite\Platform\Tasks\QueueRetry; -use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleExecutions; +use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 01dde1e88b..14d5632000 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -4,7 +4,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Cron\CronExpression; -use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; @@ -41,9 +40,9 @@ class ScheduleExecutions extends ScheduleBase \go(function () use ($schedule, $pools, $dbForConsole) { $queue = $pools->get('queue')->pop(); $connection = $queue->getResource(); - + $queueForFunctions = new Func($connection); - + $queueForFunctions ->setType('schedule') ->setFunctionId($schedule['resource']['functionId']) From 5f144f91adbb51f21f3dabf21def612e7b6834f3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:12:02 +0100 Subject: [PATCH 12/38] chore: matej review --- .../Platform/Tasks/ScheduleExecutions.php | 42 ++++++++----------- .../Platform/Tasks/ScheduleFunctions.php | 7 ---- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 14d5632000..b0f03a35ea 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; -use Cron\CronExpression; use Utopia\Database\Database; use Utopia\Pools\Group; @@ -24,8 +23,12 @@ class ScheduleExecutions extends ScheduleBase protected function enqueueResources(Group $pools, Database $dbForConsole): void { + $queue = $pools->get('queue')->pop(); + $connection = $queue->getResource(); + $queueForFunctions = new Func($connection); + foreach ($this->schedules as $schedule) { - if (!$schedule['active'] || CronExpression::isValidExpression($schedule['schedule'])) { + if (!$schedule['active']) { unset($this->schedules[$schedule['resourceId']]); continue; } @@ -37,30 +40,21 @@ class ScheduleExecutions extends ScheduleBase continue; } - \go(function () use ($schedule, $pools, $dbForConsole) { - $queue = $pools->get('queue')->pop(); - $connection = $queue->getResource(); + $queueForFunctions + ->setType('schedule') + ->setFunctionId($schedule['resource']['functionId']) + ->setExecution($schedule['resource']) + ->setProject($schedule['project']) + ->trigger(); - $queueForFunctions = new Func($connection); + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); - $queueForFunctions - ->setType('schedule') - ->setFunctionId($schedule['resource']['functionId']) - ->setExecution($schedule['resource']) - ->setMethod('POST') - ->setPath('/') - ->setProject($schedule['project']) - ->trigger(); - - $dbForConsole->deleteDocument( - 'schedules', - $schedule['$id'], - ); - - $queue->reclaim(); - - unset($this->schedules[$schedule['resourceId']]); - }); + unset($this->schedules[$schedule['resourceId']]); } + + $queue->reclaim(); } } diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 6ea972af1e..e4832d7435 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -41,11 +41,6 @@ class ScheduleFunctions extends ScheduleBase $delayedExecutions = []; // Group executions with same delay to share one coroutine foreach ($this->schedules as $key => $schedule) { - if (!$schedule['active'] || !CronExpression::isValidExpression($schedule['schedule'])) { - unset($this->schedules[$schedule['resourceId']]); - continue; - } - $cron = new CronExpression($schedule['schedule']); $nextDate = $cron->getNextRunDate(); $next = DateTime::format($nextDate); @@ -89,8 +84,6 @@ class ScheduleFunctions extends ScheduleBase $queueForFunctions ->setType('schedule') ->setFunction($schedule['resource']) - ->setMethod('POST') - ->setPath('/') ->setProject($schedule['project']) ->trigger(); } From 67fddeb4faba852b3a94974b09629dc3e2f606b6 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:23:14 +0100 Subject: [PATCH 13/38] fix: path and method --- src/Appwrite/Event/Func.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 451df2b6c1..86080593af 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -11,8 +11,8 @@ class Func extends Event protected string $jwt = ''; protected string $type = ''; protected string $body = ''; - protected string $path = ''; - protected string $method = ''; + protected ?string $path = null; + protected ?string $method = null; protected array $headers = []; protected ?string $functionId = null; protected ?Document $function = null; From f3f233eb14b1e908939ec61d21ad8f0cd60785ac Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 21 Jun 2024 19:21:05 +0100 Subject: [PATCH 14/38] feat: new session alert --- .../locale/templates/email-session-alert.tpl | 14 +++ app/config/locale/translations/en.json | 9 ++ app/controllers/api/account.php | 87 +++++++++++++++++++ app/controllers/api/projects.php | 43 ++++++++- .../Utopia/Response/Model/Project.php | 16 ++-- 5 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 app/config/locale/templates/email-session-alert.tpl diff --git a/app/config/locale/templates/email-session-alert.tpl b/app/config/locale/templates/email-session-alert.tpl new file mode 100644 index 0000000000..5c182a7df5 --- /dev/null +++ b/app/config/locale/templates/email-session-alert.tpl @@ -0,0 +1,14 @@ +

{{hello}},

+ +

{{body}}

+ +
    +
  1. {{device}}
  2. +
  3. {{ipAddress}}
  4. +
  5. {{country}}
  6. +
+ +

{{footer}}

+ +

{{thanks}}

+

{{signature}}

diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index f93dfcc0fd..7cc7fc8485 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -18,6 +18,15 @@ "emails.magicSession.securityPhrase": "Security phrase for this email is {{b}}{{phrase}}{{/b}}. You can trust this email if this phrase matches the phrase shown during sign in.", "emails.magicSession.thanks": "Thanks,", "emails.magicSession.signature": "{{project}} team", + "emails.sessionAlert.subject": "New session alert for {{project}}", + "emails.sessionAlert.hello":"Hello {{user}}", + "emails.sessionAlert.body": "We're writing to inform you that a new session has been initiated on your {{b}}{{project}}{{/b}} account, on {{b}}{{dateTime}}{{/b}}. \nHere are the details of the new session: ", + "emails.sessionAlert.device": "Device: {{b}}{{agentDevice}}{{/b}}", + "emails.sessionAlert.ipAddress": "IP Address: {{b}}{{ipAddress}}{{/b}}", + "emails.sessionAlert.country": "Country: {{b}}{{country}}{{/b}}", + "emails.sessionAlert.footer": "If you didn't request the sign in, you can safely ignore this email. If you suspect unauthorized activity, please secure your account immediately.", + "emails.sessionAlert.thanks": "Thanks,", + "emails.sessionAlert.signature": "{{project}} team", "emails.otpSession.subject": "OTP for {{project}} Login", "emails.otpSession.hello": "Hello {{user}}", "emails.otpSession.description": "Enter the following verification code when prompted to securely sign in to your {{b}}{{project}}{{/b}} account. This code will expire in 15 minutes.", diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 6504052f9a..e706457d62 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -166,6 +166,93 @@ $createSession = function (string $userId, string $secret, Request $request, Res $response->dynamic($session, Response::MODEL_SESSION); }; +$sendSessionEmail = function (Request $request, Locale $locale, Document $user, Document $project, Reader $geodb, Event $queueForMails) { + $subject = $locale->getText("emails.sessionAlert.subject"); + $customTemplate = $project->getAttribute('templates', [])['email.sessionAlert-' . $locale->default] ?? []; + + $detector = new Detector($request->getUserAgent('UNKNOWN')); + $agentDevice = $detector->getDevice(); + + $record = $geodb->get($request->getIP()); + $countryCode = $record['country']['iso_code'] ?? ''; + + $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-session-alert.tpl'); + $message + ->setParam('{{hello}}', $locale->getText("emails.sessionAlert.hello")) + ->setParam('{{body}}', $locale->getText("emails.sessionAlert.body")) + ->setParam('{{device}}', $locale->getText("emails.sessionAlert.device")) + ->setParam('{{ipAddress}}', $locale->getText("emails.sessionAlert.ipAddress")) + ->setParam('{{country}}', $locale->getText("emails.sessionAlert.country")) + ->setParam('{{footer}}', $locale->getText("emails.sessionAlert.footer")) + ->setParam('{{signature}}', $locale->getText("emails.sessionAlert.signature")); + + $body = $message->render(); + + $smtp = $project->getAttribute('smtp', []); + $smtpEnabled = $smtp['enabled'] ?? false; + + $senderEmail = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); + $senderName = System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server'); + $replyTo = ""; + + if ($smtpEnabled) { + if (!empty($smtp['senderEmail'])) { + $senderEmail = $smtp['senderEmail']; + } + if (!empty($smtp['senderName'])) { + $senderName = $smtp['senderName']; + } + if (!empty($smtp['replyTo'])) { + $replyTo = $smtp['replyTo']; + } + + $queueForMails + ->setSmtpHost($smtp['host'] ?? '') + ->setSmtpPort($smtp['port'] ?? '') + ->setSmtpUsername($smtp['username'] ?? '') + ->setSmtpPassword($smtp['password'] ?? '') + ->setSmtpSecure($smtp['secure'] ?? ''); + + if (!empty($customTemplate)) { + if (!empty($customTemplate['senderEmail'])) { + $senderEmail = $customTemplate['senderEmail']; + } + if (!empty($customTemplate['senderName'])) { + $senderName = $customTemplate['senderName']; + } + if (!empty($customTemplate['replyTo'])) { + $replyTo = $customTemplate['replyTo']; + } + + $body = $customTemplate['message'] ?? ''; + $subject = $customTemplate['subject'] ?? $subject; + } + + $queueForMails + ->setSmtpReplyTo($replyTo) + ->setSmtpSenderEmail($senderEmail) + ->setSmtpSenderName($senderName); + } + + $emailVariables = [ + 'direction' => $locale->getText('settings.direction'), + // {{user}}, {{redirect}} and {{project}} are required in default and custom templates + 'user' => $user->getAttribute('name'), + 'project' => $project->getAttribute('name'), + 'redirect' => $url, + 'agentDevice' => $agentDevice['deviceBrand'] ?? $agentDevice['deviceBrand'] ?? 'UNKNOWN', + 'ipAddress' => $request->getIP(), + 'country' => $locale->getText('countries.' . strtolower($countryCode), $locale->getText('locale.country.unknown')), + ]; + + $queueForMails + ->setSubject($subject) + ->setBody($body) + ->setVariables($emailVariables) + ->setRecipient($email) + ->trigger(); +} + App::post('/v1/account') ->desc('Create account') ->groups(['api', 'account', 'auth']) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 234c8a9b9b..653d62b157 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -93,7 +93,15 @@ App::post('/v1/projects') } $auth = Config::getParam('auth', []); - $auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false]; + $auths = [ + 'limit' => 0, + 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, + 'passwordHistory' => 0, 'passwordDictionary' => false, + 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, + 'personalDataCheck' => false, + 'sessionEmails' => false, + ]; + foreach ($auth as $index => $method) { $auths[$method['key'] ?? ''] = true; } @@ -355,7 +363,7 @@ App::patch('/v1/projects/:projectId') }); App::patch('/v1/projects/:projectId/team') - ->desc('Update Project Team') + ->desc('Update project team') ->groups(['api', 'projects']) ->label('scope', 'projects.write') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -596,6 +604,37 @@ App::patch('/v1/projects/:projectId/oauth2') $response->dynamic($project, Response::MODEL_PROJECT); }); +App::patch('/v1/projects/:projectId/auth/session-emails') + ->desc('Update project sessions emails') + ->groups(['api', 'projects']) + ->label('scope', 'projects.write') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'projects') + ->label('sdk.method', 'updateSessionEmails') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_PROJECT) + ->param('projectId', '', new UID(), 'Project unique ID.') + ->param('sessionEmails', false, new Boolean(true), 'Set to true to enable session emails.') + ->inject('response') + ->inject('dbForConsole') + ->action(function (string $projectId, bool $sessionEmails, Response $response, Database $dbForConsole) { + + $project = $dbForConsole->getDocument('projects', $projectId); + + if ($project->isEmpty()) { + throw new Exception(Exception::PROJECT_NOT_FOUND); + } + + $auths = $project->getAttribute('auths', []); + $auths['sessionEmails'] = $sessionEmails; + + $dbForConsole->updateDocument('projects', $project->getId(), $project + ->setAttribute('auths', $auths)); + + $response->dynamic($project, Response::MODEL_PROJECT); + }); + App::patch('/v1/projects/:projectId/auth/limit') ->desc('Update project users limit') ->groups(['api', 'projects']) diff --git a/src/Appwrite/Utopia/Response/Model/Project.php b/src/Appwrite/Utopia/Response/Model/Project.php index 6bab4401d7..aa56daf788 100644 --- a/src/Appwrite/Utopia/Response/Model/Project.php +++ b/src/Appwrite/Utopia/Response/Model/Project.php @@ -138,6 +138,12 @@ class Project extends Model 'default' => false, 'example' => true, ]) + ->addRule('authSessionEmails', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Whether or not to send session emails to users.', + 'default' => false, + 'example' => true, + ]) ->addRule('oAuthProviders', [ 'type' => Response::MODEL_AUTH_PROVIDER, 'description' => 'List of Auth Providers.', @@ -220,8 +226,7 @@ class Project extends Model 'description' => 'SMTP server secure protocol', 'default' => '', 'example' => 'tls', - ]) - ; + ]); $services = Config::getParam('services', []); $auth = Config::getParam('auth', []); @@ -236,8 +241,7 @@ class Project extends Model 'description' => $name . ' auth method status', 'example' => true, 'default' => true, - ]) - ; + ]); } foreach ($services as $service) { @@ -254,8 +258,7 @@ class Project extends Model 'description' => $name . ' service status', 'example' => true, 'default' => true, - ]) - ; + ]); } } @@ -321,6 +324,7 @@ class Project extends Model $document->setAttribute('authPasswordHistory', $authValues['passwordHistory'] ?? 0); $document->setAttribute('authPasswordDictionary', $authValues['passwordDictionary'] ?? false); $document->setAttribute('authPersonalDataCheck', $authValues['personalDataCheck'] ?? false); + $document->setAttribute('authSessionEmails', $authValues['sessionEmails'] ?? false); foreach ($auth as $index => $method) { $key = $method['key']; From fbeca34df8c881deb7e28f4a4475c7bc52a76c6c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:34:06 +0100 Subject: [PATCH 15/38] fix: func --- app/controllers/api/account.php | 126 +++++++++++++++----------------- 1 file changed, 57 insertions(+), 69 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e706457d62..7cd964f788 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -160,13 +160,12 @@ $createSession = function (string $userId, string $secret, Request $request, Res ->setAttribute('current', true) ->setAttribute('countryName', $countryName) ->setAttribute('expire', $expire) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : '') - ; + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : ''); $response->dynamic($session, Response::MODEL_SESSION); }; -$sendSessionEmail = function (Request $request, Locale $locale, Document $user, Document $project, Reader $geodb, Event $queueForMails) { +$sendSessionEmail = function (Request $request, Locale $locale, Document $user, Document $project, Reader $geodb, Mail $queueForMails) { $subject = $locale->getText("emails.sessionAlert.subject"); $customTemplate = $project->getAttribute('templates', [])['email.sessionAlert-' . $locale->default] ?? []; @@ -236,22 +235,22 @@ $sendSessionEmail = function (Request $request, Locale $locale, Document $user, $emailVariables = [ 'direction' => $locale->getText('settings.direction'), - // {{user}}, {{redirect}} and {{project}} are required in default and custom templates 'user' => $user->getAttribute('name'), 'project' => $project->getAttribute('name'), - 'redirect' => $url, 'agentDevice' => $agentDevice['deviceBrand'] ?? $agentDevice['deviceBrand'] ?? 'UNKNOWN', 'ipAddress' => $request->getIP(), 'country' => $locale->getText('countries.' . strtolower($countryCode), $locale->getText('locale.country.unknown')), ]; + $email = $user->getAttribute('email'); + $queueForMails ->setSubject($subject) ->setBody($body) ->setVariables($emailVariables) ->setRecipient($email) ->trigger(); -} +}; App::post('/v1/account') ->desc('Create account') @@ -315,7 +314,8 @@ App::post('/v1/account') Query::equal('providerEmail', [$email]), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); + /** Return a generic bad request to prevent exposing existing accounts */ } if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) { @@ -492,7 +492,8 @@ App::get('/v1/account/sessions') $sessions = $user->getAttribute('sessions', []); $current = Auth::sessionVerify($sessions, Auth::$secret); - foreach ($sessions as $key => $session) {/** @var Document $session */ + foreach ($sessions as $key => $session) { + /** @var Document $session */ $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session->setAttribute('countryName', $countryName); @@ -534,7 +535,8 @@ App::delete('/v1/account/sessions') $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); - foreach ($sessions as $session) {/** @var Document $session */ + foreach ($sessions as $session) { + /** @var Document $session */ $dbForProject->deleteDocument('sessions', $session->getId()); if (!Config::getParam('domainVerification')) { @@ -602,15 +604,15 @@ App::get('/v1/account/sessions/:sessionId') ? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret) : $sessionId; - foreach ($sessions as $session) {/** @var Document $session */ + foreach ($sessions as $session) { + /** @var Document $session */ if ($sessionId === $session->getId()) { $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session ->setAttribute('current', ($session->getAttribute('secret') == Auth::hash(Auth::$secret))) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : '') - ; + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : ''); return $response->dynamic($session, Response::MODEL_SESSION); } @@ -769,8 +771,7 @@ App::patch('/v1/account/sessions/:sessionId') $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) - ->setPayload($response->output($session, Response::MODEL_SESSION)) - ; + ->setPayload($response->output($session, Response::MODEL_SESSION)); return $response->dynamic($session, Response::MODEL_SESSION); }); @@ -873,8 +874,7 @@ App::post('/v1/account/sessions/email') if (!Config::getParam('domainVerification')) { $response - ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) - ; + ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); } $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); @@ -882,21 +882,18 @@ App::post('/v1/account/sessions/email') $response ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) - ->setStatusCode(Response::STATUS_CODE_CREATED) - ; + ->setStatusCode(Response::STATUS_CODE_CREATED); $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '') - ; + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : ''); $queueForEvents ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()) - ; + ->setParam('sessionId', $session->getId()); $response->dynamic($session, Response::MODEL_SESSION); }); @@ -1003,7 +1000,7 @@ App::post('/v1/account/sessions/anonymous') Authorization::setRole(Role::user($user->getId())->toString()); - $session = $dbForProject->createDocument('sessions', $session-> setAttribute('$permissions', [ + $session = $dbForProject->createDocument('sessions', $session->setAttribute('$permissions', [ Permission::read(Role::user($user->getId())), Permission::update(Role::user($user->getId())), Permission::delete(Role::user($user->getId())), @@ -1013,8 +1010,7 @@ App::post('/v1/account/sessions/anonymous') $queueForEvents ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()) - ; + ->setParam('sessionId', $session->getId()); if (!Config::getParam('domainVerification')) { $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); @@ -1025,16 +1021,14 @@ App::post('/v1/account/sessions/anonymous') $response ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) - ->setStatusCode(Response::STATUS_CODE_CREATED) - ; + ->setStatusCode(Response::STATUS_CODE_CREATED); $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '') - ; + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : ''); $response->dynamic($session, Response::MODEL_SESSION); }); @@ -1419,7 +1413,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Query::equal('providerEmail', [$email]), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); + /** Return a generic bad request to prevent exposing existing accounts */ } try { @@ -1490,7 +1485,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Query::notEqual('userInternalId', $user->getInternalId()), ]); if (!empty($identitiesWithMatchingEmail)) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); + /** Return a generic bad request to prevent exposing existing accounts */ } $dbForProject->createDocument('identities', new Document([ @@ -1563,8 +1559,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $queueForEvents ->setEvent('users.[userId].tokens.[tokenId].create') ->setParam('userId', $user->getId()) - ->setParam('tokenId', $token->getId()) - ; + ->setParam('tokenId', $token->getId()); $query['secret'] = $secret; $query['userId'] = $user->getId(); @@ -1607,8 +1602,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) - ->setPayload($response->output($session, Response::MODEL_SESSION)) - ; + ->setPayload($response->output($session, Response::MODEL_SESSION)); // TODO: Remove this deprecated workaround - support only token if ($state['success']['path'] == $oauthDefaultSuccess) { @@ -1647,8 +1641,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $response ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') ->addHeader('Pragma', 'no-cache') - ->redirect($state['success']) - ; + ->redirect($state['success']); }); App::get('/v1/account/tokens/oauth2/:provider') @@ -1960,8 +1953,7 @@ App::post('/v1/account/tokens/magic-url') $response ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($token, Response::MODEL_TOKEN) - ; + ->dynamic($token, Response::MODEL_TOKEN); }); App::post('/v1/account/tokens/email') @@ -2024,7 +2016,8 @@ App::post('/v1/account/tokens/email') Query::equal('providerEmail', [$email]), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); + /** Return a generic bad request to prevent exposing existing accounts */ } $userId = $userId === 'unique()' ? ID::unique() : $userId; @@ -2189,8 +2182,7 @@ App::post('/v1/account/tokens/email') $response ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($token, Response::MODEL_TOKEN) - ; + ->dynamic($token, Response::MODEL_TOKEN); }); App::put('/v1/account/sessions/magic-url') @@ -2419,8 +2411,7 @@ App::post('/v1/account/tokens/phone') $response ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($token, Response::MODEL_TOKEN) - ; + ->dynamic($token, Response::MODEL_TOKEN); }); App::post('/v1/account/jwts') @@ -2447,7 +2438,8 @@ App::post('/v1/account/jwts') $sessions = $user->getAttribute('sessions', []); $current = new Document(); - foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ + foreach ($sessions as $session) { + /** @var Utopia\Database\Document $session */ if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) { // If current session delete the cookies too $current = $session; } @@ -2704,7 +2696,8 @@ App::patch('/v1/account/email') Query::notEqual('userId', $user->getId()), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); + /** Return a generic bad request to prevent exposing existing accounts */ } $user @@ -2740,7 +2733,8 @@ App::patch('/v1/account/email') } $dbForProject->purgeCachedDocument('users', $user->getId()); } catch (Duplicate) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); + /** Return a generic bad request to prevent exposing existing accounts */ } $queueForEvents->setParam('userId', $user->getId()); @@ -2899,8 +2893,7 @@ App::patch('/v1/account/status') $protocol = $request->getProtocol(); $response ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) - ; + ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -3072,8 +3065,7 @@ App::post('/v1/account/recovery') ->setPayload($response->output( $recovery->setAttribute('secret', $secret), Response::MODEL_TOKEN - )) - ; + )); // Hide secret for clients $recovery->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $secret : ''); @@ -3142,12 +3134,12 @@ App::put('/v1/account/recovery') $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]); $profile = $dbForProject->updateDocument('users', $profile->getId(), $profile - ->setAttribute('password', $newPassword) - ->setAttribute('passwordHistory', $history) - ->setAttribute('passwordUpdate', DateTime::now()) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) - ->setAttribute('emailVerification', true)); + ->setAttribute('password', $newPassword) + ->setAttribute('passwordHistory', $history) + ->setAttribute('passwordUpdate', DateTime::now()) + ->setAttribute('hash', Auth::DEFAULT_ALGO) + ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) + ->setAttribute('emailVerification', true)); $user->setAttributes($profile->getArrayCopy()); @@ -3162,8 +3154,7 @@ App::put('/v1/account/recovery') $queueForEvents ->setParam('userId', $profile->getId()) - ->setParam('tokenId', $recoveryDocument->getId()) - ; + ->setParam('tokenId', $recoveryDocument->getId()); $response->dynamic($recoveryDocument, Response::MODEL_TOKEN); }); @@ -3383,8 +3374,7 @@ App::put('/v1/account/verification') $queueForEvents ->setParam('userId', $userId) - ->setParam('tokenId', $verificationDocument->getId()) - ; + ->setParam('tokenId', $verificationDocument->getId()); $response->dynamic($verificationDocument, Response::MODEL_TOKEN); }); @@ -3490,8 +3480,7 @@ App::post('/v1/account/verification/phone') ->setPayload($response->output( $verification->setAttribute('secret', $secret), Response::MODEL_TOKEN - )) - ; + )); // Hide secret for clients $verification->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $secret : ''); @@ -3553,8 +3542,7 @@ App::put('/v1/account/verification/phone') $queueForEvents ->setParam('userId', $user->getId()) - ->setParam('tokenId', $verificationDocument->getId()) - ; + ->setParam('tokenId', $verificationDocument->getId()); $response->dynamic($verificationDocument, Response::MODEL_TOKEN); }); @@ -4201,8 +4189,8 @@ App::put('/v1/account/mfa/challenge') $dbForProject->updateDocument('sessions', $sessionId, $session); $queueForEvents - ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()); + ->setParam('userId', $user->getId()) + ->setParam('sessionId', $session->getId()); $response->dynamic($session, Response::MODEL_SESSION); }); @@ -4408,8 +4396,8 @@ App::get('/v1/account/identities') $queries[] = Query::equal('userInternalId', [$user->getInternalId()]); /** - * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries - */ + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ $cursor = \array_filter($queries, function ($query) { return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); }); From 0ea24380692316ef78f17e488c631410f6223ba2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 24 Jun 2024 14:12:09 +0100 Subject: [PATCH 16/38] fix: implementation --- app/controllers/api/account.php | 236 ++++++++++-------- app/controllers/api/projects.php | 12 +- .../Utopia/Response/Model/Project.php | 6 +- 3 files changed, 136 insertions(+), 118 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 7cd964f788..0bdbc305df 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -58,114 +58,8 @@ use Utopia\Validator\WhiteList; $oauthDefaultSuccess = '/auth/oauth2/success'; $oauthDefaultFailure = '/auth/oauth2/failure'; -$createSession = function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { - $roles = Authorization::getRoles(); - $isPrivilegedUser = Auth::isPrivilegedUser($roles); - $isAppUser = Auth::isAppUser($roles); - - /** @var Utopia\Database\Document $user */ - $userFromRequest = Authorization::skip(fn () => $dbForProject->getDocument('users', $userId)); - - if ($userFromRequest->isEmpty()) { - throw new Exception(Exception::USER_INVALID_TOKEN); - } - - $verifiedToken = Auth::tokenVerify($userFromRequest->getAttribute('tokens', []), null, $secret); - - if (!$verifiedToken) { - throw new Exception(Exception::USER_INVALID_TOKEN); - } - - $user->setAttributes($userFromRequest->getArrayCopy()); - - $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; - $detector = new Detector($request->getUserAgent('UNKNOWN')); - $record = $geodb->get($request->getIP()); - $sessionSecret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); - - $factor = (match ($verifiedToken->getAttribute('type')) { - Auth::TOKEN_TYPE_MAGIC_URL, - Auth::TOKEN_TYPE_OAUTH2, - Auth::TOKEN_TYPE_EMAIL => 'email', - Auth::TOKEN_TYPE_PHONE => 'phone', - Auth::TOKEN_TYPE_GENERIC => 'token', - default => throw new Exception(Exception::USER_INVALID_TOKEN) - }); - - $session = new Document(array_merge( - [ - '$id' => ID::unique(), - 'userId' => $user->getId(), - 'userInternalId' => $user->getInternalId(), - 'provider' => Auth::getSessionProviderByTokenType($verifiedToken->getAttribute('type')), - 'secret' => Auth::hash($sessionSecret), // One way hash encryption to protect DB leak - 'userAgent' => $request->getUserAgent('UNKNOWN'), - 'ip' => $request->getIP(), - 'factors' => [$factor], - 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', - 'expire' => DateTime::addSeconds(new \DateTime(), $duration) - ], - $detector->getOS(), - $detector->getClient(), - $detector->getDevice() - )); - - Authorization::setRole(Role::user($user->getId())->toString()); - - $session = $dbForProject->createDocument('sessions', $session - ->setAttribute('$permissions', [ - Permission::read(Role::user($user->getId())), - Permission::update(Role::user($user->getId())), - Permission::delete(Role::user($user->getId())), - ])); - - $dbForProject->purgeCachedDocument('users', $user->getId()); - Authorization::skip(fn () => $dbForProject->deleteDocument('tokens', $verifiedToken->getId())); - $dbForProject->purgeCachedDocument('users', $user->getId()); - - // Magic URL + Email OTP - if ($verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_MAGIC_URL || $verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_EMAIL) { - $user->setAttribute('emailVerification', true); - } - - if ($verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_PHONE) { - $user->setAttribute('phoneVerification', true); - } - - try { - $dbForProject->updateDocument('users', $user->getId(), $user); - } catch (\Throwable $th) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed saving user to DB'); - } - - $queueForEvents - ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()); - - if (!Config::getParam('domainVerification')) { - $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $sessionSecret)])); - } - - $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); - $protocol = $request->getProtocol(); - - $response - ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $sessionSecret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $sessionSecret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) - ->setStatusCode(Response::STATUS_CODE_CREATED); - - $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); - - $session - ->setAttribute('current', true) - ->setAttribute('countryName', $countryName) - ->setAttribute('expire', $expire) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : ''); - - $response->dynamic($session, Response::MODEL_SESSION); -}; - -$sendSessionEmail = function (Request $request, Locale $locale, Document $user, Document $project, Reader $geodb, Mail $queueForMails) { +function sendSessionAlert(Request $request, Locale $locale, Document $user, Document $project, Reader $geodb, Mail $queueForMails) +{ $subject = $locale->getText("emails.sessionAlert.subject"); $customTemplate = $project->getAttribute('templates', [])['email.sessionAlert-' . $locale->default] ?? []; @@ -252,6 +146,122 @@ $sendSessionEmail = function (Request $request, Locale $locale, Document $user, ->trigger(); }; + +$createSession = function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails) { + $roles = Authorization::getRoles(); + $isPrivilegedUser = Auth::isPrivilegedUser($roles); + $isAppUser = Auth::isAppUser($roles); + + /** @var Utopia\Database\Document $user */ + $userFromRequest = Authorization::skip(fn () => $dbForProject->getDocument('users', $userId)); + + if ($userFromRequest->isEmpty()) { + throw new Exception(Exception::USER_INVALID_TOKEN); + } + + $verifiedToken = Auth::tokenVerify($userFromRequest->getAttribute('tokens', []), null, $secret); + + if (!$verifiedToken) { + throw new Exception(Exception::USER_INVALID_TOKEN); + } + + $user->setAttributes($userFromRequest->getArrayCopy()); + + $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG; + $detector = new Detector($request->getUserAgent('UNKNOWN')); + $record = $geodb->get($request->getIP()); + $sessionSecret = Auth::tokenGenerator(Auth::TOKEN_LENGTH_SESSION); + + $factor = (match ($verifiedToken->getAttribute('type')) { + Auth::TOKEN_TYPE_MAGIC_URL, + Auth::TOKEN_TYPE_OAUTH2, + Auth::TOKEN_TYPE_EMAIL => 'email', + Auth::TOKEN_TYPE_PHONE => 'phone', + Auth::TOKEN_TYPE_GENERIC => 'token', + default => throw new Exception(Exception::USER_INVALID_TOKEN) + }); + + $session = new Document(array_merge( + [ + '$id' => ID::unique(), + 'userId' => $user->getId(), + 'userInternalId' => $user->getInternalId(), + 'provider' => Auth::getSessionProviderByTokenType($verifiedToken->getAttribute('type')), + 'secret' => Auth::hash($sessionSecret), // One way hash encryption to protect DB leak + 'userAgent' => $request->getUserAgent('UNKNOWN'), + 'ip' => $request->getIP(), + 'factors' => [$factor], + 'countryCode' => ($record) ? \strtolower($record['country']['iso_code']) : '--', + 'expire' => DateTime::addSeconds(new \DateTime(), $duration) + ], + $detector->getOS(), + $detector->getClient(), + $detector->getDevice() + )); + + Authorization::setRole(Role::user($user->getId())->toString()); + + $session = $dbForProject->createDocument('sessions', $session + ->setAttribute('$permissions', [ + Permission::read(Role::user($user->getId())), + Permission::update(Role::user($user->getId())), + Permission::delete(Role::user($user->getId())), + ])); + + $dbForProject->purgeCachedDocument('users', $user->getId()); + Authorization::skip(fn () => $dbForProject->deleteDocument('tokens', $verifiedToken->getId())); + $dbForProject->purgeCachedDocument('users', $user->getId()); + + // Magic URL + Email OTP + if ($verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_MAGIC_URL || $verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_EMAIL) { + $user->setAttribute('emailVerification', true); + } + + if ($verifiedToken->getAttribute('type') === Auth::TOKEN_TYPE_PHONE) { + $user->setAttribute('phoneVerification', true); + } + + try { + $dbForProject->updateDocument('users', $user->getId(), $user); + } catch (\Throwable $th) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed saving user to DB'); + } + + if ( + in_array($verifiedToken->getAttribute('type'), [Auth::TOKEN_TYPE_MAGIC_URL, Auth::TOKEN_TYPE_EMAIL]) && + $project->getAttribute('auths', [])['sessionAlerts'] ?? false && + $user->getAttribute('emailVerification') + ) { + sendSessionAlert($request, $locale, $user, $project, $geodb, $queueForMails); + } + + $queueForEvents + ->setParam('userId', $user->getId()) + ->setParam('sessionId', $session->getId()); + + if (!Config::getParam('domainVerification')) { + $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $sessionSecret)])); + } + + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); + $protocol = $request->getProtocol(); + + $response + ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $sessionSecret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) + ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $sessionSecret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ->setStatusCode(Response::STATUS_CODE_CREATED); + + $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); + + $session + ->setAttribute('current', true) + ->setAttribute('countryName', $countryName) + ->setAttribute('expire', $expire) + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : ''); + + $response->dynamic($session, Response::MODEL_SESSION); +}; + App::post('/v1/account') ->desc('Create account') ->groups(['api', 'account', 'auth']) @@ -805,8 +815,9 @@ App::post('/v1/account/sessions/email') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') + ->inject('queueForMails') ->inject('hooks') - ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Hooks $hooks) { + ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Mail $queueForMails, Hooks $hooks) { $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -895,6 +906,10 @@ App::post('/v1/account/sessions/email') ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()); + if ($project->getAttribute('auths', [])['sessionAlerts'] ?? false && $user->getAttribute('emailVerification')) { + sendSessionAlert($request, $locale, $user, $project, $geodb, $queueForMails); + } + $response->dynamic($session, Response::MODEL_SESSION); }); @@ -1060,6 +1075,7 @@ App::post('/v1/account/sessions/token') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') + ->inject('queueForMails') ->action($createSession); App::get('/v1/account/sessions/oauth2/:provider') @@ -2213,6 +2229,7 @@ App::put('/v1/account/sessions/magic-url') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') + ->inject('queueForMails') ->action($createSession); App::put('/v1/account/sessions/phone') @@ -2243,6 +2260,7 @@ App::put('/v1/account/sessions/phone') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') + ->inject('queueForMails') ->action($createSession); App::post('/v1/account/tokens/phone') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 653d62b157..06f1deb0b7 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -99,7 +99,7 @@ App::post('/v1/projects') 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false, - 'sessionEmails' => false, + 'sessionAlerts' => false, ]; foreach ($auth as $index => $method) { @@ -604,21 +604,21 @@ App::patch('/v1/projects/:projectId/oauth2') $response->dynamic($project, Response::MODEL_PROJECT); }); -App::patch('/v1/projects/:projectId/auth/session-emails') +App::patch('/v1/projects/:projectId/auth/session-alerts') ->desc('Update project sessions emails') ->groups(['api', 'projects']) ->label('scope', 'projects.write') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') - ->label('sdk.method', 'updateSessionEmails') + ->label('sdk.method', 'updateSessionAlerts') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('sessionEmails', false, new Boolean(true), 'Set to true to enable session emails.') + ->param('sessionAlerts', false, new Boolean(true), 'Set to true to enable session emails.') ->inject('response') ->inject('dbForConsole') - ->action(function (string $projectId, bool $sessionEmails, Response $response, Database $dbForConsole) { + ->action(function (string $projectId, bool $sessionAlerts, Response $response, Database $dbForConsole) { $project = $dbForConsole->getDocument('projects', $projectId); @@ -627,7 +627,7 @@ App::patch('/v1/projects/:projectId/auth/session-emails') } $auths = $project->getAttribute('auths', []); - $auths['sessionEmails'] = $sessionEmails; + $auths['sessionAlerts'] = $sessionAlerts; $dbForConsole->updateDocument('projects', $project->getId(), $project ->setAttribute('auths', $auths)); diff --git a/src/Appwrite/Utopia/Response/Model/Project.php b/src/Appwrite/Utopia/Response/Model/Project.php index aa56daf788..acefd918e1 100644 --- a/src/Appwrite/Utopia/Response/Model/Project.php +++ b/src/Appwrite/Utopia/Response/Model/Project.php @@ -138,9 +138,9 @@ class Project extends Model 'default' => false, 'example' => true, ]) - ->addRule('authSessionEmails', [ + ->addRule('authSessionAlerts', [ 'type' => self::TYPE_BOOLEAN, - 'description' => 'Whether or not to send session emails to users.', + 'description' => 'Whether or not to send session alert emails to users.', 'default' => false, 'example' => true, ]) @@ -324,7 +324,7 @@ class Project extends Model $document->setAttribute('authPasswordHistory', $authValues['passwordHistory'] ?? 0); $document->setAttribute('authPasswordDictionary', $authValues['passwordDictionary'] ?? false); $document->setAttribute('authPersonalDataCheck', $authValues['personalDataCheck'] ?? false); - $document->setAttribute('authSessionEmails', $authValues['sessionEmails'] ?? false); + $document->setAttribute('authSessionAlerts', $authValues['sessionAlerts'] ?? false); foreach ($auth as $index => $method) { $key = $method['key']; From edf7af34f187be124d885040697702b5a5f0c14a Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 24 Jun 2024 14:39:47 +0100 Subject: [PATCH 17/38] test: session alerts --- .../Account/AccountCustomClientTest.php | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 23771712e8..221b5a629e 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1191,6 +1191,49 @@ class AccountCustomClientTest extends Scope return $data; } + /** + * @depends testCreateAccountSession + */ + public function testSessionAlert($data): void + { + $email = $data['email'] ?? ''; + $password = $data['password'] ?? ''; + + // Enable sessionAlerts for the project + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/session-alerts', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'sessionAlerts' => true, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Create a new session + $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'email' => $email, + 'password' => $password, + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Check if an email alert was sent + $lastEmail = $this->getLastEmail(); + + $this->assertEquals($email, $lastEmail['to'][0]['address']); + $this->assertStringContainsString('New session alert', $lastEmail['subject']); + $this->assertStringContainsString($response['body']['$id'], $lastEmail['text']); // Session ID + $this->assertStringContainsString($response['body']['ip'], $lastEmail['text']); // IP Address + $this->assertStringContainsString($response['body']['osName'], $lastEmail['text']); // OS Name + $this->assertStringContainsString($response['body']['clientType'], $lastEmail['text']); // Client Type + } + /** * @depends testCreateAccountSession */ From 0d987045bf69fbeb58ff7f4c100566e2c5f476dc Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:46:51 +0100 Subject: [PATCH 18/38] chore: revert fmt --- app/controllers/api/account.php | 126 ++++++++++-------- .../Utopia/Response/Model/Project.php | 9 +- .../Account/AccountCustomClientTest.php | 2 - 3 files changed, 73 insertions(+), 64 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 0bdbc305df..2f23d08432 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -227,11 +227,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed saving user to DB'); } - if ( - in_array($verifiedToken->getAttribute('type'), [Auth::TOKEN_TYPE_MAGIC_URL, Auth::TOKEN_TYPE_EMAIL]) && - $project->getAttribute('auths', [])['sessionAlerts'] ?? false && - $user->getAttribute('emailVerification') - ) { + if ($project->getAttribute('auths', [])['sessionAlerts'] ?? false) { sendSessionAlert($request, $locale, $user, $project, $geodb, $queueForMails); } @@ -257,7 +253,8 @@ $createSession = function (string $userId, string $secret, Request $request, Res ->setAttribute('current', true) ->setAttribute('countryName', $countryName) ->setAttribute('expire', $expire) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : ''); + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $sessionSecret) : '') + ; $response->dynamic($session, Response::MODEL_SESSION); }; @@ -324,8 +321,7 @@ App::post('/v1/account') Query::equal('providerEmail', [$email]), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ } if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) { @@ -502,8 +498,7 @@ App::get('/v1/account/sessions') $sessions = $user->getAttribute('sessions', []); $current = Auth::sessionVerify($sessions, Auth::$secret); - foreach ($sessions as $key => $session) { - /** @var Document $session */ + foreach ($sessions as $key => $session) {/** @var Document $session */ $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session->setAttribute('countryName', $countryName); @@ -545,8 +540,7 @@ App::delete('/v1/account/sessions') $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); - foreach ($sessions as $session) { - /** @var Document $session */ + foreach ($sessions as $session) {/** @var Document $session */ $dbForProject->deleteDocument('sessions', $session->getId()); if (!Config::getParam('domainVerification')) { @@ -614,15 +608,15 @@ App::get('/v1/account/sessions/:sessionId') ? Auth::sessionVerify($user->getAttribute('sessions'), Auth::$secret) : $sessionId; - foreach ($sessions as $session) { - /** @var Document $session */ + foreach ($sessions as $session) {/** @var Document $session */ if ($sessionId === $session->getId()) { $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session ->setAttribute('current', ($session->getAttribute('secret') == Auth::hash(Auth::$secret))) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : ''); + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $session->getAttribute('secret', '') : '') + ; return $response->dynamic($session, Response::MODEL_SESSION); } @@ -781,7 +775,8 @@ App::patch('/v1/account/sessions/:sessionId') $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) - ->setPayload($response->output($session, Response::MODEL_SESSION)); + ->setPayload($response->output($session, Response::MODEL_SESSION)) + ; return $response->dynamic($session, Response::MODEL_SESSION); }); @@ -885,7 +880,8 @@ App::post('/v1/account/sessions/email') if (!Config::getParam('domainVerification')) { $response - ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); + ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) + ; } $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); @@ -893,20 +889,23 @@ App::post('/v1/account/sessions/email') $response ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) - ->setStatusCode(Response::STATUS_CODE_CREATED); + ->setStatusCode(Response::STATUS_CODE_CREATED) + ; $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : ''); + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '') + ; $queueForEvents ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()); + ->setParam('sessionId', $session->getId()) + ; - if ($project->getAttribute('auths', [])['sessionAlerts'] ?? false && $user->getAttribute('emailVerification')) { + if ($project->getAttribute('auths', [])['sessionAlerts'] ?? false) { sendSessionAlert($request, $locale, $user, $project, $geodb, $queueForMails); } @@ -1015,7 +1014,7 @@ App::post('/v1/account/sessions/anonymous') Authorization::setRole(Role::user($user->getId())->toString()); - $session = $dbForProject->createDocument('sessions', $session->setAttribute('$permissions', [ + $session = $dbForProject->createDocument('sessions', $session-> setAttribute('$permissions', [ Permission::read(Role::user($user->getId())), Permission::update(Role::user($user->getId())), Permission::delete(Role::user($user->getId())), @@ -1025,7 +1024,8 @@ App::post('/v1/account/sessions/anonymous') $queueForEvents ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()); + ->setParam('sessionId', $session->getId()) + ; if (!Config::getParam('domainVerification')) { $response->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])); @@ -1036,14 +1036,16 @@ App::post('/v1/account/sessions/anonymous') $response ->addCookie(Auth::$cookieName . '_legacy', Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, Auth::encodeSession($user->getId(), $secret), (new \DateTime($expire))->getTimestamp(), '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) - ->setStatusCode(Response::STATUS_CODE_CREATED); + ->setStatusCode(Response::STATUS_CODE_CREATED) + ; $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session ->setAttribute('current', true) ->setAttribute('countryName', $countryName) - ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : ''); + ->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? Auth::encodeSession($user->getId(), $secret) : '') + ; $response->dynamic($session, Response::MODEL_SESSION); }); @@ -1429,8 +1431,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Query::equal('providerEmail', [$email]), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ } try { @@ -1501,8 +1502,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Query::notEqual('userInternalId', $user->getInternalId()), ]); if (!empty($identitiesWithMatchingEmail)) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ } $dbForProject->createDocument('identities', new Document([ @@ -1575,7 +1575,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $queueForEvents ->setEvent('users.[userId].tokens.[tokenId].create') ->setParam('userId', $user->getId()) - ->setParam('tokenId', $token->getId()); + ->setParam('tokenId', $token->getId()) + ; $query['secret'] = $secret; $query['userId'] = $user->getId(); @@ -1618,7 +1619,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $queueForEvents ->setParam('userId', $user->getId()) ->setParam('sessionId', $session->getId()) - ->setPayload($response->output($session, Response::MODEL_SESSION)); + ->setPayload($response->output($session, Response::MODEL_SESSION)) + ; // TODO: Remove this deprecated workaround - support only token if ($state['success']['path'] == $oauthDefaultSuccess) { @@ -1657,7 +1659,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $response ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') ->addHeader('Pragma', 'no-cache') - ->redirect($state['success']); + ->redirect($state['success']) + ; }); App::get('/v1/account/tokens/oauth2/:provider') @@ -1969,7 +1972,8 @@ App::post('/v1/account/tokens/magic-url') $response ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($token, Response::MODEL_TOKEN); + ->dynamic($token, Response::MODEL_TOKEN) + ; }); App::post('/v1/account/tokens/email') @@ -2032,8 +2036,7 @@ App::post('/v1/account/tokens/email') Query::equal('providerEmail', [$email]), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ } $userId = $userId === 'unique()' ? ID::unique() : $userId; @@ -2198,7 +2201,8 @@ App::post('/v1/account/tokens/email') $response ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($token, Response::MODEL_TOKEN); + ->dynamic($token, Response::MODEL_TOKEN) + ; }); App::put('/v1/account/sessions/magic-url') @@ -2429,7 +2433,8 @@ App::post('/v1/account/tokens/phone') $response ->setStatusCode(Response::STATUS_CODE_CREATED) - ->dynamic($token, Response::MODEL_TOKEN); + ->dynamic($token, Response::MODEL_TOKEN) + ; }); App::post('/v1/account/jwts') @@ -2456,8 +2461,7 @@ App::post('/v1/account/jwts') $sessions = $user->getAttribute('sessions', []); $current = new Document(); - foreach ($sessions as $session) { - /** @var Utopia\Database\Document $session */ + foreach ($sessions as $session) { /** @var Utopia\Database\Document $session */ if ($session->getAttribute('secret') == Auth::hash(Auth::$secret)) { // If current session delete the cookies too $current = $session; } @@ -2714,8 +2718,7 @@ App::patch('/v1/account/email') Query::notEqual('userId', $user->getId()), ]); if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ } $user @@ -2751,8 +2754,7 @@ App::patch('/v1/account/email') } $dbForProject->purgeCachedDocument('users', $user->getId()); } catch (Duplicate) { - throw new Exception(Exception::GENERAL_BAD_REQUEST); - /** Return a generic bad request to prevent exposing existing accounts */ + throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */ } $queueForEvents->setParam('userId', $user->getId()); @@ -2911,7 +2913,8 @@ App::patch('/v1/account/status') $protocol = $request->getProtocol(); $response ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) - ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')); + ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) + ; $response->dynamic($user, Response::MODEL_ACCOUNT); }); @@ -3083,7 +3086,8 @@ App::post('/v1/account/recovery') ->setPayload($response->output( $recovery->setAttribute('secret', $secret), Response::MODEL_TOKEN - )); + )) + ; // Hide secret for clients $recovery->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $secret : ''); @@ -3152,12 +3156,12 @@ App::put('/v1/account/recovery') $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]); $profile = $dbForProject->updateDocument('users', $profile->getId(), $profile - ->setAttribute('password', $newPassword) - ->setAttribute('passwordHistory', $history) - ->setAttribute('passwordUpdate', DateTime::now()) - ->setAttribute('hash', Auth::DEFAULT_ALGO) - ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) - ->setAttribute('emailVerification', true)); + ->setAttribute('password', $newPassword) + ->setAttribute('passwordHistory', $history) + ->setAttribute('passwordUpdate', DateTime::now()) + ->setAttribute('hash', Auth::DEFAULT_ALGO) + ->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS) + ->setAttribute('emailVerification', true)); $user->setAttributes($profile->getArrayCopy()); @@ -3172,7 +3176,8 @@ App::put('/v1/account/recovery') $queueForEvents ->setParam('userId', $profile->getId()) - ->setParam('tokenId', $recoveryDocument->getId()); + ->setParam('tokenId', $recoveryDocument->getId()) + ; $response->dynamic($recoveryDocument, Response::MODEL_TOKEN); }); @@ -3392,7 +3397,8 @@ App::put('/v1/account/verification') $queueForEvents ->setParam('userId', $userId) - ->setParam('tokenId', $verificationDocument->getId()); + ->setParam('tokenId', $verificationDocument->getId()) + ; $response->dynamic($verificationDocument, Response::MODEL_TOKEN); }); @@ -3498,7 +3504,8 @@ App::post('/v1/account/verification/phone') ->setPayload($response->output( $verification->setAttribute('secret', $secret), Response::MODEL_TOKEN - )); + )) + ; // Hide secret for clients $verification->setAttribute('secret', ($isPrivilegedUser || $isAppUser) ? $secret : ''); @@ -3560,7 +3567,8 @@ App::put('/v1/account/verification/phone') $queueForEvents ->setParam('userId', $user->getId()) - ->setParam('tokenId', $verificationDocument->getId()); + ->setParam('tokenId', $verificationDocument->getId()) + ; $response->dynamic($verificationDocument, Response::MODEL_TOKEN); }); @@ -4207,8 +4215,8 @@ App::put('/v1/account/mfa/challenge') $dbForProject->updateDocument('sessions', $sessionId, $session); $queueForEvents - ->setParam('userId', $user->getId()) - ->setParam('sessionId', $session->getId()); + ->setParam('userId', $user->getId()) + ->setParam('sessionId', $session->getId()); $response->dynamic($session, Response::MODEL_SESSION); }); @@ -4414,8 +4422,8 @@ App::get('/v1/account/identities') $queries[] = Query::equal('userInternalId', [$user->getInternalId()]); /** - * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries - */ + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ $cursor = \array_filter($queries, function ($query) { return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); }); diff --git a/src/Appwrite/Utopia/Response/Model/Project.php b/src/Appwrite/Utopia/Response/Model/Project.php index acefd918e1..da97ba2c19 100644 --- a/src/Appwrite/Utopia/Response/Model/Project.php +++ b/src/Appwrite/Utopia/Response/Model/Project.php @@ -226,7 +226,8 @@ class Project extends Model 'description' => 'SMTP server secure protocol', 'default' => '', 'example' => 'tls', - ]); + ]) + ; $services = Config::getParam('services', []); $auth = Config::getParam('auth', []); @@ -241,7 +242,8 @@ class Project extends Model 'description' => $name . ' auth method status', 'example' => true, 'default' => true, - ]); + ]) + ; } foreach ($services as $service) { @@ -258,7 +260,8 @@ class Project extends Model 'description' => $name . ' service status', 'example' => true, 'default' => true, - ]); + ]) + ; } } diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 221b5a629e..e929d6fb20 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1199,7 +1199,6 @@ class AccountCustomClientTest extends Scope $email = $data['email'] ?? ''; $password = $data['password'] ?? ''; - // Enable sessionAlerts for the project $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/session-alerts', array_merge([ 'origin' => 'http://localhost', 'content-type' => 'application/json', @@ -1211,7 +1210,6 @@ class AccountCustomClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - // Create a new session $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([ 'origin' => 'http://localhost', 'content-type' => 'application/json', From 0edbec4f44edabbf781ed3e814a24fffc93fdc36 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 25 Jun 2024 15:03:07 +0530 Subject: [PATCH 19/38] OPR v4 support --- app/config/collections.php | 2 +- app/config/runtimes.php | 2 +- app/controllers/api/functions.php | 3 ++- app/controllers/general.php | 1 + docker-compose.yml | 2 +- src/Appwrite/Platform/Workers/Functions.php | 3 ++- src/Appwrite/Utopia/Response/Model/Func.php | 2 +- src/Executor/Executor.php | 2 ++ 8 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 3e0dd9b7a7..46a2d2554b 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3029,7 +3029,7 @@ $projectCollections = array_merge([ 'size' => 8, 'signed' => true, 'required' => false, - 'default' => 'v3', + 'default' => 'v4', 'array' => false, 'filters' => [], ], diff --git a/app/config/runtimes.php b/app/config/runtimes.php index a55e0b3fb4..980613ebec 100644 --- a/app/config/runtimes.php +++ b/app/config/runtimes.php @@ -6,4 +6,4 @@ use Appwrite\Runtimes\Runtimes; -return (new Runtimes('v3'))->getAll(); +return (new Runtimes('v4'))->getAll(); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 068fc1c836..69086b4fca 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -221,7 +221,7 @@ App::post('/v1/functions') 'commands' => $commands, 'scopes' => $scopes, 'search' => implode(' ', [$functionId, $name, $runtime]), - 'version' => 'v3', + 'version' => 'v4', 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), 'providerRepositoryId' => $providerRepositoryId, @@ -1813,6 +1813,7 @@ App::post('/v1/functions/:functionId/executions') method: $method, headers: $headers, runtimeEntrypoint: $command, + logging: $function->getAttribute('logging', true), requestTimeout: 30 ); diff --git a/app/controllers/general.php b/app/controllers/general.php index 71ecc80d54..d4217477fb 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -270,6 +270,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo method: $method, headers: $headers, runtimeEntrypoint: $command, + logging: $function->getAttribute('logging', true), requestTimeout: 30 ); diff --git a/docker-compose.yml b/docker-compose.yml index 250eb8b7aa..7535555817 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -803,7 +803,7 @@ services: - OPR_EXECUTOR_ENV=$_APP_ENV - OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET - - OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v3 + - OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v4 - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG - OPR_EXECUTOR_STORAGE_DEVICE=$_APP_STORAGE_DEVICE - OPR_EXECUTOR_STORAGE_S3_ACCESS_KEY=$_APP_STORAGE_S3_ACCESS_KEY diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index cbba9657ad..acef40382a 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -412,7 +412,8 @@ class Functions extends Action path: $path, method: $method, headers: $headers, - runtimeEntrypoint: $command + runtimeEntrypoint: $command, + logging: $function->getAttribute('logging', true), ); $status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed'; diff --git a/src/Appwrite/Utopia/Response/Model/Func.php b/src/Appwrite/Utopia/Response/Model/Func.php index 66c356e2b1..46c45d06c5 100644 --- a/src/Appwrite/Utopia/Response/Model/Func.php +++ b/src/Appwrite/Utopia/Response/Model/Func.php @@ -119,7 +119,7 @@ class Func extends Model ->addRule('version', [ 'type' => self::TYPE_STRING, 'description' => 'Version of Open Runtimes used for the function.', - 'default' => 'v3', + 'default' => 'v4', 'example' => 'v2', ]) ->addRule('installationId', [ diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 76c66de231..d80b0037af 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -177,6 +177,7 @@ class Executor string $method, array $headers, string $runtimeEntrypoint = null, + bool $logging, int $requestTimeout = null ) { if (empty($headers['host'])) { @@ -201,6 +202,7 @@ class Executor 'memory' => $this->memory, 'version' => $version, 'runtimeEntrypoint' => $runtimeEntrypoint, + 'logging' => $logging, ]; // Safety timeout. Executor has timeout, and open runtime has soft timeout. From 6353de6f10eef7d312765f331d6f7bce373dbd1b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 09:42:01 +0100 Subject: [PATCH 20/38] chore: fix template --- .../locale/templates/email-session-alert.tpl | 6 ++--- app/config/locale/translations/en.json | 8 +++---- app/controllers/api/account.php | 7 +++--- .../Account/AccountCustomClientTest.php | 24 +++++++++++++++---- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/app/config/locale/templates/email-session-alert.tpl b/app/config/locale/templates/email-session-alert.tpl index 5c182a7df5..9855175b6f 100644 --- a/app/config/locale/templates/email-session-alert.tpl +++ b/app/config/locale/templates/email-session-alert.tpl @@ -3,9 +3,9 @@

{{body}}

    -
  1. {{device}}
  2. -
  3. {{ipAddress}}
  4. -
  5. {{country}}
  6. +
  7. {{listDevice}}
  8. +
  9. {{listIpAddress}}
  10. +
  11. {{listCountry}}

{{footer}}

diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index 7cc7fc8485..bd922ef754 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -21,9 +21,9 @@ "emails.sessionAlert.subject": "New session alert for {{project}}", "emails.sessionAlert.hello":"Hello {{user}}", "emails.sessionAlert.body": "We're writing to inform you that a new session has been initiated on your {{b}}{{project}}{{/b}} account, on {{b}}{{dateTime}}{{/b}}. \nHere are the details of the new session: ", - "emails.sessionAlert.device": "Device: {{b}}{{agentDevice}}{{/b}}", - "emails.sessionAlert.ipAddress": "IP Address: {{b}}{{ipAddress}}{{/b}}", - "emails.sessionAlert.country": "Country: {{b}}{{country}}{{/b}}", + "emails.sessionAlert.listDevice": "Device: {{b}}{{agentDevice}}{{/b}}", + "emails.sessionAlert.listIpAddress": "IP Address: {{b}}{{ipAddress}}{{/b}}", + "emails.sessionAlert.listCountry": "Country: {{b}}{{country}}{{/b}}", "emails.sessionAlert.footer": "If you didn't request the sign in, you can safely ignore this email. If you suspect unauthorized activity, please secure your account immediately.", "emails.sessionAlert.thanks": "Thanks,", "emails.sessionAlert.signature": "{{project}} team", @@ -43,7 +43,7 @@ "emails.recovery.subject": "Password Reset", "emails.recovery.hello": "Hello {{user}}", "emails.recovery.body": "Follow this link to reset your {{b}}{{project}}{{/b}} password.", - "emails.recovery.footer": "If you didn’t ask to reset your password, you can ignore this message.", + "emails.recovery.footer": "If you didn't ask to reset your password, you can ignore this message.", "emails.recovery.thanks": "Thanks", "emails.recovery.signature": "{{project}} team", "emails.invitation.subject": "Invitation to %s Team at %s", diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 2f23d08432..3c29364f4b 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -73,9 +73,9 @@ function sendSessionAlert(Request $request, Locale $locale, Document $user, Docu $message ->setParam('{{hello}}', $locale->getText("emails.sessionAlert.hello")) ->setParam('{{body}}', $locale->getText("emails.sessionAlert.body")) - ->setParam('{{device}}', $locale->getText("emails.sessionAlert.device")) - ->setParam('{{ipAddress}}', $locale->getText("emails.sessionAlert.ipAddress")) - ->setParam('{{country}}', $locale->getText("emails.sessionAlert.country")) + ->setParam('{{listDevice}}', $locale->getText("emails.sessionAlert.listDevice")) + ->setParam('{{listIpAddress}}', $locale->getText("emails.sessionAlert.listIpAddress")) + ->setParam('{{listCountry}}', $locale->getText("emails.sessionAlert.listCountry")) ->setParam('{{footer}}', $locale->getText("emails.sessionAlert.footer")) ->setParam('{{signature}}', $locale->getText("emails.sessionAlert.signature")); @@ -129,6 +129,7 @@ function sendSessionAlert(Request $request, Locale $locale, Document $user, Docu $emailVariables = [ 'direction' => $locale->getText('settings.direction'), + 'dateTime' => DateTime::format(new \DateTime(), 'Y-m-d H:i:s'), 'user' => $user->getAttribute('name'), 'project' => $project->getAttribute('name'), 'agentDevice' => $agentDevice['deviceBrand'] ?? $agentDevice['deviceBrand'] ?? 'UNKNOWN', diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index e929d6fb20..95d187cd88 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1196,9 +1196,11 @@ class AccountCustomClientTest extends Scope */ public function testSessionAlert($data): void { - $email = $data['email'] ?? ''; - $password = $data['password'] ?? ''; + $email = uniqid() . 'session-alert@appwrite.io'; + $password = 'password123'; + $name = 'Session Alert Tester'; + // Enable session alerts $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $this->getProject()['$id'] . '/auth/session-alerts', array_merge([ 'origin' => 'http://localhost', 'content-type' => 'application/json', @@ -1210,6 +1212,21 @@ class AccountCustomClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); + // Create a new account + $response = $this->client->call(Client::METHOD_POST, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'userId' => ID::unique(), + 'email' => $email, + 'password' => $password, + 'name' => $name, + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Create a session for the new account $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([ 'origin' => 'http://localhost', 'content-type' => 'application/json', @@ -1221,12 +1238,11 @@ class AccountCustomClientTest extends Scope $this->assertEquals(201, $response['headers']['status-code']); - // Check if an email alert was sent + // Check the alert email $lastEmail = $this->getLastEmail(); $this->assertEquals($email, $lastEmail['to'][0]['address']); $this->assertStringContainsString('New session alert', $lastEmail['subject']); - $this->assertStringContainsString($response['body']['$id'], $lastEmail['text']); // Session ID $this->assertStringContainsString($response['body']['ip'], $lastEmail['text']); // IP Address $this->assertStringContainsString($response['body']['osName'], $lastEmail['text']); // OS Name $this->assertStringContainsString($response['body']['clientType'], $lastEmail['text']); // Client Type From 878f6c86df4b0f6796824ca5a76f8b2066d95738 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:01:43 +0100 Subject: [PATCH 21/38] chore: matej review p1 --- .../Platform/Tasks/ScheduleExecutions.php | 2 ++ .../Functions/FunctionsCustomClientTest.php | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index b0f03a35ea..ea1339511d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -42,6 +42,8 @@ class ScheduleExecutions extends ScheduleBase $queueForFunctions ->setType('schedule') + // Set functionId rather than function as we don't have access to $dbForProject + // TODO: Refactor to use function instead of functionId ->setFunctionId($schedule['resource']['functionId']) ->setExecution($schedule['resource']) ->setProject($schedule['project']) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index a187ee43ae..0c06b42d2a 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -59,7 +59,6 @@ class FunctionsCustomClientTest extends Scope 'execute' => [Role::user($this->getUser()['$id'])->toString()], 'runtime' => 'php-8.0', 'entrypoint' => 'index.php', - 'logging' => true, 'events' => [ 'users.*.create', 'users.*.delete', @@ -256,7 +255,7 @@ class FunctionsCustomClientTest extends Scope \sleep(1); } - $this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body'])); + $this->assertEquals('ready', $deployment['body']['status']); $function = $this->client->call(Client::METHOD_PATCH, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ 'content-type' => 'application/json', @@ -278,7 +277,7 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(202, $execution['headers']['status-code']); - $this->assertEquals('scheduled', $execution['body']['status'], \json_encode($execution['body'])); + $this->assertEquals('scheduled', $execution['body']['status']); $executionId = $execution['body']['$id']; @@ -291,7 +290,21 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(200, $execution['headers']['status-code']); - $this->assertEquals('completed', $execution['body']['status'], \json_encode($execution['body'])); + $this->assertEquals('completed', $execution['body']['status']); + + /* Test for FAILURE */ + + // Schedule synchronous execution + + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'async' => false, + 'scheduledAt' => $futureTime, + ]); + + $this->assertEquals(400, $execution['headers']['status-code']); // Cleanup : Delete function $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $function['body']['$id'], [ From 303ce498becd949e43f916f8b861fc523520ba28 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:30:23 +0100 Subject: [PATCH 22/38] feat: allow custom path method body --- app/config/collections.php | 11 +++++++++++ app/controllers/api/functions.php | 8 ++++++++ src/Appwrite/Event/Func.php | 4 ++-- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 6 +++++- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 2 ++ .../Services/Functions/FunctionsCustomClientTest.php | 11 +++++++++++ 6 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 66bb2606cc..2b7777e85b 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4550,6 +4550,17 @@ $consoleCollections = array_merge([ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('metadata'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65535, + 'signed' => true, + 'required' => false, + 'default' => new \stdClass(), + 'array' => false, + 'filters' => ['json'], + ], [ '$id' => ID::custom('active'), 'type' => Database::VAR_BOOLEAN, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7137fbf9d3..2fcdfa92de 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1764,6 +1764,13 @@ App::post('/v1/functions/:functionId/executions') ->setParam('executionId', $execution->getId()) ->trigger(); } else { + $metadata = [ + 'headers' => $headers, + 'path' => $path, + 'method' => $method, + 'body' => $body, + ]; + $dbForConsole->createDocument('schedules', new Document([ 'region' => System::getEnv('_APP_REGION', 'default'), 'resourceType' => 'execution', @@ -1772,6 +1779,7 @@ App::post('/v1/functions/:functionId/executions') 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), 'schedule' => $scheduledAt, + 'metadata' => $metadata, 'active' => true, ])); } diff --git a/src/Appwrite/Event/Func.php b/src/Appwrite/Event/Func.php index 86080593af..451df2b6c1 100644 --- a/src/Appwrite/Event/Func.php +++ b/src/Appwrite/Event/Func.php @@ -11,8 +11,8 @@ class Func extends Event protected string $jwt = ''; protected string $type = ''; protected string $body = ''; - protected ?string $path = null; - protected ?string $method = null; + protected string $path = ''; + protected string $method = ''; protected array $headers = []; protected ?string $functionId = null; protected ?Document $function = null; diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index ea1339511d..55cefb2a44 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -42,10 +42,14 @@ class ScheduleExecutions extends ScheduleBase $queueForFunctions ->setType('schedule') - // Set functionId rather than function as we don't have access to $dbForProject + // Set functionId instead of function as we don't have $dbForProject // TODO: Refactor to use function instead of functionId ->setFunctionId($schedule['resource']['functionId']) ->setExecution($schedule['resource']) + ->setMethod($schedule['metadata']['method'] ?? 'POST') + ->setPath($schedule['metadata']['path'] ?? '/') + ->setHeaders($schedule['metadata']['headers'] ?? []) + ->setBody($schedule['metadata']['body'] ?? '') ->setProject($schedule['project']) ->trigger(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index e4832d7435..e2c278714f 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -84,6 +84,8 @@ class ScheduleFunctions extends ScheduleBase $queueForFunctions ->setType('schedule') ->setFunction($schedule['resource']) + ->setMethod('POST') + ->setPath('/') ->setProject($schedule['project']) ->trigger(); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0c06b42d2a..38ed9c9564 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -268,16 +268,27 @@ class FunctionsCustomClientTest extends Scope // Schedule execution for the future \date_default_timezone_set('UTC'); $futureTime = (new \DateTime())->add(new \DateInterval('PT10S'))->format('Y-m-d H:i:s'); + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $function['body']['$id'] . '/executions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'async' => true, 'scheduledAt' => $futureTime, + 'path' => '/custom', + 'method' => 'GET', + 'body' => 'hello', + 'headers' => [ + 'content-type' => 'application/plain', + ], ]); $this->assertEquals(202, $execution['headers']['status-code']); + $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('scheduled', $execution['body']['status']); + $this->assertEquals('/custom', $execution['requestPath']); + $this->assertEquals('GET', $execution['requestMethod']); + $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); $executionId = $execution['body']['$id']; From 9eb8f02b5367b08c11566928810ddbd4d95543ea Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:42:35 +0100 Subject: [PATCH 23/38] fix: test --- .../e2e/Services/Functions/FunctionsCustomClientTest.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 38ed9c9564..50767a4c1e 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -284,11 +284,7 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(202, $execution['headers']['status-code']); - $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('scheduled', $execution['body']['status']); - $this->assertEquals('/custom', $execution['requestPath']); - $this->assertEquals('GET', $execution['requestMethod']); - $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); $executionId = $execution['body']['$id']; @@ -301,7 +297,11 @@ class FunctionsCustomClientTest extends Scope ]); $this->assertEquals(200, $execution['headers']['status-code']); + $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('completed', $execution['body']['status']); + $this->assertEquals('/custom', $execution['requestPath']); + $this->assertEquals('GET', $execution['requestMethod']); + $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); /* Test for FAILURE */ From e8840090756f120eb37335d764c44d84d5038b02 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:14:07 +0100 Subject: [PATCH 24/38] fix: test --- app/controllers/api/functions.php | 1 + tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 2fcdfa92de..b4ef380d8f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1769,6 +1769,7 @@ App::post('/v1/functions/:functionId/executions') 'path' => $path, 'method' => $method, 'body' => $body, + 'jwt' => $jwt, ]; $dbForConsole->createDocument('schedules', new Document([ diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 50767a4c1e..a8e5a1ed3c 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -299,9 +299,9 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals(200, $execution['headers']['status-code']); $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertEquals('completed', $execution['body']['status']); - $this->assertEquals('/custom', $execution['requestPath']); - $this->assertEquals('GET', $execution['requestMethod']); - $this->assertEquals(['content-type' => 'application/plain'], $execution['requestHeaders']); + $this->assertEquals('/custom', $execution['body']['requestPath']); + $this->assertEquals('GET', $execution['body']['requestMethod']); + $this->assertEquals(['content-type' => 'application/plain'], $execution['body']['requestHeaders']); /* Test for FAILURE */ From fd12449cc3a3abb4e80ebeab7c8be150a14e14e7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:01:53 +0100 Subject: [PATCH 25/38] test: fix --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index a8e5a1ed3c..f824f47131 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -301,7 +301,6 @@ class FunctionsCustomClientTest extends Scope $this->assertEquals('completed', $execution['body']['status']); $this->assertEquals('/custom', $execution['body']['requestPath']); $this->assertEquals('GET', $execution['body']['requestMethod']); - $this->assertEquals(['content-type' => 'application/plain'], $execution['body']['requestHeaders']); /* Test for FAILURE */ From d0125398f1d6974a6dcdfa17a5662228d102a30b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:46:12 +0100 Subject: [PATCH 26/38] fix: test --- app/config/locale/translations/en.json | 2 +- app/controllers/api/account.php | 19 +++++++------------ app/controllers/api/projects.php | 2 +- .../Account/AccountCustomClientTest.php | 5 +++-- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/app/config/locale/translations/en.json b/app/config/locale/translations/en.json index 7d6bad9989..953888013a 100644 --- a/app/config/locale/translations/en.json +++ b/app/config/locale/translations/en.json @@ -21,7 +21,7 @@ "emails.sessionAlert.subject": "New session alert for {{project}}", "emails.sessionAlert.hello":"Hello {{user}}", "emails.sessionAlert.body": "We're writing to inform you that a new session has been initiated on your {{b}}{{project}}{{/b}} account, on {{b}}{{dateTime}}{{/b}}. \nHere are the details of the new session: ", - "emails.sessionAlert.listDevice": "Device: {{b}}{{agentDevice}}{{/b}}", + "emails.sessionAlert.listDevice": "Device: {{b}}{{device}}{{/b}}", "emails.sessionAlert.listIpAddress": "IP Address: {{b}}{{ipAddress}}{{/b}}", "emails.sessionAlert.listCountry": "Country: {{b}}{{country}}{{/b}}", "emails.sessionAlert.footer": "If you didn't request the sign in, you can safely ignore this email. If you suspect unauthorized activity, please secure your account immediately.", diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 52db2cbd53..8f46ca3c62 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -58,17 +58,11 @@ use Utopia\Validator\WhiteList; $oauthDefaultSuccess = '/auth/oauth2/success'; $oauthDefaultFailure = '/auth/oauth2/failure'; -function sendSessionAlert(Request $request, Locale $locale, Document $user, Document $project, Reader $geodb, Mail $queueForMails) +function sendSessionAlert(Locale $locale, Document $user, Document $project, Document $session, Mail $queueForMails) { $subject = $locale->getText("emails.sessionAlert.subject"); $customTemplate = $project->getAttribute('templates', [])['email.sessionAlert-' . $locale->default] ?? []; - $detector = new Detector($request->getUserAgent('UNKNOWN')); - $agentDevice = $detector->getDevice(); - - $record = $geodb->get($request->getIP()); - $countryCode = $record['country']['iso_code'] ?? ''; - $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-session-alert.tpl'); $message ->setParam('{{hello}}', $locale->getText("emails.sessionAlert.hello")) @@ -77,6 +71,7 @@ function sendSessionAlert(Request $request, Locale $locale, Document $user, Docu ->setParam('{{listIpAddress}}', $locale->getText("emails.sessionAlert.listIpAddress")) ->setParam('{{listCountry}}', $locale->getText("emails.sessionAlert.listCountry")) ->setParam('{{footer}}', $locale->getText("emails.sessionAlert.footer")) + ->setParam('{{thanks}}', $locale->getText("emails.sessionAlert.thanks")) ->setParam('{{signature}}', $locale->getText("emails.sessionAlert.signature")); $body = $message->render(); @@ -132,9 +127,9 @@ function sendSessionAlert(Request $request, Locale $locale, Document $user, Docu 'dateTime' => DateTime::format(new \DateTime(), 'Y-m-d H:i:s'), 'user' => $user->getAttribute('name'), 'project' => $project->getAttribute('name'), - 'agentDevice' => $agentDevice['deviceBrand'] ?? $agentDevice['deviceBrand'] ?? 'UNKNOWN', - 'ipAddress' => $request->getIP(), - 'country' => $locale->getText('countries.' . strtolower($countryCode), $locale->getText('locale.country.unknown')), + 'device' => $session->getAttribute('clientName'), + 'ipAddress' => $session->getAttribute('ip'), + 'country' => $locale->getText('countries.' . $session->getAttribute('countryCode'), $locale->getText('locale.country.unknown')), ]; $email = $user->getAttribute('email'); @@ -229,7 +224,7 @@ $createSession = function (string $userId, string $secret, Request $request, Res } if ($project->getAttribute('auths', [])['sessionAlerts'] ?? false) { - sendSessionAlert($request, $locale, $user, $project, $geodb, $queueForMails); + sendSessionAlert($locale, $user, $project, $session, $queueForMails); } $queueForEvents @@ -909,7 +904,7 @@ App::post('/v1/account/sessions/email') ; if ($project->getAttribute('auths', [])['sessionAlerts'] ?? false) { - sendSessionAlert($request, $locale, $user, $project, $geodb, $queueForMails); + sendSessionAlert($locale, $user, $project, $session, $queueForMails); } $response->dynamic($session, Response::MODEL_SESSION); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index dd97657448..1533456f03 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -101,7 +101,7 @@ App::post('/v1/projects') $auths = [ 'limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, - 'passwordHistory' => 0, + 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false, diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 0405714f0e..7a6b469d9b 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1230,6 +1230,7 @@ class AccountCustomClientTest extends Scope 'origin' => 'http://localhost', 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], + 'user-agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', ]), [ 'email' => $email, 'password' => $password, @@ -1243,8 +1244,8 @@ class AccountCustomClientTest extends Scope $this->assertEquals($email, $lastEmail['to'][0]['address']); $this->assertStringContainsString('New session alert', $lastEmail['subject']); $this->assertStringContainsString($response['body']['ip'], $lastEmail['text']); // IP Address - $this->assertStringContainsString($response['body']['osName'], $lastEmail['text']); // OS Name - $this->assertStringContainsString($response['body']['clientType'], $lastEmail['text']); // Client Type + $this->assertStringContainsString('Unknown', $lastEmail['text']); // Country + $this->assertStringContainsString($response['body']['clientName'], $lastEmail['text']); // Client name } /** From 66c761b73eea464990f8a2bada9501fbddef4c46 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 28 Jun 2024 19:51:33 +0530 Subject: [PATCH 27/38] Create execution even when logging is disabled --- app/controllers/api/functions.php | 11 +++-------- src/Appwrite/Platform/Workers/Functions.php | 14 +++++--------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 69086b4fca..8c7611b79c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1702,6 +1702,7 @@ App::post('/v1/functions/:functionId/executions') } $executionId = ID::unique(); + var_dump("creating execution document"); $execution = new Document([ '$id' => $executionId, @@ -1729,10 +1730,7 @@ App::post('/v1/functions/:functionId/executions') ->setContext('function', $function); if ($async) { - if ($function->getAttribute('logging')) { - /** @var Document $execution */ - $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); - } + $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); $queueForFunctions ->setType('http') @@ -1850,10 +1848,7 @@ App::post('/v1/functions/:functionId/executions') ; } - if ($function->getAttribute('logging')) { - /** @var Document $execution */ - $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); - } + $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index acef40382a..43578fa6c2 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -328,9 +328,7 @@ class Functions extends Action 'search' => implode(' ', [$functionId, $executionId]), ]); - if ($function->getAttribute('logging')) { - $execution = $dbForProject->createDocument('executions', $execution); - } + $execution = $dbForProject->createDocument('executions', $execution); // TODO: @Meldiron Trigger executions.create event here @@ -342,9 +340,7 @@ class Functions extends Action if ($execution->getAttribute('status') !== 'processing') { $execution->setAttribute('status', 'processing'); - if ($function->getAttribute('logging')) { - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - } + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); } $durationStart = \microtime(true); @@ -455,9 +451,9 @@ class Functions extends Action ; } - if ($function->getAttribute('logging')) { - $execution = $dbForProject->updateDocument('executions', $executionId, $execution); - } + + $execution = $dbForProject->updateDocument('executions', $executionId, $execution); + /** Trigger Webhook */ $executionModel = new Execution(); $queueForEvents From bb3ee810654e172ef48c625c53b9e9412726b1b1 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 28 Jun 2024 22:42:55 +0100 Subject: [PATCH 28/38] chore: rename metadata to data --- app/config/collections.php | 4 ++-- app/controllers/api/functions.php | 4 ++-- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 2b7777e85b..1b7036c587 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4551,7 +4551,7 @@ $consoleCollections = array_merge([ 'filters' => [], ], [ - '$id' => ID::custom('metadata'), + '$id' => ID::custom('data'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 65535, @@ -4559,7 +4559,7 @@ $consoleCollections = array_merge([ 'required' => false, 'default' => new \stdClass(), 'array' => false, - 'filters' => ['json'], + 'filters' => ['json', 'encrypt'], ], [ '$id' => ID::custom('active'), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b4ef380d8f..c6ab7d95d2 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1764,7 +1764,7 @@ App::post('/v1/functions/:functionId/executions') ->setParam('executionId', $execution->getId()) ->trigger(); } else { - $metadata = [ + $data = [ 'headers' => $headers, 'path' => $path, 'method' => $method, @@ -1780,7 +1780,7 @@ App::post('/v1/functions/:functionId/executions') 'resourceUpdatedAt' => DateTime::now(), 'projectId' => $project->getId(), 'schedule' => $scheduledAt, - 'metadata' => $metadata, + 'data' => $data, 'active' => true, ])); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 55cefb2a44..a47e7b120e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -46,10 +46,10 @@ class ScheduleExecutions extends ScheduleBase // TODO: Refactor to use function instead of functionId ->setFunctionId($schedule['resource']['functionId']) ->setExecution($schedule['resource']) - ->setMethod($schedule['metadata']['method'] ?? 'POST') - ->setPath($schedule['metadata']['path'] ?? '/') - ->setHeaders($schedule['metadata']['headers'] ?? []) - ->setBody($schedule['metadata']['body'] ?? '') + ->setMethod($schedule['data']['method'] ?? 'POST') + ->setPath($schedule['data']['path'] ?? '/') + ->setHeaders($schedule['data']['headers'] ?? []) + ->setBody($schedule['data']['body'] ?? '') ->setProject($schedule['project']) ->trigger(); From 7bdcd5c43653b9b9ceb64b0f9f4a6002e4a3f1a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 1 Jul 2024 06:57:18 +0000 Subject: [PATCH 29/38] Multipoart support --- app/controllers/general.php | 9 +- src/Appwrite/Utopia/Fetch/BodyMultipart.php | 152 ++++++++++++++++++++ src/Executor/Executor.php | 20 ++- 3 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 src/Appwrite/Utopia/Fetch/BodyMultipart.php diff --git a/app/controllers/general.php b/app/controllers/general.php index d4217477fb..75429af8ed 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -274,6 +274,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo requestTimeout: 30 ); + \var_dump($executionResponse); + $headersFiltered = []; foreach ($executionResponse['headers'] as $key => $value) { if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_RESPONSE)) { @@ -325,13 +327,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $body = $execution['responseBody'] ?? ''; - $encodingKey = \array_search('x-open-runtimes-encoding', \array_column($execution['responseHeaders'], 'name')); - if ($encodingKey !== false) { - if (($execution['responseHeaders'][$encodingKey]['value'] ?? '') === 'base64') { - $body = \base64_decode($body); - } - } - $contentType = 'text/plain'; foreach ($execution['responseHeaders'] as $header) { if (\strtolower($header['name']) === 'content-type') { diff --git a/src/Appwrite/Utopia/Fetch/BodyMultipart.php b/src/Appwrite/Utopia/Fetch/BodyMultipart.php new file mode 100644 index 0000000000..ed009ed8da --- /dev/null +++ b/src/Appwrite/Utopia/Fetch/BodyMultipart.php @@ -0,0 +1,152 @@ + $parts + */ + private array $parts = []; + private string $boundary = ""; + + public function __construct(string $boundary = null) + { + if (is_null($boundary)) { + $this->boundary = self::generateBoundary(); + } else { + $this->boundary = $boundary; + } + } + + public static function generateBoundary(): string + { + return '-----------------------------' . \uniqid(); + } + + public function load(string $body): self + { + $eol = "\r\n"; + + $sections = \explode('--' . $this->boundary, $body); + + foreach ($sections as $section) { + if (empty($section)) { + continue; + } + + if (strpos($section, $eol) === 0) { + $section = substr($section, \strlen($eol)); + } + + if (substr($section, -2) === $eol) { + $section = substr($section, 0, -1 * \strlen($eol)); + } + + if ($section == '--') { + continue; + } + + $partChunks = \explode($eol . $eol, $section, 2); + + if (\count($partChunks) < 2) { + continue; // Broken part + } + + [ $partHeaders, $partBody ] = $partChunks; + $partHeaders = \explode($eol, $partHeaders); + + $partName = ""; + foreach ($partHeaders as $partHeader) { + if (!empty($partName)) { + break; + } + + $partHeaderArray = \explode(':', $partHeader, 2); + + $partHeaderName = \strtolower($partHeaderArray[0] ?? ''); + $partHeaderValue = $partHeaderArray[1] ?? ''; + if ($partHeaderName == "content-disposition") { + $dispositionChunks = \explode("; ", $partHeaderValue); + foreach ($dispositionChunks as $dispositionChunk) { + $dispositionChunkValues = \explode("=", $dispositionChunk, 2); + if (\count($dispositionChunkValues) >= 2) { + if ($dispositionChunkValues[0] === "name") { + $partName = \trim($dispositionChunkValues[1], "\""); + break; + } + } + } + } + } + + if (!empty($partName)) { + $this->parts[$partName] = $partBody; + } + } + return $this; + } + + /** + * @return array + */ + public function getParts(): array + { + return $this->parts ?? []; + } + + public function getPart(string $key, mixed $default = ''): mixed + { + return $this->parts[$key] ?? $default; + } + + public function setPart(string $key, mixed $value): self + { + $this->parts[$key] = $value; + return $this; + } + + public function getBoundary(): string + { + return $this->boundary; + } + + public function setBoundary(string $boundary): self + { + $this->boundary = $boundary; + return $this; + } + + public function exportHeader(): string + { + return 'multipart/form-data; boundary=' . $this->boundary; + } + + public function exportBody(): string + { + $eol = "\r\n"; + $query = '--' . $this->boundary; + + foreach ($this->parts as $key => $value) { + $query .= $eol . 'Content-Disposition: form-data; name="' . $key . '"'; + + if (\is_array($value)) { + $query .= $eol . 'Content-Type: application/json'; + $value = \json_encode($value); + } else { + $isBinary = ! mb_check_encoding($value, 'UTF-8'); + if ($isBinary) { + $query .= $eol . 'Content-Transfer-Encoding: binary'; + } + } + + $query .= $eol . $eol; + $query .= $value . $eol; + $query .= '--' . $this->boundary; + } + + $query .= "--" . $eol; + + return $query; + } +} \ No newline at end of file diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index d80b0037af..ab225b67b1 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -2,6 +2,7 @@ namespace Executor; +use Appwrite\Utopia\Fetch\BodyMultipart; use Exception; use Utopia\System\System; @@ -250,7 +251,13 @@ class Executor break; case 'multipart/form-data': - $query = $this->flatten($params); + $multipart = new BodyMultipart(); + foreach ($params as $key => $value) { + $multipart->setPart($key, $value); + } + + $headers['content-type'] = $multipart->exportHeader(); + $query = $multipart->exportBody(); break; default: @@ -315,7 +322,16 @@ class Executor $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($decode) { - switch (substr($responseType, 0, strpos($responseType, ';'))) { + $strpos = strpos($responseType, ';'); + $strpos = \is_bool($strpos) ? \strlen($responseType) : $strpos; + switch (substr($responseType, 0, $strpos)) { + case 'multipart/form-data': + $boundary = \explode('boundary=', $responseHeaders['content-type'] ?? '')[1] ?? ''; + $multipartResponse = new BodyMultipart($boundary); + $multipartResponse->load(\is_bool($responseBody) ? '' : $responseBody); + + $responseBody = $multipartResponse->getParts(); + break; case 'application/json': $json = json_decode($responseBody, true); From f87e2bc8794dd1dc4d47730161e555ed90d22907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 1 Jul 2024 10:35:06 +0200 Subject: [PATCH 30/38] Binary support & tests for Router --- app/controllers/general.php | 2 - composer.lock | 95 +++++---- src/Appwrite/Utopia/Fetch/BodyMultipart.php | 7 +- src/Executor/Executor.php | 7 +- tests/e2e/Client.php | 3 +- .../Functions/FunctionsCustomServerTest.php | 201 ++++++++++++++++++ .../functions/php-binary-request/index.php | 6 + .../functions/php-binary-response/index.php | 6 + 8 files changed, 270 insertions(+), 57 deletions(-) create mode 100644 tests/resources/functions/php-binary-request/index.php create mode 100644 tests/resources/functions/php-binary-response/index.php diff --git a/app/controllers/general.php b/app/controllers/general.php index 75429af8ed..a44423a791 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -274,8 +274,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo requestTimeout: 30 ); - \var_dump($executionResponse); - $headersFiltered = []; foreach ($executionResponse['headers'] as $key => $value) { if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_RESPONSE)) { diff --git a/composer.lock b/composer.lock index 5c30d5a1cc..6eee10ba6b 100644 --- a/composer.lock +++ b/composer.lock @@ -1045,16 +1045,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.29.0", + "version": "v1.30.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b" + "reference": "77fa7995ac1b21ab60769b7323d600a991a90433" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", - "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/77fa7995ac1b21ab60769b7323d600a991a90433", + "reference": "77fa7995ac1b21ab60769b7323d600a991a90433", "shasum": "" }, "require": { @@ -1105,7 +1105,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.30.0" }, "funding": [ { @@ -1121,7 +1121,7 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-05-31T15:07:36+00:00" }, { "name": "thecodingmachine/safe", @@ -2730,16 +2730,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.38.6", + "version": "0.38.8", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "d7016d6d72545e84709892faca972eb4bf5bd699" + "reference": "6367c57ddbcf7b88cacb900c4fe7ef3f28bf38ef" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d7016d6d72545e84709892faca972eb4bf5bd699", - "reference": "d7016d6d72545e84709892faca972eb4bf5bd699", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6367c57ddbcf7b88cacb900c4fe7ef3f28bf38ef", + "reference": "6367c57ddbcf7b88cacb900c4fe7ef3f28bf38ef", "shasum": "" }, "require": { @@ -2775,9 +2775,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.38.6" + "source": "https://github.com/appwrite/sdk-generator/tree/0.38.8" }, - "time": "2024-05-20T18:00:16+00:00" + "time": "2024-06-17T00:42:27+00:00" }, { "name": "doctrine/deprecations", @@ -2898,16 +2898,16 @@ }, { "name": "laravel/pint", - "version": "v1.16.0", + "version": "v1.16.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "1b3a3dc5bc6a81ff52828ba7277621f1d49d6d98" + "reference": "9266a47f1b9231b83e0cfd849009547329d871b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/1b3a3dc5bc6a81ff52828ba7277621f1d49d6d98", - "reference": "1b3a3dc5bc6a81ff52828ba7277621f1d49d6d98", + "url": "https://api.github.com/repos/laravel/pint/zipball/9266a47f1b9231b83e0cfd849009547329d871b1", + "reference": "9266a47f1b9231b83e0cfd849009547329d871b1", "shasum": "" }, "require": { @@ -2918,13 +2918,13 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.57.1", - "illuminate/view": "^10.48.10", - "larastan/larastan": "^2.9.6", + "friendsofphp/php-cs-fixer": "^3.59.3", + "illuminate/view": "^10.48.12", + "larastan/larastan": "^2.9.7", "laravel-zero/framework": "^10.4.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.15.1", - "pestphp/pest": "^2.34.7" + "pestphp/pest": "^2.34.8" }, "bin": [ "builds/pint" @@ -2960,7 +2960,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-05-21T18:08:25+00:00" + "time": "2024-06-18T16:50:05+00:00" }, { "name": "matthiasmullie/minify", @@ -3088,16 +3088,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.11.1", + "version": "1.12.0", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c" + "reference": "3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", - "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c", + "reference": "3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c", "shasum": "" }, "require": { @@ -3105,11 +3105,12 @@ }, "conflict": { "doctrine/collections": "<1.6.8", - "doctrine/common": "<2.13.3 || >=3,<3.2.2" + "doctrine/common": "<2.13.3 || >=3 <3.2.2" }, "require-dev": { "doctrine/collections": "^1.6.8", "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" }, "type": "library", @@ -3135,7 +3136,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.1" + "source": "https://github.com/myclabs/DeepCopy/tree/1.12.0" }, "funding": [ { @@ -3143,7 +3144,7 @@ "type": "tidelift" } ], - "time": "2023-03-08T13:26:56+00:00" + "time": "2024-06-12T14:39:25+00:00" }, { "name": "nikic/php-parser", @@ -3567,16 +3568,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.29.0", + "version": "1.29.1", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc" + "reference": "fcaefacf2d5c417e928405b71b400d4ce10daaf4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/536889f2b340489d328f5ffb7b02bb6b183ddedc", - "reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/fcaefacf2d5c417e928405b71b400d4ce10daaf4", + "reference": "fcaefacf2d5c417e928405b71b400d4ce10daaf4", "shasum": "" }, "require": { @@ -3608,9 +3609,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.29.0" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.29.1" }, - "time": "2024-05-06T12:04:23+00:00" + "time": "2024-05-31T08:52:43+00:00" }, { "name": "phpunit/php-code-coverage", @@ -5081,16 +5082,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.29.0", + "version": "v1.30.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4" + "reference": "0424dff1c58f028c451efff2045f5d92410bd540" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4", - "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/0424dff1c58f028c451efff2045f5d92410bd540", + "reference": "0424dff1c58f028c451efff2045f5d92410bd540", "shasum": "" }, "require": { @@ -5140,7 +5141,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.30.0" }, "funding": [ { @@ -5156,20 +5157,20 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-05-31T15:07:36+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.29.0", + "version": "v1.30.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" + "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", - "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fd22ab50000ef01661e2a31d850ebaa297f8e03c", + "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c", "shasum": "" }, "require": { @@ -5220,7 +5221,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.30.0" }, "funding": [ { @@ -5236,7 +5237,7 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-06-19T12:30:46+00:00" }, { "name": "textalk/websocket", @@ -5475,5 +5476,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Utopia/Fetch/BodyMultipart.php b/src/Appwrite/Utopia/Fetch/BodyMultipart.php index ed009ed8da..3869150758 100644 --- a/src/Appwrite/Utopia/Fetch/BodyMultipart.php +++ b/src/Appwrite/Utopia/Fetch/BodyMultipart.php @@ -133,11 +133,6 @@ class BodyMultipart if (\is_array($value)) { $query .= $eol . 'Content-Type: application/json'; $value = \json_encode($value); - } else { - $isBinary = ! mb_check_encoding($value, 'UTF-8'); - if ($isBinary) { - $query .= $eol . 'Content-Transfer-Encoding: binary'; - } } $query .= $eol . $eol; @@ -149,4 +144,4 @@ class BodyMultipart return $query; } -} \ No newline at end of file +} diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index ab225b67b1..fe4393c7e5 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -212,7 +212,7 @@ class Executor $requestTimeout = $timeout + 15; } - $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $requestTimeout); + $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout); $status = $response['headers']['status-code']; if ($status >= 400) { @@ -220,6 +220,11 @@ class Executor throw new \Exception($message, $status); } + $response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true); + $response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500); + $response['body']['duration'] = \intval($response['body']['duration'] ?? 0); + $response['body']['startTime'] = \intval($response['body']['startTime'] ?? \microtime(true)); + return $response['body']; } diff --git a/tests/e2e/Client.php b/tests/e2e/Client.php index 7083095da1..c8bf36a2ef 100644 --- a/tests/e2e/Client.php +++ b/tests/e2e/Client.php @@ -163,7 +163,7 @@ class Client * @return array * @throws Exception */ - public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true): array + public function call(string $method, string $path = '', array $headers = [], mixed $params = [], bool $decode = true): array { $headers = array_merge($this->headers, $headers); $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); @@ -174,6 +174,7 @@ class Client 'application/json' => json_encode($params), 'multipart/form-data' => $this->flatten($params), 'application/graphql' => $params[0], + 'text/plain' => $params, default => http_build_query($params), }; diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 4b2a8e0bc7..c35a1fc746 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1716,4 +1716,205 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); } + + public function testFunctionsDomainBianryResponse() + { + $timeout = 15; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-response/code.tar.gz"; + $this->packageCode('php-binary-response'); + + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test PHP Binary executions', + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'timeout' => $timeout, + 'execute' => ['any'] + ]); + + $functionId = $function['body']['$id'] ?? ''; + + $this->assertEquals(201, $function['headers']['status-code']); + + $rules = $this->client->call(Client::METHOD_GET, '/proxy/rules', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::equal('resourceId', [$functionId])->toString(), + Query::equal('resourceType', ['function'])->toString(), + ], + ]); + + $this->assertEquals(200, $rules['headers']['status-code']); + $this->assertEquals(1, $rules['body']['total']); + $this->assertCount(1, $rules['body']['rules']); + $this->assertNotEmpty($rules['body']['rules'][0]['domain']); + + $domain = $rules['body']['rules'][0]['domain']; + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $deployment['headers']['status-code']); + + // Wait a little for activation to finish + sleep(5); + + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + + $response = $proxyClient->call(Client::METHOD_GET, '/', [], [], false); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $bytes = unpack('C*byte', $response['body']); + $this->assertCount(3, $bytes); + $this->assertEquals(0, $bytes['byte1']); + $this->assertEquals(10, $bytes['byte2']); + $this->assertEquals(255, $bytes['byte3']); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } + + public function testFunctionsDomainBianryRequest() + { + $timeout = 15; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-request/code.tar.gz"; + $this->packageCode('php-binary-request'); + + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test PHP Binary executions', + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'timeout' => $timeout, + 'execute' => ['any'] + ]); + + $functionId = $function['body']['$id'] ?? ''; + + $this->assertEquals(201, $function['headers']['status-code']); + + $rules = $this->client->call(Client::METHOD_GET, '/proxy/rules', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::equal('resourceId', [$functionId])->toString(), + Query::equal('resourceType', ['function'])->toString(), + ], + ]); + + $this->assertEquals(200, $rules['headers']['status-code']); + $this->assertEquals(1, $rules['body']['total']); + $this->assertCount(1, $rules['body']['rules']); + $this->assertNotEmpty($rules['body']['rules'][0]['domain']); + + $domain = $rules['body']['rules'][0]['domain']; + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $deployment['headers']['status-code']); + + // Wait a little for activation to finish + sleep(5); + + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + + $bytes = pack('C*', ...[0,20,255]); + + $response = $proxyClient->call(Client::METHOD_POST, '/', [ 'content-type' => 'text/plain' ], $bytes, false); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(\md5($bytes), $response['body']); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } } diff --git a/tests/resources/functions/php-binary-request/index.php b/tests/resources/functions/php-binary-request/index.php new file mode 100644 index 0000000000..53df8705e5 --- /dev/null +++ b/tests/resources/functions/php-binary-request/index.php @@ -0,0 +1,6 @@ +req->bodyBinary); + return $context->res->send($hash); +}; diff --git a/tests/resources/functions/php-binary-response/index.php b/tests/resources/functions/php-binary-response/index.php new file mode 100644 index 0000000000..7715663388 --- /dev/null +++ b/tests/resources/functions/php-binary-response/index.php @@ -0,0 +1,6 @@ +res->binary($bytes); +}; From fef22825fb0d384490197aa9aaae3ec8da40dad0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:35:37 +0100 Subject: [PATCH 31/38] chore: delete schedule if not active --- src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index a47e7b120e..2fdbd98da3 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -29,6 +29,11 @@ class ScheduleExecutions extends ScheduleBase foreach ($this->schedules as $schedule) { if (!$schedule['active']) { + $dbForConsole->deleteDocument( + 'schedules', + $schedule['$id'], + ); + unset($this->schedules[$schedule['resourceId']]); continue; } From 6fdc5a342590392d8e914d52a6a8d24d81b752bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 Jul 2024 08:43:04 +0000 Subject: [PATCH 32/38] PR review changes --- app/init.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/init.php b/app/init.php index c6fe6e2409..2291389c23 100644 --- a/app/init.php +++ b/app/init.php @@ -1323,6 +1323,7 @@ App::setResource('console', function () { 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds + 'sessionAlerts' => true ], 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], From e9034a9d9d7d8495d0d6af030154fe48a97c668f Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:08:59 +0530 Subject: [PATCH 33/38] Fix missing closing bracket --- app/controllers/api/functions.php | 3 +- composer.lock | 176 +++++++++++++++--------------- 2 files changed, 90 insertions(+), 89 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0cec9aa8fd..4af12fa9e5 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1854,7 +1854,8 @@ App::post('/v1/functions/:functionId/executions') ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function ; - $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); + } $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); diff --git a/composer.lock b/composer.lock index 341342455c..8a72c47a65 100644 --- a/composer.lock +++ b/composer.lock @@ -1126,6 +1126,86 @@ }, "time": "2022-03-17T08:00:35+00:00" }, + { + "name": "symfony/polyfill-mbstring", + "version": "v1.30.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-mbstring.git", + "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fd22ab50000ef01661e2a31d850ebaa297f8e03c", + "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "provide": { + "ext-mbstring": "*" + }, + "suggest": { + "ext-mbstring": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for the Mbstring extension", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "mbstring", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.30.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-06-19T12:30:46+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.30.0", @@ -3326,16 +3406,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.0.2", + "version": "v5.1.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13" + "reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/139676794dc1e9231bf7bcd123cfc0c99182cb13", - "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/683130c2ff8c2739f4822ff7ac5c873ec529abd1", + "reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1", "shasum": "" }, "require": { @@ -3346,7 +3426,7 @@ }, "require-dev": { "ircmaxell/php-yacc": "^0.0.7", - "phpunit/phpunit": "^7.0 || ^8.0 || ^9.0" + "phpunit/phpunit": "^9.0" }, "bin": [ "bin/php-parse" @@ -3378,9 +3458,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.1.0" }, - "time": "2024-03-05T20:51:40+00:00" + "time": "2024-07-01T20:03:41+00:00" }, { "name": "phar-io/manifest", @@ -5337,86 +5417,6 @@ ], "time": "2024-05-31T15:07:36+00:00" }, - { - "name": "symfony/polyfill-mbstring", - "version": "v1.30.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fd22ab50000ef01661e2a31d850ebaa297f8e03c", - "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c", - "shasum": "" - }, - "require": { - "php": ">=7.1" - }, - "provide": { - "ext-mbstring": "*" - }, - "suggest": { - "ext-mbstring": "For best performance" - }, - "type": "library", - "extra": { - "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Mbstring\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill for the Mbstring extension", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "mbstring", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.30.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-06-19T12:30:46+00:00" - }, { "name": "textalk/websocket", "version": "1.5.7", @@ -5615,5 +5615,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.3.0" } From f1603ecb6a233ff07a9646087ede113ed5c81649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 Jul 2024 17:15:20 +0000 Subject: [PATCH 34/38] Upgrade executor --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b68584d685..91826b4c68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -850,7 +850,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.5.5 + image: openruntimes/executor:0.6.0 restart: unless-stopped networks: - appwrite From 7dec12698796f129a32db2a9e5472362b206d433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 2 Jul 2024 17:16:53 +0000 Subject: [PATCH 35/38] Revert unwanted change --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 91826b4c68..b68584d685 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -850,7 +850,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.6.0 + image: openruntimes/executor:0.5.5 restart: unless-stopped networks: - appwrite From 5ef196ec9abc15f6feb55bd951e128afd096dc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 3 Jul 2024 07:11:15 +0000 Subject: [PATCH 36/38] PR review changes --- .env | 1 + app/config/variables.php | 9 +++++++++ app/controllers/api/projects.php | 6 +++--- app/init.php | 2 +- app/views/install/compose.phtml | 1 + docker-compose.yml | 1 + 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 19215e23ee..9cccf5ee7e 100644 --- a/.env +++ b/.env @@ -4,6 +4,7 @@ _APP_LOCALE=en _APP_WORKER_PER_CORE=6 _APP_CONSOLE_WHITELIST_ROOT=disabled _APP_CONSOLE_WHITELIST_EMAILS= +_APP_CONSOLE_SESSION_ALERTS=enabled _APP_CONSOLE_WHITELIST_IPS= _APP_CONSOLE_COUNTRIES_DENYLIST=AQ _APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io diff --git a/app/config/variables.php b/app/config/variables.php index ef30d4d17b..b986ce4247 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -250,6 +250,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_CONSOLE_SESSION_ALERTS', + 'description' => 'This option allows you configure if a new login in the Appwrite Console should send an alert email to the user. It\'s disabled by default with value "disabled", and to enable it, pass value "enabled".', + 'introduction' => '1.6.0', + 'default' => 'disabled', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], [ diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 1533456f03..ff22337481 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -616,10 +616,10 @@ App::patch('/v1/projects/:projectId/auth/session-alerts') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('sessionAlerts', false, new Boolean(true), 'Set to true to enable session emails.') + ->param('alerts', false, new Boolean(true), 'Set to true to enable session emails.') ->inject('response') ->inject('dbForConsole') - ->action(function (string $projectId, bool $sessionAlerts, Response $response, Database $dbForConsole) { + ->action(function (string $projectId, bool $alerts, Response $response, Database $dbForConsole) { $project = $dbForConsole->getDocument('projects', $projectId); @@ -628,7 +628,7 @@ App::patch('/v1/projects/:projectId/auth/session-alerts') } $auths = $project->getAttribute('auths', []); - $auths['sessionAlerts'] = $sessionAlerts; + $auths['sessionAlerts'] = $alerts; $dbForConsole->updateDocument('projects', $project->getId(), $project ->setAttribute('auths', $auths)); diff --git a/app/init.php b/app/init.php index 2291389c23..fe28407724 100644 --- a/app/init.php +++ b/app/init.php @@ -1323,7 +1323,7 @@ App::setResource('console', function () { 'invites' => System::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled', 'limit' => (System::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds - 'sessionAlerts' => true + 'sessionAlerts' => System::getEnv('_APP_CONSOLE_SESSION_ALERTS', 'disabled') === 'enabled' ], 'authWhitelistEmails' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [], 'authWhitelistIPs' => (!empty(System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', System::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [], diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index a6e3521e52..2f1be0b2dd 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -75,6 +75,7 @@ $image = $this->getParam('image', ''); - _APP_LOCALE - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS + - _APP_CONSOLE_SESSION_ALERTS - _APP_CONSOLE_WHITELIST_IPS - _APP_CONSOLE_HOSTNAMES - _APP_SYSTEM_EMAIL_NAME diff --git a/docker-compose.yml b/docker-compose.yml index d5d27ce461..1bcbd78a12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,7 @@ services: - _APP_LOCALE - _APP_CONSOLE_WHITELIST_ROOT - _APP_CONSOLE_WHITELIST_EMAILS + - _APP_CONSOLE_SESSION_ALERTS - _APP_CONSOLE_WHITELIST_IPS - _APP_CONSOLE_HOSTNAMES - _APP_SYSTEM_EMAIL_NAME From 5fb41fd2169b2f85fc754cdf7afcfff04eaae205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 3 Jul 2024 08:00:17 +0000 Subject: [PATCH 37/38] Fix executions --- app/controllers/api/functions.php | 1 - docker-compose.yml | 2 +- src/Executor/Executor.php | 5 ++++- tests/e2e/General/HTTPTest.php | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 4af12fa9e5..f9863c3607 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1704,7 +1704,6 @@ App::post('/v1/functions/:functionId/executions') } $executionId = ID::unique(); - var_dump("creating execution document"); $execution = new Document([ '$id' => $executionId, diff --git a/docker-compose.yml b/docker-compose.yml index ba4277a053..f7834a29ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -823,7 +823,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.5.5 + image: openruntimes/executor:0.6.0 restart: unless-stopped networks: - appwrite diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 37c2963be9..34cdae38d2 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -191,7 +191,6 @@ class Executor $params = [ 'runtimeId' => $runtimeId, 'variables' => $variables, - 'body' => $body, 'timeout' => $timeout, 'path' => $path, 'method' => $method, @@ -206,6 +205,10 @@ class Executor 'logging' => $logging, ]; + if(!empty($body)) { + $params['body'] = $body; + } + // Safety timeout. Executor has timeout, and open runtime has soft timeout. // This one shouldn't really happen, but prevents from unexpected networking behaviours. if ($requestTimeout == null) { diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 67b3add8a6..92bc52561c 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -108,7 +108,6 @@ class HTTPTest extends Scope '0.14.x', ]; - // var_dump($files); foreach ($files as $file) { if (in_array($file, ['.', '..'])) { continue; From f59a6cf22ed1e5a14d014b47d6398d6d020cfe51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 3 Jul 2024 08:02:16 +0000 Subject: [PATCH 38/38] Fix test --- tests/e2e/Services/Account/AccountCustomClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 7a6b469d9b..321b1110fd 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1206,7 +1206,7 @@ class AccountCustomClientTest extends Scope 'x-appwrite-project' => 'console', 'cookie' => 'a_session_console=' . $this->getRoot()['session'], ]), [ - 'sessionAlerts' => true, + 'alerts' => true, ]); $this->assertEquals(200, $response['headers']['status-code']);