From 7a2ee683e3c231eabc15da4c8ea7bb3af21e090a Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 28 Jan 2024 11:28:59 +0200 Subject: [PATCH 01/23] refactor usage poc --- Dockerfile | 3 +- app/controllers/api/users.php | 6 + app/worker.php | 15 +-- bin/worker-usage-dump | 3 + docker-compose.yml | 32 ++++++ src/Appwrite/Event/Event.php | 3 + src/Appwrite/Event/Usage.php | 9 ++ src/Appwrite/Event/UsageDump.php | 47 ++++++++ src/Appwrite/Platform/Services/Workers.php | 4 +- src/Appwrite/Platform/Workers/Usage.php | 69 +++++++++--- src/Appwrite/Platform/Workers/UsageDump.php | 115 ++++++++++++++++++++ src/Appwrite/Platform/Workers/UsageHook.php | 103 ------------------ 12 files changed, 278 insertions(+), 131 deletions(-) create mode 100644 bin/worker-usage-dump create mode 100644 src/Appwrite/Event/UsageDump.php create mode 100644 src/Appwrite/Platform/Workers/UsageDump.php delete mode 100644 src/Appwrite/Platform/Workers/UsageHook.php diff --git a/Dockerfile b/Dockerfile index 324a3a5484..4088d7c8f9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -95,7 +95,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-hamster && \ - chmod +x /usr/local/bin/worker-usage + chmod +x /usr/local/bin/worker-usage && \ + chmod +x /usr/local/bin/worker-usage-dump # Cloud Executabless diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 38d65fba7e..f291213cb9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -9,6 +9,7 @@ use Appwrite\Event\Event; use Appwrite\Network\Validator\Email; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; +use Utopia\CLI\Console; use Utopia\Database\Validator\Queries; use Appwrite\Utopia\Database\Validator\Queries\Users; use Utopia\Database\Validator\Query\Limit; @@ -141,6 +142,8 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); + //Todo debug (to be removed laster @shimon) + //Console::log('@create user=' . time() . '=' . $user->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1191,6 +1194,9 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); + //Todo debug (to be removed laster @shimon) + //Console::log('@delete user=' . $userId . '=' . time() . '=' . $user->getId()); + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($clone); diff --git a/app/worker.php b/app/worker.php index 1b1f4b9f93..4cf0edbae6 100644 --- a/app/worker.php +++ b/app/worker.php @@ -14,6 +14,7 @@ use Appwrite\Event\Mail; use Appwrite\Event\Migration; use Appwrite\Event\Phone; use Appwrite\Event\Usage; +use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; use Utopia\App; @@ -146,6 +147,9 @@ Server::setResource('log', fn() => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { return new Usage($queue); }, ['queue']); +Server::setResource('queueForUsageDump', function (Connection $queue) { + return new UsageDump($queue); +}, ['queue']); Server::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); @@ -299,12 +303,9 @@ $worker Console::error('[Error] Line: ' . $error->getLine()); }); -try { - $workerStart = $worker->getWorkerStart(); -} catch (\Throwable $error) { - $worker->workerStart(); -} finally { - Console::info("Worker $workerName started"); -} +$worker->workerStart() + ->action(function () use ($workerName) { + Console::info("Worker $workerName started"); + }); $worker->start(); diff --git a/bin/worker-usage-dump b/bin/worker-usage-dump new file mode 100644 index 0000000000..43ca87fcb3 --- /dev/null +++ b/bin/worker-usage-dump @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php usage-dump $@ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5c645e3bcd..2133de0dce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -676,6 +676,38 @@ services: - _APP_LOGGING_CONFIG - _APP_USAGE_AGGREGATION_INTERVAL + appwrite-worker-usage-dump: + entrypoint: worker-usage-dump + <<: *x-logging + container_name: appwrite-worker-usage-dump + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_USAGE_STATS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_USAGE_AGGREGATION_INTERVAL + + appwrite-schedule: entrypoint: schedule <<: *x-logging diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index fc12c5b5b3..9f71ef5ebc 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -27,6 +27,9 @@ class Event public const USAGE_QUEUE_NAME = 'v1-usage'; public const USAGE_CLASS_NAME = 'UsageV1'; + public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump'; + public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1'; + public const WEBHOOK_QUEUE_NAME = 'v1-webhooks'; public const WEBHOOK_CLASS_NAME = 'WebhooksV1'; diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 398c3319f2..4f53c7df14 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -2,6 +2,7 @@ namespace Appwrite\Event; +use Utopia\CLI\Console; use Utopia\Queue\Client; use Utopia\Queue\Connection; use Utopia\Database\Document; @@ -42,6 +43,14 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { + //Todo debug (to be removed laster @shimon) +// if ($key === 'users') { +// if ($value < 0) { +// console::log('@negative=' . $value); +// } else { +// console::log('@positive=' . $value); +// } +// } $this->metrics[] = [ 'key' => $key, 'value' => $value, diff --git a/src/Appwrite/Event/UsageDump.php b/src/Appwrite/Event/UsageDump.php new file mode 100644 index 0000000000..8f87908849 --- /dev/null +++ b/src/Appwrite/Event/UsageDump.php @@ -0,0 +1,47 @@ +setQueue(Event::USAGE_DUMP_QUEUE_NAME) + ->setClass(Event::USAGE_DUMP_CLASS_NAME); + } + + /** + * Add Stats. + * + * @param array $stats + * @return self + */ + public function setStats(array $stats): self + { + $this->stats = $stats; + + return $this; + } + + /** + * Sends metrics to the usage worker. + * + * @return string|bool + */ + public function trigger(): string|bool + { + $client = new Client($this->queue, $this->connection); + + return $client->enqueue([ + 'stats' => $this->stats, + ]); + } +} diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 6573b3124c..22e6dcd56a 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; use Appwrite\Platform\Workers\Hamster; use Appwrite\Platform\Workers\Usage; -use Appwrite\Platform\Workers\UsageHook; +use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -33,7 +33,7 @@ class Workers extends Service ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) ->addAction(Hamster::getName(), new Hamster()) - ->addAction(UsageHook::getName(), new UsageHook()) + ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3809d000f7..3527427996 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -4,22 +4,22 @@ namespace Appwrite\Platform\Workers; use Exception; use Utopia\CLI\Console; -use Utopia\Database\Database; -use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; +use Appwrite\Event\UsageDump; use Utopia\Queue\Message; class Usage extends Action { - protected static array $stats = []; - protected array $periods = [ - '1h' => 'Y-m-d H:00', - '1d' => 'Y-m-d 00:00', - 'inf' => '0000-00-00 00:00' - ]; + private array $stats = []; + private int $lastTriggeredTime = 0; + private int $keys = 0; + private const INFINITY_PERIOD = '_inf_'; + private const KEYS_THRESHOLD = 5; + private const KEYS_SENT_THRESHOLD = 60; + + - protected const INFINITY_PERIOD = '_inf_'; public static function getName(): string { return 'usage'; @@ -35,26 +35,29 @@ class Usage extends Action ->desc('Usage worker') ->inject('message') ->inject('getProjectDB') - ->callback(function (Message $message, callable $getProjectDB) { - $this->action($message, $getProjectDB); + ->inject('queueForUsageDump') + ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) { + $this->action($message, $getProjectDB, $queueForUsageDump); }); + + $this->lastTriggeredTime = time(); } /** * @param Message $message * @param callable $getProjectDB + * @param UsageDump $queueForUsageDump * @return void * @throws \Utopia\Database\Exception * @throws Exception */ - public function action(Message $message, callable $getProjectDB): void + public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void { $payload = $message->getPayload() ?? []; if (empty($payload)) { throw new Exception('Missing payload'); } - $payload = $message->getPayload() ?? []; $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -69,17 +72,47 @@ class Usage extends Action getProjectDB: $getProjectDB ); } - self::$stats[$projectId]['project'] = $project; + foreach ($payload['metrics'] ?? [] as $metric) { - if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) { - self::$stats[$projectId]['keys'][$metric['key']] = $metric['value']; + if ($metric['key'] === 'users') { + if ($metric['value'] < 0) { + $this->stats[$metric['key']]['negative'] += $metric['value']; + } else { + $this->stats[$metric['key']]['positive'] += $metric['value']; + } + } + } + + $this->stats[$projectId]['project'] = $project; + foreach ($payload['metrics'] ?? [] as $metric) { + $this->keys++; + if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { + $this->stats[$projectId]['keys'][$metric['key']] = $metric['value']; continue; } - self::$stats[$projectId]['keys'][$metric['key']] += $metric['value']; + + $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; + } + + // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) + if ( + $this->keys >= self::KEYS_THRESHOLD || + (time() - $this->lastTriggeredTime > self::KEYS_SENT_THRESHOLD && $this->keys > 0) + ) { + $offset = count($this->stats); + $chunk = array_slice($this->stats, 0, $offset, true); + array_splice($this->stats, 0, $offset); + + $queueForUsageDump + ->setStats($chunk) + ->trigger(); + + //$this->stats = []; + $this->keys = 0; + $this->lastTriggeredTime = time(); } } - /** * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files. * When we remove a parent document we need to deduct his children aggregation from the project scope. diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php new file mode 100644 index 0000000000..e65aec5199 --- /dev/null +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -0,0 +1,115 @@ + 'Y-m-d H:00', + '1d' => 'Y-m-d 00:00', + 'inf' => '0000-00-00 00:00' + ]; + + public static function getName(): string + { + return 'usage-dump'; + } + + /** + * @throws \Exception + */ + public function __construct() + { + + $this + ->inject('message') + ->inject('getProjectDB') + ->callback(function (Message $message, callable $getProjectDB) { + $this->action($message, $getProjectDB); + }) + ; + } + + /** + * @param Message $message + * @param callable $getProjectDB + * @return void + * @throws Exception + * @throws \Utopia\Database\Exception + */ + public function action(Message $message, callable $getProjectDB): void + { + + $payload = $message->getPayload() ?? []; + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + foreach ($payload['stats'] ?? [] as $stats) { + $project = new Document($stats['project'] ?? []); + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; + $projectInternalId = $project->getInternalId(); + + if ($numberOfKeys === 0) { + continue; + } + + try { + $dbForProject = $getProjectDB($project); + foreach ($stats['keys'] ?? [] as $key => $value) { + if ($value == 0) { + continue; + } + + foreach ($this->periods as $period => $format) { + $time = 'inf' === $period ? null : date($format, time()); + $id = \md5("{$time}_{$period}_{$key}"); + + try { + $dbForProject->createDocument('stats_v2', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $key, + 'value' => $value, + 'region' => App::getEnv('_APP_REGION', 'default'), + ])); + } catch (Duplicate $th) { + if ($value < 0) { + var_dump([ + 'id' => $time . '_' . $period . '_' . $key, + 'value' => $value, + ]); + $dbForProject->decreaseDocumentAttribute( + 'stats_v2', + $id, + 'value', + abs($value) + ); + } else { + $dbForProject->increaseDocumentAttribute( + 'stats_v2', + $id, + 'value', + $value + ); + } + } + } + } + } catch (\Exception $e) { + console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); + } + } + } +} diff --git a/src/Appwrite/Platform/Workers/UsageHook.php b/src/Appwrite/Platform/Workers/UsageHook.php deleted file mode 100644 index 4781b1e892..0000000000 --- a/src/Appwrite/Platform/Workers/UsageHook.php +++ /dev/null @@ -1,103 +0,0 @@ -setType(Action::TYPE_WORKER_START) - ->inject('register') - ->inject('getProjectDB') - ->callback(function ($register, callable $getProjectDB) { - $this->action($register, $getProjectDB); - }) - ; - } - - /** - * @param $register - * @param $getProjectDB - * @return void - */ - public function action($register, $getProjectDB): void - { - - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000'); - Timer::tick($interval, function () use ($register, $getProjectDB) { - - $offset = count(self::$stats); - $projects = array_slice(self::$stats, 0, $offset, true); - array_splice(self::$stats, 0, $offset); - foreach ($projects as $data) { - $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0; - $projectInternalId = $data['project']->getInternalId(); - $database = $data['project']['database'] ?? ''; - - console::warning('Ticker started ' . DateTime::now()); - - if ($numberOfKeys === 0) { - continue; - } - - try { - $dbForProject = $getProjectDB($data['project']); - foreach ($data['keys'] ?? [] as $key => $value) { - if ($value == 0) { - continue; - } - - foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : date($format, time()); - $id = \md5("{$time}_{$period}_{$key}"); - - try { - $dbForProject->createDocument('stats_v2', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $key, - 'value' => $value, - 'region' => App::getEnv('_APP_REGION', 'default'), - ])); - } catch (Duplicate $th) { - if ($value < 0) { - $dbForProject->decreaseDocumentAttribute( - 'stats_v2', - $id, - 'value', - abs($value) - ); - } else { - $dbForProject->increaseDocumentAttribute( - 'stats_v2', - $id, - 'value', - $value - ); - } - } - } - } - } catch (\Exception $e) { - console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); - } - } - }); - } -} From 36fddd415614062cf651ceb0ff79b8c0aa2695df Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 28 Jan 2024 11:46:13 +0200 Subject: [PATCH 02/23] refactor usage poc --- src/Appwrite/Platform/Workers/UsageDump.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index e65aec5199..d0e383be19 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -86,10 +86,11 @@ class UsageDump extends Action ])); } catch (Duplicate $th) { if ($value < 0) { - var_dump([ - 'id' => $time . '_' . $period . '_' . $key, - 'value' => $value, - ]); + //Todo debug (to be removed laster @shimon) +// var_dump([ +// 'id' => $time . '_' . $period . '_' . $key, +// 'value' => $value, +// ]); $dbForProject->decreaseDocumentAttribute( 'stats_v2', $id, From 44ed6826a9c81ec124b20954cc2f025f27d0da2b Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 28 Jan 2024 15:33:04 +0200 Subject: [PATCH 03/23] refactor usage poc --- app/controllers/api/users.php | 4 ++-- src/Appwrite/Event/Usage.php | 2 +- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index f291213cb9..f3faa7838a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -142,7 +142,7 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) //Console::log('@create user=' . time() . '=' . $user->getId()); $response @@ -1194,7 +1194,7 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) //Console::log('@delete user=' . $userId . '=' . time() . '=' . $user->getId()); $queueForDeletes diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 4f53c7df14..0774534e36 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -43,7 +43,7 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) // if ($key === 'users') { // if ($value < 0) { // console::log('@negative=' . $value); diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index d0e383be19..5221d58b2c 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -86,7 +86,7 @@ class UsageDump extends Action ])); } catch (Duplicate $th) { if ($value < 0) { - //Todo debug (to be removed laster @shimon) + //Todo debug (to be removed later @shimon) // var_dump([ // 'id' => $time . '_' . $period . '_' . $key, // 'value' => $value, From 5518ca165e0f6f5eb82b1af69a8e29ff3e4d38b9 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 30 Jan 2024 13:24:57 +0200 Subject: [PATCH 04/23] remarks --- app/controllers/api/users.php | 5 ----- src/Appwrite/Platform/Workers/Usage.php | 8 ++------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index f3faa7838a..cc41ea85b0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -142,8 +142,6 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - //Todo debug (to be removed later @shimon) - //Console::log('@create user=' . time() . '=' . $user->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) @@ -1194,9 +1192,6 @@ App::delete('/v1/users/:userId') $dbForProject->deleteDocument('users', $userId); - //Todo debug (to be removed later @shimon) - //Console::log('@delete user=' . $userId . '=' . time() . '=' . $user->getId()); - $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($clone); diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 3527427996..74b8f4c224 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -99,15 +99,11 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > self::KEYS_SENT_THRESHOLD && $this->keys > 0) ) { - $offset = count($this->stats); - $chunk = array_slice($this->stats, 0, $offset, true); - array_splice($this->stats, 0, $offset); - $queueForUsageDump - ->setStats($chunk) + ->setStats($this->stats) ->trigger(); - //$this->stats = []; + $this->stats = []; $this->keys = 0; $this->lastTriggeredTime = time(); } From 6a6c3445fa646cabfc5ba025eab8405b3d7e3617 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 4 Feb 2024 20:02:01 +0200 Subject: [PATCH 05/23] updates --- app/controllers/api/users.php | 18 +- composer.json | 20 +- composer.lock | 536 ++++++++++++++++++++---- docker-compose.yml | 1 + src/Appwrite/Platform/Workers/Usage.php | 10 - 5 files changed, 484 insertions(+), 101 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index cc41ea85b0..7449ac27c0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -142,7 +142,6 @@ App::post('/v1/users') ->inject('hooks') ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) { $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks); - $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($user, Response::MODEL_USER); @@ -1190,15 +1189,16 @@ App::delete('/v1/users/:userId') // clone user object to send to workers $clone = clone $user; - $dbForProject->deleteDocument('users', $userId); + $affected = $dbForProject->deleteDocument('users', $userId); + if (!empty($affected)) { + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($clone); - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($clone); - - $queueForEvents - ->setParam('userId', $user->getId()) - ->setPayload($response->output($clone, Response::MODEL_USER)); + $queueForEvents + ->setParam('userId', $user->getId()) + ->setPayload($response->output($clone, Response::MODEL_USER)); + } $response->noContent(); }); diff --git a/composer.json b/composer.json index 83bcd26468..79021ad522 100644 --- a/composer.json +++ b/composer.json @@ -45,18 +45,18 @@ "appwrite/php-clamav": "2.0.*", "utopia-php/abuse": "0.33.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.35.*", + "utopia-php/audit": "0.38.*", "utopia-php/cache": "0.9.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.45.*", - "utopia-php/domains": "0.3.*", - "utopia-php/dsn": "0.1.*", + "utopia-php/database": "0.45.6", + "utopia-php/domains": "0.5.*", + "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", - "utopia-php/image": "0.5.*", + "utopia-php/image": "0.6.*", "utopia-php/locale": "0.4.*", "utopia-php/logger": "0.3.*", - "utopia-php/messaging": "0.2.*", + "utopia-php/messaging": "0.9.*", "utopia-php/migration": "0.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.5.*", @@ -73,8 +73,9 @@ "phpmailer/phpmailer": "6.8.0", "chillerlan/php-qrcode": "4.3.4", "adhocore/jwt": "1.1.2", + "spomky-labs/otphp": "^10.0", "webonyx/graphql-php": "14.11.*", - "league/csv": "9.7.1" + "league/csv": "^9.14" }, "repositories": [ { @@ -88,14 +89,15 @@ "phpunit/phpunit": "9.5.20", "squizlabs/php_codesniffer": "^3.7", "swoole/ide-helper": "5.0.2", - "textalk/websocket": "1.5.7" + "textalk/websocket": "1.5.7", + "utopia-php/fetch": "0.1.*" }, "provide": { "ext-phpiredis": "*" }, "config": { "platform": { - "php": "8.0" + "php": "8.2" } } } diff --git a/composer.lock b/composer.lock index 61a8f297d7..d7a827ccb4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b3fd261ed93452413b8e6ba77dbab55", + "content-hash": "b70152e4a3c53762b42d9623ea204cf6", "packages": [ { "name": "adhocore/jwt", @@ -197,6 +197,73 @@ ], "time": "2023-11-22T15:36:00+00:00" }, + { + "name": "beberlei/assert", + "version": "v3.3.2", + "source": { + "type": "git", + "url": "https://github.com/beberlei/assert.git", + "reference": "cb70015c04be1baee6f5f5c953703347c0ac1655" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/beberlei/assert/zipball/cb70015c04be1baee6f5f5c953703347c0ac1655", + "reference": "cb70015c04be1baee6f5f5c953703347c0ac1655", + "shasum": "" + }, + "require": { + "ext-ctype": "*", + "ext-json": "*", + "ext-mbstring": "*", + "ext-simplexml": "*", + "php": "^7.0 || ^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "*", + "phpstan/phpstan": "*", + "phpunit/phpunit": ">=6.0.0", + "yoast/phpunit-polyfills": "^0.1.0" + }, + "suggest": { + "ext-intl": "Needed to allow Assertion::count(), Assertion::isCountable(), Assertion::minCount(), and Assertion::maxCount() to operate on ResourceBundles" + }, + "type": "library", + "autoload": { + "files": [ + "lib/Assert/functions.php" + ], + "psr-4": { + "Assert\\": "lib/Assert" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-2-Clause" + ], + "authors": [ + { + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de", + "role": "Lead Developer" + }, + { + "name": "Richard Quadling", + "email": "rquadling@gmail.com", + "role": "Collaborator" + } + ], + "description": "Thin assertion library for input validation in business models.", + "keywords": [ + "assert", + "assertion", + "validation" + ], + "support": { + "issues": "https://github.com/beberlei/assert/issues", + "source": "https://github.com/beberlei/assert/tree/v3.3.2" + }, + "time": "2021-12-16T21:41:27+00:00" + }, { "name": "chillerlan/php-qrcode", "version": "4.3.4", @@ -463,34 +530,39 @@ }, { "name": "league/csv", - "version": "9.7.1", + "version": "9.14.0", "source": { "type": "git", "url": "https://github.com/thephpleague/csv.git", - "reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1" + "reference": "34bf0df7340b60824b9449b5c526fcc3325070d5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/csv/zipball/0ec57e8264ec92565974ead0d1724cf1026e10c1", - "reference": "0ec57e8264ec92565974ead0d1724cf1026e10c1", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/34bf0df7340b60824b9449b5c526fcc3325070d5", + "reference": "34bf0df7340b60824b9449b5c526fcc3325070d5", "shasum": "" }, "require": { + "ext-filter": "*", "ext-json": "*", "ext-mbstring": "*", - "php": "^7.3 || ^8.0" + "php": "^8.1.2" }, "require-dev": { - "ext-curl": "*", + "doctrine/collections": "^2.1.4", "ext-dom": "*", - "friendsofphp/php-cs-fixer": "^2.16", - "phpstan/phpstan": "^0.12.0", - "phpstan/phpstan-phpunit": "^0.12.0", - "phpstan/phpstan-strict-rules": "^0.12.0", - "phpunit/phpunit": "^9.5" + "ext-xdebug": "*", + "friendsofphp/php-cs-fixer": "^v3.22.0", + "phpbench/phpbench": "^1.2.15", + "phpstan/phpstan": "^1.10.50", + "phpstan/phpstan-deprecation-rules": "^1.1.4", + "phpstan/phpstan-phpunit": "^1.3.15", + "phpstan/phpstan-strict-rules": "^1.5.2", + "phpunit/phpunit": "^10.5.3", + "symfony/var-dumper": "^6.4.0" }, "suggest": { - "ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes", + "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes", "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" }, "type": "library", @@ -520,7 +592,7 @@ } ], "description": "CSV data manipulation made easy in PHP", - "homepage": "http://csv.thephpleague.com", + "homepage": "https://csv.thephpleague.com", "keywords": [ "convert", "csv", @@ -543,7 +615,7 @@ "type": "github" } ], - "time": "2021-04-17T16:32:08+00:00" + "time": "2023-12-29T07:34:53+00:00" }, { "name": "matomo/device-detector", @@ -733,6 +805,73 @@ ], "time": "2019-09-10T13:16:29+00:00" }, + { + "name": "paragonie/constant_time_encoding", + "version": "v2.6.3", + "source": { + "type": "git", + "url": "https://github.com/paragonie/constant_time_encoding.git", + "reference": "58c3f47f650c94ec05a151692652a868995d2938" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/58c3f47f650c94ec05a151692652a868995d2938", + "reference": "58c3f47f650c94ec05a151692652a868995d2938", + "shasum": "" + }, + "require": { + "php": "^7|^8" + }, + "require-dev": { + "phpunit/phpunit": "^6|^7|^8|^9", + "vimeo/psalm": "^1|^2|^3|^4" + }, + "type": "library", + "autoload": { + "psr-4": { + "ParagonIE\\ConstantTime\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Paragon Initiative Enterprises", + "email": "security@paragonie.com", + "homepage": "https://paragonie.com", + "role": "Maintainer" + }, + { + "name": "Steve 'Sc00bz' Thomas", + "email": "steve@tobtu.com", + "homepage": "https://www.tobtu.com", + "role": "Original Developer" + } + ], + "description": "Constant-time Implementations of RFC 4648 Encoding (Base-64, Base-32, Base-16)", + "keywords": [ + "base16", + "base32", + "base32_decode", + "base32_encode", + "base64", + "base64_decode", + "base64_encode", + "bin2hex", + "encoding", + "hex", + "hex2bin", + "rfc4648" + ], + "support": { + "email": "info@paragonie.com", + "issues": "https://github.com/paragonie/constant_time_encoding/issues", + "source": "https://github.com/paragonie/constant_time_encoding" + }, + "time": "2022-06-14T06:56:20+00:00" + }, { "name": "phpmailer/phpmailer", "version": "v6.8.0", @@ -813,6 +952,81 @@ ], "time": "2023-03-06T14:43:22+00:00" }, + { + "name": "spomky-labs/otphp", + "version": "v10.0.3", + "source": { + "type": "git", + "url": "https://github.com/Spomky-Labs/otphp.git", + "reference": "9784d9f7c790eed26e102d6c78f12c754036c366" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Spomky-Labs/otphp/zipball/9784d9f7c790eed26e102d6c78f12c754036c366", + "reference": "9784d9f7c790eed26e102d6c78f12c754036c366", + "shasum": "" + }, + "require": { + "beberlei/assert": "^3.0", + "ext-mbstring": "*", + "paragonie/constant_time_encoding": "^2.0", + "php": "^7.2|^8.0", + "thecodingmachine/safe": "^0.1.14|^1.0|^2.0" + }, + "require-dev": { + "php-coveralls/php-coveralls": "^2.0", + "phpstan/phpstan": "^0.12", + "phpstan/phpstan-beberlei-assert": "^0.12", + "phpstan/phpstan-deprecation-rules": "^0.12", + "phpstan/phpstan-phpunit": "^0.12", + "phpstan/phpstan-strict-rules": "^0.12", + "phpunit/phpunit": "^8.0", + "thecodingmachine/phpstan-safe-rule": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "v10.0": "10.0.x-dev", + "v9.0": "9.0.x-dev", + "v8.3": "8.3.x-dev" + } + }, + "autoload": { + "psr-4": { + "OTPHP\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Florent Morselli", + "homepage": "https://github.com/Spomky" + }, + { + "name": "All contributors", + "homepage": "https://github.com/Spomky-Labs/otphp/contributors" + } + ], + "description": "A PHP library for generating one time passwords according to RFC 4226 (HOTP Algorithm) and the RFC 6238 (TOTP Algorithm) and compatible with Google Authenticator", + "homepage": "https://github.com/Spomky-Labs/otphp", + "keywords": [ + "FreeOTP", + "RFC 4226", + "RFC 6238", + "google authenticator", + "hotp", + "otp", + "totp" + ], + "support": { + "issues": "https://github.com/Spomky-Labs/otphp/issues", + "source": "https://github.com/Spomky-Labs/otphp/tree/v10.0.3" + }, + "time": "2022-03-17T08:00:35+00:00" + }, { "name": "symfony/polyfill-php80", "version": "v1.28.0", @@ -896,6 +1110,145 @@ ], "time": "2023-01-26T09:26:14+00:00" }, + { + "name": "thecodingmachine/safe", + "version": "v2.5.0", + "source": { + "type": "git", + "url": "https://github.com/thecodingmachine/safe.git", + "reference": "3115ecd6b4391662b4931daac4eba6b07a2ac1f0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thecodingmachine/safe/zipball/3115ecd6b4391662b4931daac4eba6b07a2ac1f0", + "reference": "3115ecd6b4391662b4931daac4eba6b07a2ac1f0", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpstan/phpstan": "^1.5", + "phpunit/phpunit": "^9.5", + "squizlabs/php_codesniffer": "^3.2", + "thecodingmachine/phpstan-strict-rules": "^1.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.2.x-dev" + } + }, + "autoload": { + "files": [ + "deprecated/apc.php", + "deprecated/array.php", + "deprecated/datetime.php", + "deprecated/libevent.php", + "deprecated/misc.php", + "deprecated/password.php", + "deprecated/mssql.php", + "deprecated/stats.php", + "deprecated/strings.php", + "lib/special_cases.php", + "deprecated/mysqli.php", + "generated/apache.php", + "generated/apcu.php", + "generated/array.php", + "generated/bzip2.php", + "generated/calendar.php", + "generated/classobj.php", + "generated/com.php", + "generated/cubrid.php", + "generated/curl.php", + "generated/datetime.php", + "generated/dir.php", + "generated/eio.php", + "generated/errorfunc.php", + "generated/exec.php", + "generated/fileinfo.php", + "generated/filesystem.php", + "generated/filter.php", + "generated/fpm.php", + "generated/ftp.php", + "generated/funchand.php", + "generated/gettext.php", + "generated/gmp.php", + "generated/gnupg.php", + "generated/hash.php", + "generated/ibase.php", + "generated/ibmDb2.php", + "generated/iconv.php", + "generated/image.php", + "generated/imap.php", + "generated/info.php", + "generated/inotify.php", + "generated/json.php", + "generated/ldap.php", + "generated/libxml.php", + "generated/lzf.php", + "generated/mailparse.php", + "generated/mbstring.php", + "generated/misc.php", + "generated/mysql.php", + "generated/network.php", + "generated/oci8.php", + "generated/opcache.php", + "generated/openssl.php", + "generated/outcontrol.php", + "generated/pcntl.php", + "generated/pcre.php", + "generated/pgsql.php", + "generated/posix.php", + "generated/ps.php", + "generated/pspell.php", + "generated/readline.php", + "generated/rpminfo.php", + "generated/rrd.php", + "generated/sem.php", + "generated/session.php", + "generated/shmop.php", + "generated/sockets.php", + "generated/sodium.php", + "generated/solr.php", + "generated/spl.php", + "generated/sqlsrv.php", + "generated/ssdeep.php", + "generated/ssh2.php", + "generated/stream.php", + "generated/strings.php", + "generated/swoole.php", + "generated/uodbc.php", + "generated/uopz.php", + "generated/url.php", + "generated/var.php", + "generated/xdiff.php", + "generated/xml.php", + "generated/xmlrpc.php", + "generated/yaml.php", + "generated/yaz.php", + "generated/zip.php", + "generated/zlib.php" + ], + "classmap": [ + "lib/DateTime.php", + "lib/DateTimeImmutable.php", + "lib/Exceptions/", + "deprecated/Exceptions/", + "generated/Exceptions/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHP core functions that throw exceptions instead of returning FALSE on error", + "support": { + "issues": "https://github.com/thecodingmachine/safe/issues", + "source": "https://github.com/thecodingmachine/safe/tree/v2.5.0" + }, + "time": "2023-04-05T11:54:14+00:00" + }, { "name": "utopia-php/abuse", "version": "0.33.0", @@ -1190,16 +1543,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.5", + "version": "0.45.6", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6" + "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6", - "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd", + "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd", "shasum": "" }, "require": { @@ -1240,22 +1593,22 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.5" + "source": "https://github.com/utopia-php/database/tree/0.45.6" }, - "time": "2024-01-08T17:08:15+00:00" + "time": "2024-02-01T02:33:43+00:00" }, { "name": "utopia-php/domains", - "version": "0.3.2", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb" + "reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/aaa8c9a96c69ccb397997b1f4f2299c66f77eefb", - "reference": "aaa8c9a96c69ccb397997b1f4f2299c66f77eefb", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/bf07f60326f8389f378ddf6fcde86217e5cfe18c", + "reference": "bf07f60326f8389f378ddf6fcde86217e5cfe18c", "shasum": "" }, "require": { @@ -1300,22 +1653,22 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/0.3.2" + "source": "https://github.com/utopia-php/domains/tree/0.5.0" }, - "time": "2023-07-19T16:39:24+00:00" + "time": "2024-01-03T22:04:27+00:00" }, { "name": "utopia-php/dsn", - "version": "0.1.0", + "version": "0.2.0", "source": { "type": "git", "url": "https://github.com/utopia-php/dsn.git", - "reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea" + "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/dsn/zipball/17a5935eab1b89fb4b95600db50a1b6d5faa6cea", - "reference": "17a5935eab1b89fb4b95600db50a1b6d5faa6cea", + "url": "https://api.github.com/repos/utopia-php/dsn/zipball/c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", + "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", "shasum": "" }, "require": { @@ -1347,22 +1700,22 @@ ], "support": { "issues": "https://github.com/utopia-php/dsn/issues", - "source": "https://github.com/utopia-php/dsn/tree/0.1.0" + "source": "https://github.com/utopia-php/dsn/tree/0.2.0" }, - "time": "2022-10-26T10:06:20+00:00" + "time": "2023-11-02T12:01:43+00:00" }, { "name": "utopia-php/framework", - "version": "0.33.0", + "version": "0.33.2", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "e3ff6b933082d57b48e7c4267bb605c0bf2250fd" + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/e3ff6b933082d57b48e7c4267bb605c0bf2250fd", - "reference": "e3ff6b933082d57b48e7c4267bb605c0bf2250fd", + "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", "shasum": "" }, "require": { @@ -1392,22 +1745,22 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.0" + "source": "https://github.com/utopia-php/http/tree/0.33.2" }, - "time": "2024-01-08T13:30:27+00:00" + "time": "2024-01-31T10:35:59+00:00" }, { "name": "utopia-php/image", - "version": "0.5.4", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/image.git", - "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336" + "reference": "88f7209172bdabd81e76ac981c95fac117dc6e08" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/image/zipball/ca5f436f9aa22dedaa6648f24f3687733808e336", - "reference": "ca5f436f9aa22dedaa6648f24f3687733808e336", + "url": "https://api.github.com/repos/utopia-php/image/zipball/88f7209172bdabd81e76ac981c95fac117dc6e08", + "reference": "88f7209172bdabd81e76ac981c95fac117dc6e08", "shasum": "" }, "require": { @@ -1415,6 +1768,8 @@ "php": ">=8.0" }, "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.13.1" }, @@ -1428,12 +1783,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple Image manipulation library", "keywords": [ "framework", @@ -1444,9 +1793,9 @@ ], "support": { "issues": "https://github.com/utopia-php/image/issues", - "source": "https://github.com/utopia-php/image/tree/0.5.4" + "source": "https://github.com/utopia-php/image/tree/0.6.0" }, - "time": "2022-05-11T12:30:41+00:00" + "time": "2024-01-24T06:59:44+00:00" }, { "name": "utopia-php/locale", @@ -1554,26 +1903,28 @@ }, { "name": "utopia-php/messaging", - "version": "0.2.0", + "version": "0.9.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "2d0f474a106bb1da285f85e105c29b46085d3a43" + "reference": "df54ba51570e886724590edeb03dbd455bb0464d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/2d0f474a106bb1da285f85e105c29b46085d3a43", - "reference": "2d0f474a106bb1da285f85e105c29b46085d3a43", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/df54ba51570e886724590edeb03dbd455bb0464d", + "reference": "df54ba51570e886724590edeb03dbd455bb0464d", "shasum": "" }, "require": { "ext-curl": "*", + "ext-openssl": "*", "php": ">=8.0.0" }, "require-dev": { - "laravel/pint": "^1.2", + "laravel/pint": "1.13.*", "phpmailer/phpmailer": "6.8.*", - "phpunit/phpunit": "9.6.*" + "phpstan/phpstan": "1.10.*", + "phpunit/phpunit": "9.6.10" }, "type": "library", "autoload": { @@ -1596,9 +1947,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.2.0" + "source": "https://github.com/utopia-php/messaging/tree/0.9.0" }, - "time": "2023-09-14T20:48:42+00:00" + "time": "2024-01-31T11:51:27+00:00" }, { "name": "utopia-php/migration", @@ -2420,16 +2771,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.36.0", + "version": "0.36.2", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a" + "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3a10f1f895ed71120442ff71eb6adec3fd6b4e8a", - "reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0aa67479d75f0e0cb7b60454031534d7f0abaece", + "reference": "0aa67479d75f0e0cb7b60454031534d7f0abaece", "shasum": "" }, "require": { @@ -2465,22 +2816,22 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.36.0" + "source": "https://github.com/appwrite/sdk-generator/tree/0.36.2" }, - "time": "2023-11-20T10:03:06+00:00" + "time": "2024-01-19T01:04:35+00:00" }, { "name": "doctrine/deprecations", - "version": "1.1.2", + "version": "1.1.3", "source": { "type": "git", "url": "https://github.com/doctrine/deprecations.git", - "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931" + "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931", - "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab", + "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab", "shasum": "" }, "require": { @@ -2512,9 +2863,9 @@ "homepage": "https://www.doctrine-project.org/", "support": { "issues": "https://github.com/doctrine/deprecations/issues", - "source": "https://github.com/doctrine/deprecations/tree/1.1.2" + "source": "https://github.com/doctrine/deprecations/tree/1.1.3" }, - "time": "2023-09-27T20:04:15+00:00" + "time": "2024-01-30T19:34:25+00:00" }, { "name": "doctrine/instantiator", @@ -5105,6 +5456,45 @@ } ], "time": "2023-11-21T18:54:41+00:00" + }, + { + "name": "utopia-php/fetch", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, + "time": "2023-10-10T11:58:32+00:00" } ], "aliases": [], @@ -5131,7 +5521,7 @@ "ext-fileinfo": "*" }, "platform-overrides": { - "php": "8.0" + "php": "8.2" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 2133de0dce..c7080cf2b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 74b8f4c224..32f636a0f6 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -73,16 +73,6 @@ class Usage extends Action ); } - foreach ($payload['metrics'] ?? [] as $metric) { - if ($metric['key'] === 'users') { - if ($metric['value'] < 0) { - $this->stats[$metric['key']]['negative'] += $metric['value']; - } else { - $this->stats[$metric['key']]['positive'] += $metric['value']; - } - } - } - $this->stats[$projectId]['project'] = $project; foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; From 93a99b561c68394921470832ac043066b6365d50 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 7 Feb 2024 11:54:18 +0200 Subject: [PATCH 06/23] remove debug lines --- src/Appwrite/Event/Usage.php | 8 -------- src/Appwrite/Platform/Workers/UsageDump.php | 5 ----- 2 files changed, 13 deletions(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 0774534e36..ded276e166 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -43,14 +43,6 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { - //Todo debug (to be removed later @shimon) -// if ($key === 'users') { -// if ($value < 0) { -// console::log('@negative=' . $value); -// } else { -// console::log('@positive=' . $value); -// } -// } $this->metrics[] = [ 'key' => $key, 'value' => $value, diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 5221d58b2c..24fe4ccccd 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -86,11 +86,6 @@ class UsageDump extends Action ])); } catch (Duplicate $th) { if ($value < 0) { - //Todo debug (to be removed later @shimon) -// var_dump([ -// 'id' => $time . '_' . $period . '_' . $key, -// 'value' => $value, -// ]); $dbForProject->decreaseDocumentAttribute( 'stats_v2', $id, From 990698e4239d56cb97997358d0ecead8ee0c0be0 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 09:32:05 +0200 Subject: [PATCH 07/23] composer update --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index e72cc04a15..4d5535b8f8 100644 --- a/composer.lock +++ b/composer.lock @@ -5124,5 +5124,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } From 790cdc297b1663cd79c873e7324e18b705de55dd Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 10:26:45 +0200 Subject: [PATCH 08/23] composer update --- .env | 2 +- app/controllers/api/users.php | 17 ++++++++--------- docker-compose.yml | 1 - src/Appwrite/Platform/Workers/Usage.php | 9 +++++---- src/Appwrite/Platform/Workers/UsageDump.php | 1 + 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.env b/.env index 0247356a25..3efd72e2a2 100644 --- a/.env +++ b/.env @@ -78,7 +78,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=60000 +_APP_USAGE_AGGREGATION_INTERVAL=60 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7449ac27c0..5ce2263f47 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1189,16 +1189,15 @@ App::delete('/v1/users/:userId') // clone user object to send to workers $clone = clone $user; - $affected = $dbForProject->deleteDocument('users', $userId); - if (!empty($affected)) { - $queueForDeletes - ->setType(DELETE_TYPE_DOCUMENT) - ->setDocument($clone); + $dbForProject->deleteDocument('users', $userId); - $queueForEvents - ->setParam('userId', $user->getId()) - ->setPayload($response->output($clone, Response::MODEL_USER)); - } + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($clone); + + $queueForEvents + ->setParam('userId', $user->getId()) + ->setPayload($response->output($clone, Response::MODEL_USER)); $response->noContent(); }); diff --git a/docker-compose.yml b/docker-compose.yml index 2106ff8593..de71e3937a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,6 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - mariadb - redis diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 32f636a0f6..859fa6f97e 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Exception; +use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Document; use Utopia\Platform\Action; @@ -15,9 +16,7 @@ class Usage extends Action private int $lastTriggeredTime = 0; private int $keys = 0; private const INFINITY_PERIOD = '_inf_'; - private const KEYS_THRESHOLD = 5; - private const KEYS_SENT_THRESHOLD = 60; - + private const KEYS_THRESHOLD = 10000; public static function getName(): string @@ -57,7 +56,9 @@ class Usage extends Action if (empty($payload)) { throw new Exception('Missing payload'); } + //Todo Figure out way to preserve keys when the container is being recreated @shimonewman + $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -87,7 +88,7 @@ class Usage extends Action // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || - (time() - $this->lastTriggeredTime > self::KEYS_SENT_THRESHOLD && $this->keys > 0) + (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { $queueForUsageDump ->setStats($this->stats) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 24fe4ccccd..948b3fa681 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -55,6 +55,7 @@ class UsageDump extends Action throw new Exception('Missing payload'); } + //Todo rename both usage workers @shimonewman foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; From f42a78ef9ca876369b8b6682ea0aec5c9c09922a Mon Sep 17 00:00:00 2001 From: Souptik Datta Date: Fri, 9 Feb 2024 00:14:37 +0530 Subject: [PATCH 09/23] fix: Fix account activity logs total count Signed-off-by: Souptik Datta --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index be6ed68e05..3d7fc4c5d8 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1790,7 +1790,7 @@ App::get('/v1/account/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getId()), + 'total' => $audit->countLogsByUser($user->getInternalId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); From 8d7be399d34bf00b1c85575f6371ed1581c03b5e Mon Sep 17 00:00:00 2001 From: Souptik Datta Date: Fri, 9 Feb 2024 00:15:48 +0530 Subject: [PATCH 10/23] fix: Fix project user activity log's missing fields and total count Signed-off-by: Souptik Datta --- app/controllers/api/users.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 38d65fba7e..0765b25fb0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -615,6 +615,9 @@ App::get('/v1/users/:userId/logs') $output[$i] = new Document([ 'event' => $log['event'], + 'userId' => ID::custom($log['data']['userId']), + 'userEmail' => $log['data']['userEmail'] ?? null, + 'userName' => $log['data']['userName'] ?? null, 'ip' => $log['ip'], 'time' => $log['time'], 'osCode' => $os['osCode'], @@ -643,7 +646,7 @@ App::get('/v1/users/:userId/logs') } $response->dynamic(new Document([ - 'total' => $audit->countLogsByUser($user->getId()), + 'total' => $audit->countLogsByUser($user->getInternalId()), 'logs' => $output, ]), Response::MODEL_LOG_LIST); }); From 72ee7d7f343485cdb27cb7229e2353a9dc17de9d Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 20:51:54 +0200 Subject: [PATCH 11/23] interval update --- .env | 2 +- src/Appwrite/Platform/Workers/Usage.php | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 3efd72e2a2..b915f91516 100644 --- a/.env +++ b/.env @@ -78,7 +78,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_AGGREGATION_INTERVAL=60 +_APP_USAGE_AGGREGATION_INTERVAL=20 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400 _APP_USAGE_STATS=enabled diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 859fa6f97e..78efca3585 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers; use Exception; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Platform\Action; use Appwrite\Event\UsageDump; @@ -58,7 +59,7 @@ class Usage extends Action } //Todo Figure out way to preserve keys when the container is being recreated @shimonewman - $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60'); + $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20'); $project = new Document($payload['project'] ?? []); $projectId = $project->getInternalId(); foreach ($payload['reduce'] ?? [] as $document) { @@ -84,12 +85,12 @@ class Usage extends Action $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; } - // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { + var_dump(DateTime::now()); $queueForUsageDump ->setStats($this->stats) ->trigger(); From a1539395c636ab99df81ec71712e86e4b93b9a5d Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 21:08:29 +0200 Subject: [PATCH 12/23] interval update --- src/Appwrite/Platform/Workers/Usage.php | 3 ++- src/Appwrite/Platform/Workers/UsageDump.php | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 78efca3585..589e775e97 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -90,7 +90,8 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { - var_dump(DateTime::now()); + console::warning('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); + $queueForUsageDump ->setStats($this->stats) ->trigger(); diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 948b3fa681..ce221a2f62 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -59,12 +59,13 @@ class UsageDump extends Action foreach ($payload['stats'] ?? [] as $stats) { $project = new Document($stats['project'] ?? []); $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; - $projectInternalId = $project->getInternalId(); if ($numberOfKeys === 0) { continue; } + console::warning('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + try { $dbForProject = $getProjectDB($project); foreach ($stats['keys'] ?? [] as $key => $value) { @@ -105,7 +106,7 @@ class UsageDump extends Action } } } catch (\Exception $e) { - console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage()); + console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage()); } } } From c1fb4ecf39d01dfd30f5376f141fc1c0648e62c6 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 8 Feb 2024 21:41:23 +0200 Subject: [PATCH 13/23] interval update --- src/Appwrite/Platform/Workers/Usage.php | 2 +- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 589e775e97..8a9c322c80 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -90,7 +90,7 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { - console::warning('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); + console::log('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); $queueForUsageDump ->setStats($this->stats) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index ce221a2f62..a45cd40995 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -64,7 +64,7 @@ class UsageDump extends Action continue; } - console::warning('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + console::log('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); try { $dbForProject = $getProjectDB($project); From 3788d5bcebc3eb7709d9bb4f3de6d6f500e5bd70 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 9 Feb 2024 12:56:37 +0200 Subject: [PATCH 14/23] interval update --- src/Appwrite/Platform/Workers/Usage.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 8a9c322c80..8490c31c0a 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -85,6 +85,10 @@ class Usage extends Action $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; } + var_dump(time() - $this->lastTriggeredTime > $aggregationInterval); + var_dump(time() - $this->lastTriggeredTime); + var_dump($aggregationInterval); + var_dump($this->keys); // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || From d0985db27dc00de6b953ac96e8d6c6a0e24abc30 Mon Sep 17 00:00:00 2001 From: shimon Date: Fri, 9 Feb 2024 13:22:48 +0200 Subject: [PATCH 15/23] interval update --- src/Appwrite/Platform/Workers/Usage.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 8490c31c0a..d65ab0968d 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -85,10 +85,7 @@ class Usage extends Action $this->stats[$projectId]['keys'][$metric['key']] += $metric['value']; } - var_dump(time() - $this->lastTriggeredTime > $aggregationInterval); - var_dump(time() - $this->lastTriggeredTime); - var_dump($aggregationInterval); - var_dump($this->keys); + // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats) if ( $this->keys >= self::KEYS_THRESHOLD || From 5abcb86fd7a3fa50bcda8365b5f733cee108f3bf Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 9 Feb 2024 23:18:18 +0530 Subject: [PATCH 16/23] fix: use atomic operations for updating team count --- add-ssh.sh | 19 +++++++++++++++++++ app/controllers/api/teams.php | 5 ++--- 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100755 add-ssh.sh diff --git a/add-ssh.sh b/add-ssh.sh new file mode 100755 index 0000000000..b576d3d37a --- /dev/null +++ b/add-ssh.sh @@ -0,0 +1,19 @@ + + +#!/bin/bash + +HOSTS=("178.128.204.91 ","164.92.166.207 ","68.183.65.132 ","157.230.18.191 ","68.183.76.206 ","165.227.130.23 ","161.35.212.211 ","64.226.126.100 ","167.172.97.253 ","167.172.102.140 ","167.172.106.74 ","138.68.126.117 ","161.35.192.225 ","138.68.100.209 ","157.245.27.194 ","165.227.156.19 ","67.207.78.138 ","46.101.107.220 ","161.35.70.96 ","167.172.111.219"," 157.230.79.70"," 207.154.241.8 ","165.22.19.119 ","164.90.232.240 ","167.71.33.99 ","64.227.125.178 ","134.122.89.202 ","161.35.18.34 ","206.189.57.113 ","206.189.59.102 ","134.122.74.51 ","165.232.72.163 ","164.90.184.157 ","165.232.72.170 ","165.232.72.165 ","164.90.176.216 ","165.232.72.169 ","167.71.45.112 ","164.90.163.212") + +PUB_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOJpG0jmkAgYLij/BBkTN6HYp4kmH1kicNukcSulEhZV4+R/2ihdTyZgC4OxuHel/5HgWxDQ/E5U2/IgayFtF/68RwGWLdMusaCQbh/JTiYQgCrGW07N+GKFqKgUx2ZNSuPgrHrkXBQOZWGVPieFZ8Z85eGyR+5yNagA093+bgESb/VChuIaa1m3Y7su7f1D8tECR5He8uYKnGmnLKbqcA7riT6xN1a23QdZC1v7j5D8eI1BWPoZ17EP3/DqsQ32qWxkzxU0x5nQhZrhd4Mf4u06mm48ctg/CprMJUzZxG7sZjcmGOg80Vot1oFpw/DCrtjmeIJiYrpT5J1q2kZpOOPafJpi2sogoe5h+ocfQ/QFC5dItihx+fK9e528rKPVZ6j/XTcBg0gGkVvWLRchxg1ZdFSJAkE+c6vcR+N+PxwLoVpe/oL3Yy+h7Gqctor8ac28UWeU2boLlQu+emeFFB+A7FfI+KvvNw3oEvoAuCdVacvCr1KYflwvP0YEIL2osOS27eJwFnrB3zEegL2T5AjqBRtI5HHS8rD/1CfTZV9SymHtp/yFUJCZu93FbWSSr/yLabD/Tor6p3DY6JEs29KQ93oSw2pcTlFo6AWY0/olNTyTcUT7KGI43YuHBKmq6ApQvc9OiLHC01uFo/fIF7qofxqjk9Bz9AaBIZmSk4Bw== shimonewman@gmail.com" + +for HOST in "${HOSTS[@]}"; do + echo "Adding key to $HOST" + echo "$PUB_KEY" | ssh "root@$HOST" 'cat >> ~/.ssh/authorized_keys' +done + +echo "Done." + + + + + diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index a374c57cf1..2801e45bb5 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -967,7 +967,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status') $dbForProject->deleteCachedDocument('users', $user->getId()); - $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team->setAttribute('total', $team->getAttribute('total', 0) + 1))); + Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1)); $queueForEvents ->setParam('teamId', $team->getId()) @@ -1047,8 +1047,7 @@ App::delete('/v1/teams/:teamId/memberships/:membershipId') $dbForProject->deleteCachedDocument('users', $user->getId()); if ($membership->getAttribute('confirm')) { // Count only confirmed members - $team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0)); - Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team)); + Authorization::skip(fn() => $dbForProject->decreaseDocumentAttribute('teams', $team->getId(), 'total', 1, 0)); } $queueForEvents From fb68dcc89ec8d1af069610e376f2ae39f1a2bbd2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 9 Feb 2024 23:20:37 +0530 Subject: [PATCH 17/23] fix: use atomic operations for updating team count --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 4d5535b8f8..ca39c7d084 100644 --- a/composer.lock +++ b/composer.lock @@ -5124,5 +5124,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } From f3e8994bb83233683a99156926806f2af14a3051 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 9 Feb 2024 23:21:17 +0530 Subject: [PATCH 18/23] fix: use atomic operations for updating team count --- add-ssh.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100755 add-ssh.sh diff --git a/add-ssh.sh b/add-ssh.sh deleted file mode 100755 index b576d3d37a..0000000000 --- a/add-ssh.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -#!/bin/bash - -HOSTS=("178.128.204.91 ","164.92.166.207 ","68.183.65.132 ","157.230.18.191 ","68.183.76.206 ","165.227.130.23 ","161.35.212.211 ","64.226.126.100 ","167.172.97.253 ","167.172.102.140 ","167.172.106.74 ","138.68.126.117 ","161.35.192.225 ","138.68.100.209 ","157.245.27.194 ","165.227.156.19 ","67.207.78.138 ","46.101.107.220 ","161.35.70.96 ","167.172.111.219"," 157.230.79.70"," 207.154.241.8 ","165.22.19.119 ","164.90.232.240 ","167.71.33.99 ","64.227.125.178 ","134.122.89.202 ","161.35.18.34 ","206.189.57.113 ","206.189.59.102 ","134.122.74.51 ","165.232.72.163 ","164.90.184.157 ","165.232.72.170 ","165.232.72.165 ","164.90.176.216 ","165.232.72.169 ","167.71.45.112 ","164.90.163.212") - -PUB_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDOJpG0jmkAgYLij/BBkTN6HYp4kmH1kicNukcSulEhZV4+R/2ihdTyZgC4OxuHel/5HgWxDQ/E5U2/IgayFtF/68RwGWLdMusaCQbh/JTiYQgCrGW07N+GKFqKgUx2ZNSuPgrHrkXBQOZWGVPieFZ8Z85eGyR+5yNagA093+bgESb/VChuIaa1m3Y7su7f1D8tECR5He8uYKnGmnLKbqcA7riT6xN1a23QdZC1v7j5D8eI1BWPoZ17EP3/DqsQ32qWxkzxU0x5nQhZrhd4Mf4u06mm48ctg/CprMJUzZxG7sZjcmGOg80Vot1oFpw/DCrtjmeIJiYrpT5J1q2kZpOOPafJpi2sogoe5h+ocfQ/QFC5dItihx+fK9e528rKPVZ6j/XTcBg0gGkVvWLRchxg1ZdFSJAkE+c6vcR+N+PxwLoVpe/oL3Yy+h7Gqctor8ac28UWeU2boLlQu+emeFFB+A7FfI+KvvNw3oEvoAuCdVacvCr1KYflwvP0YEIL2osOS27eJwFnrB3zEegL2T5AjqBRtI5HHS8rD/1CfTZV9SymHtp/yFUJCZu93FbWSSr/yLabD/Tor6p3DY6JEs29KQ93oSw2pcTlFo6AWY0/olNTyTcUT7KGI43YuHBKmq6ApQvc9OiLHC01uFo/fIF7qofxqjk9Bz9AaBIZmSk4Bw== shimonewman@gmail.com" - -for HOST in "${HOSTS[@]}"; do - echo "Adding key to $HOST" - echo "$PUB_KEY" | ssh "root@$HOST" 'cat >> ~/.ssh/authorized_keys' -done - -echo "Done." - - - - - From b09386727b375ece173e5df334c792459ac1158d Mon Sep 17 00:00:00 2001 From: shimon Date: Sat, 10 Feb 2024 10:34:44 +0200 Subject: [PATCH 19/23] Usage logs text update --- src/Appwrite/Platform/Workers/Usage.php | 2 +- src/Appwrite/Platform/Workers/UsageDump.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index d65ab0968d..49f3344906 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -91,7 +91,7 @@ class Usage extends Action $this->keys >= self::KEYS_THRESHOLD || (time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0) ) { - console::log('[' . DateTime::now() . '] stats aggregation sent to worker with ' . $this->keys . ' keys'); + console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys'); $queueForUsageDump ->setStats($this->stats) diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index a45cd40995..f563578984 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -64,7 +64,7 @@ class UsageDump extends Action continue; } - console::log('[' . DateTime::now() . '] aggregation received project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); + console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys'); try { $dbForProject = $getProjectDB($project); From e462d73f65e7f1bb14924b9d5bcd83d6d8b3ca21 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Feb 2024 09:31:09 +0000 Subject: [PATCH 20/23] prevent user with active team, from deletion --- app/config/errors.php | 5 ++ app/controllers/api/account.php | 19 ++++- src/Appwrite/Extend/Exception.php | 5 +- .../Account/AccountConsoleClientTest.php | 77 ++++++++++++++++++- 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index b95f27915d..7dedc373ef 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -251,6 +251,11 @@ return [ 'description' => 'User phone is already verified', 'code' => 409 ], + Exception::USER_DELETION_PROHIBITED => [ + 'name' => Exception::USER_DELETION_PROHIBITED, + 'description' => 'User deletion is not allowed for users with active memberships. Please delete all confirmed memberships before deleting the account.', + 'code' => 400 + ], /** Teams */ Exception::TEAM_NOT_FOUND => [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index f1575a08d7..f340823993 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3033,12 +3033,27 @@ App::delete('/v1/account') ->label('sdk.description', '/docs/references/account/delete.md') ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) ->label('sdk.response.model', Response::MODEL_NONE) - ->inject('user') + ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') ->inject('queueForDeletes') - ->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { + ->action(function (Document $user, Document $project, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) { + if ($user->isEmpty()) { + throw new Exception(Exception::USER_NOT_FOUND); + } + + if ($project->getId() === 'console') { + // get all memberships + $memberships = $user->getAttribute('memberships', []); + foreach ($memberships as $membership) { + // prevent deletion if at least one active membership + if ($membership->getAttribute('confirm', false)) { + throw new Exception(Exception::USER_DELETION_PROHIBITED); + } + } + } + if ($user->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND); } diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 43095334ad..3cedbf11c7 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -84,8 +84,9 @@ class Exception extends \Exception public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request'; public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized'; public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; - public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified'; - public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; + public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified'; + public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; + public const USER_DELETION_PROHIBITED = 'user_deletion_prohibited'; /** Teams */ public const TEAM_NOT_FOUND = 'team_not_found'; diff --git a/tests/e2e/Services/Account/AccountConsoleClientTest.php b/tests/e2e/Services/Account/AccountConsoleClientTest.php index 3d4445dcc9..e3de522541 100644 --- a/tests/e2e/Services/Account/AccountConsoleClientTest.php +++ b/tests/e2e/Services/Account/AccountConsoleClientTest.php @@ -2,17 +2,90 @@ namespace Tests\E2E\Services\Account; -use Appwrite\Extend\Exception; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; use Utopia\Database\Helpers\ID; use Tests\E2E\Client; -use Utopia\Database\Validator\Datetime as DatetimeValidator; class AccountConsoleClientTest extends Scope { use AccountBase; use ProjectConsole; use SideClient; + + public function testDeleteAccount(): void + { + $email = uniqid() . 'user@localhost.test'; + $password = 'password'; + $name = 'User Name'; + + $response = $this->client->call(Client::METHOD_POST, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'userId' => ID::unique(), + 'email' => $email, + 'password' => $password, + 'name' => $name, + ]); + + $this->assertEquals($response['headers']['status-code'], 201); + + $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ]), [ + 'email' => $email, + 'password' => $password, + ]); + + $this->assertEquals($response['headers']['status-code'], 201); + + $session = $response['cookies']['a_session_' . $this->getProject()['$id']]; + + // create team + $team = $this->client->call(Client::METHOD_POST, '/teams', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ], [ + 'teamId' => 'unique()', + 'name' => 'myteam' + ]); + $this->assertEquals($team['headers']['status-code'], 201); + + $teamId = $team['body']['$id']; + + $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals($response['headers']['status-code'], 400); + + // DELETE TEAM + $response = $this->client->call(Client::METHOD_DELETE, '/teams/' . $teamId, array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + $this->assertEquals($response['headers']['status-code'], 204); + sleep(2); + + $response = $this->client->call(Client::METHOD_DELETE, '/account', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ])); + + $this->assertEquals($response['headers']['status-code'], 204); + } } From 2634c928c7ec1f4f037996e8ec6035c1cb572bff Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Feb 2024 09:34:13 +0000 Subject: [PATCH 21/23] fix format --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index f340823993..a687dbea08 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3053,7 +3053,7 @@ App::delete('/v1/account') } } } - + if ($user->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND); } From 36eabfa2ddaf492fd3ac7fb033bc555ba40b49b7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Feb 2024 09:45:36 +0000 Subject: [PATCH 22/23] fix check twice --- app/controllers/api/account.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index a687dbea08..2299b3eea1 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3054,10 +3054,6 @@ App::delete('/v1/account') } } - if ($user->isEmpty()) { - throw new Exception(Exception::USER_NOT_FOUND); - } - $dbForProject->deleteDocument('users', $user->getId()); $queueForDeletes From eb7e2fd23bc19be5c728a828793ebf4c9993b3d9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 11 Feb 2024 09:46:39 +0000 Subject: [PATCH 23/23] fix injection --- app/controllers/api/account.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 2299b3eea1..68d2a544d8 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -3033,6 +3033,7 @@ App::delete('/v1/account') ->label('sdk.description', '/docs/references/account/delete.md') ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) ->label('sdk.response.model', Response::MODEL_NONE) + ->inject('user') ->inject('project') ->inject('response') ->inject('dbForProject')