From ff9bccf1ff621e66fd40f9072203ff0c43d4c01a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 20 Dec 2022 18:26:56 +0530 Subject: [PATCH 01/27] Revert "Feat: Build output size" --- CHANGES.md | 1 - app/config/collections.php | 24 ++++++------ app/controllers/api/functions.php | 2 +- app/workers/builds.php | 15 +++---- app/workers/deletes.php | 12 +++--- app/workers/functions.php | 2 +- docker-compose.yml | 2 +- src/Appwrite/Migration/Version/V17.php | 41 +------------------- src/Appwrite/Usage/Stats.php | 2 - src/Appwrite/Utopia/Response/Model/Build.php | 12 +++--- 10 files changed, 36 insertions(+), 77 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6a51a738e3..7a553d5d5e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,6 @@ - Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673) - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776) - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780) -- Store build output file size [#4844](https://github.com/appwrite/appwrite/pull/4844) - Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814) # Version 1.1.2 diff --git a/app/config/collections.php b/app/config/collections.php index a8c7a46b18..afde813fa1 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2561,6 +2561,17 @@ $collections = [ 'array' => false, 'filters' => ['datetime'], ], + [ + '$id' => ID::custom('endTime'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], [ '$id' => ID::custom('duration'), 'type' => Database::VAR_INTEGER, @@ -2606,7 +2617,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('path'), + '$id' => ID::custom('outputPath'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 2048, @@ -2616,17 +2627,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('size'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('stderr'), 'type' => Database::VAR_STRING, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 4c84795712..6fe7c67e69 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1197,7 +1197,7 @@ App::post('/v1/functions/:functionId/executions') variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/app/workers/builds.php b/app/workers/builds.php index 081537fdfb..d26f07ab75 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -99,13 +99,13 @@ class BuildsV1 extends Worker 'startTime' => $startTime, 'deploymentId' => $deployment->getId(), 'status' => 'processing', - 'path' => '', - 'size' => 0, + 'outputPath' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), 'sourceType' => $device, 'stdout' => '', 'stderr' => '', + 'endTime' => null, 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); @@ -186,12 +186,14 @@ class BuildsV1 extends Worker ] ); + $endTime = new \DateTime(); + $endTime->setTimestamp($response['endTimeUnix']); + /** Update the build document */ - $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); + $build->setAttribute('endTime', DateTime::format($endTime)); $build->setAttribute('duration', \intval($response['duration'])); $build->setAttribute('status', $response['status']); - $build->setAttribute('path', $response['path']); - $build->setAttribute('size', $response['size']); + $build->setAttribute('outputPath', $response['outputPath']); $build->setAttribute('stderr', $response['stderr']); $build->setAttribute('stdout', $response['stdout']); @@ -222,7 +224,7 @@ class BuildsV1 extends Worker } catch (\Throwable $th) { $endTime = DateTime::now(); $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime)); - + $build->setAttribute('endTime', $endTime); $build->setAttribute('duration', $interval->format('%s') + 0); $build->setAttribute('status', 'failed'); $build->setAttribute('stderr', $th->getMessage()); @@ -258,7 +260,6 @@ class BuildsV1 extends Worker ->setParam('builds.{scope}.compute', 1) ->setParam('buildStatus', $build->getAttribute('status', '')) ->setParam('buildTime', $build->getAttribute('duration')) - ->setParam('buildSize', $build->getAttribute('size')) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->submit(); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 6bf29cc3ff..458b341a45 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -486,10 +486,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('path', '')); + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); } }); } @@ -535,10 +535,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds) { - if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('path', '')); + if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); } }); diff --git a/app/workers/functions.php b/app/workers/functions.php index 2e8dccc7aa..31e64a2bb4 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -133,7 +133,7 @@ Server::setResource('execute', function () { variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $build->getAttribute('outputPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), ); diff --git a/docker-compose.yml b/docker-compose.yml index a7c1ddf97c..feac2c89d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -658,7 +658,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.2.0 + image: openruntimes/executor:0.1.6 networks: - appwrite - runtimes diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index e6877e7225..66a02662d1 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -56,48 +56,9 @@ class V17 extends Migration } catch (\Throwable $th) { Console::warning("'mimeType' from {$id}: {$th->getMessage()}"); } + break; - case 'builds': - try { - /** - * Create 'size' attribute - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'size'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'size' from {$id}: {$th->getMessage()}"); - } - try { - /** - * Delete 'endTime' attribute (use startTime+duration if needed) - */ - $this->projectDB->deleteAttribute($id, 'endTime'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'endTime' from {$id}: {$th->getMessage()}"); - } - - try { - /** - * Rename 'outputPath' to 'path' - */ - $this->projectDB->renameAttribute($id, 'outputPath', 'path'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'path' from {$id}: {$th->getMessage()}"); - } - - try { - /** - * Create 'size' - */ - $this->createAttributeFromCollection($this->projectDB, $id, 'size'); - $this->projectDB->deleteCachedCollection($id); - } catch (\Throwable $th) { - Console::warning("'size' from {$id}: {$th->getMessage()}"); - } - break; default: break; } diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 38fdc123cf..e6e0056664 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -184,7 +184,6 @@ class Stats $functionBuild = $this->params['builds.{scope}.compute'] ?? 0; $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms - $functionBuildSize = ($this->params['buildSize'] ?? 0); // bytes $functionBuildStatus = $this->params['buildStatus'] ?? ''; $functionCompute = $functionExecutionTime + $functionBuildTime; $functionTags = $tags . ',functionId=' . $functionId; @@ -208,7 +207,6 @@ class Stats if ($functionBuild >= 1) { $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus); $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime); - $this->statsd->count('builds.{scope}.storage.size' . $functionTags, $functionBuildSize); } if ($functionBuild + $functionExecution >= 1) { $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute); diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index 4d7a6cd27c..b76f0ee083 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -51,18 +51,18 @@ class Build extends Model 'default' => '', 'example' => self::TYPE_DATETIME_EXAMPLE, ]) + ->addRule('endTime', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'The time the build was finished in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) ->addRule('duration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The build duration in seconds.', 'default' => 0, 'example' => 0, ]) - ->addRule('size', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The code size in bytes.', - 'default' => 0, - 'example' => 128, - ]) ; } From 7ea8d761e475603a97d9b7340c53586ae3a6e91c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 21 Dec 2022 02:08:37 +0530 Subject: [PATCH 02/27] feat: aggregate stats --- Dockerfile | 1 + bin/stat | 3 + src/Appwrite/Platform/Services/Tasks.php | 2 + src/Appwrite/Platform/Tasks/Stat.php | 212 +++++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 bin/stat create mode 100644 src/Appwrite/Platform/Tasks/Stat.php diff --git a/Dockerfile b/Dockerfile index 54aa327769..9a8e35f730 100755 --- a/Dockerfile +++ b/Dockerfile @@ -314,6 +314,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ + chmod +x /usr/local/bin/stat && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/vars && \ chmod +x /usr/local/bin/worker-audits && \ diff --git a/bin/stat b/bin/stat new file mode 100644 index 0000000000..d7030a0958 --- /dev/null +++ b/bin/stat @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php stat $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2e15cd015c..2cb0878589 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,6 +12,7 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; +use Appwrite\Platform\Tasks\Stat; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -27,6 +28,7 @@ class Tasks extends Service ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) + ->addAction(Stat::getName(), new Stat()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) diff --git a/src/Appwrite/Platform/Tasks/Stat.php b/src/Appwrite/Platform/Tasks/Stat.php new file mode 100644 index 0000000000..8ac5687bae --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Stat.php @@ -0,0 +1,212 @@ +desc('Get stats for project') + ->callback(fn () => $this->action()); + } + + function getConnection(string $dsn): PDO + { + if (empty($dsn)) { + throw new Exception("Missing value for DSN connection"); + } + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + $connection = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + + return $connection; + } + + + function getStats(Database $dbForProject): array + { + $range = '90d'; + $periods = [ + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = [ + 'files.$all.count.total', + 'buckets.$all.count.total', + 'databases.$all.count.total', + 'documents.$all.count.total', + 'collections.$all.count.total', + 'project.$all.storage.size', + 'project.$all.network.requests', + 'project.$all.network.bandwidth', + 'users.$all.count.total', + 'sessions.$all.requests.create', + 'executions.$all.compute.total', + ]; + + $stats = []; + + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + // Calculate aggregate of each metric + $stats[$metric . '.sum'] = array_sum(array_column($stats[$metric], 'value')); + } + }); + + // return only the ahhggregate values + return array_filter($stats, fn ($key) => strpos($key, '.sum') !== false, ARRAY_FILTER_USE_KEY); + } + + + public function action(): void + { + Console::success('Getting stats...'); + + $databases = [ + 'console' => [ + 'type' => 'database', + 'dsns' => '', + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'projects' => [ + 'type' => 'database', + 'dsns' => '', + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + ]; + + $dsns = explode(',', $databases['projects']['dsns']); + $projectdsns = []; + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = 'database' . '_' . $dsn[0]; + $dsn = $dsn[1] ?? ''; + $projectdsns[$name] = $dsn; + } + + $cache = new Cache(new None()); + $consoledsn = explode('=', $databases['console']['dsns']); + $consoledsn = $consoledsn[1] ?? ''; + $adapter = new MySQL($this->getConnection($consoledsn)); + $dbForConsole = new Database($adapter, $cache); + $dbForConsole->setDefaultDatabase('appwrite'); + $dbForConsole->setNamespace('console'); + + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Iterating through : {$totalProjects} projects"); + + $app = new App('UTC'); + $console = $app->getResource('console'); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + + $stats = []; + + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + Console::info("Getting stats for {$project->getId()}"); + + try { + // TODO: Iterate through all project DBs + $db = $project->getAttribute('database'); + $dsn = $projectdsns[$db] ?? ''; + $cache = new Cache(new None()); + $adapter = new MySQL($this->getConnection($dsn)); + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + $statsPerProject = $this->getStats($dbForProject); + + foreach ($statsPerProject as $key => $value) { + $stats[$key] = isset($stats[$key]) ? $stats[$key] + $value : $value; + } + + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } + + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); + } + + var_dump($stats); + } +} From 4142ffdb708257f81048e57fae64f0c82904dd75 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Jan 2023 17:26:41 +0530 Subject: [PATCH 03/27] feat: update account codes --- app/config/errors.php | 9 +++++++-- app/console | 2 +- app/controllers/api/account.php | 6 +++++- src/Appwrite/Extend/Exception.php | 4 +++- tests/e2e/Services/Account/AccountConsoleClientTest.php | 4 ++-- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index a071b0cb7e..74e2de7db2 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -88,6 +88,11 @@ return [ 'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.', 'code' => 500, ], + Exception::GENERAL_CODES_DISABLED => [ + 'name' => Exception::GENERAL_CODES_DISABLED, + 'description' => 'Invitation codes are disabled on this server. Please contact the server administrator.', + 'code' => 500, + ], /** User Errors */ Exception::USER_COUNT_EXCEEDED => [ @@ -125,8 +130,8 @@ return [ 'description' => 'Console registration is restricted to specific emails. Contact your administrator for more information.', 'code' => 401, ], - Exception::USER_CODE_INVALID => [ - 'name' => Exception::USER_CODE_INVALID, + Exception::USER_INVALID_CODE => [ + 'name' => Exception::USER_INVALID_CODE, 'description' => 'The specified code is not valid. Contact your administrator for more information.', 'code' => 401, ], diff --git a/app/console b/app/console index 43891a526e..aea8c5f2bb 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1 +Subproject commit aea8c5f2bbe0836fc9fb4720b21d44fe01ac93d1 diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e04b86e57e..634ba6e0f2 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -82,8 +82,12 @@ App::post('/v1/account/invite') $whitelistCodes = (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null)) : []; + if (empty($whitelistCodes)) { + throw new Exception(Exception::GENERAL_CODES_DISABLED); + } + if (!empty($whitelistCodes) && !\in_array($code, $whitelistCodes)) { - throw new Exception(Exception::USER_CODE_INVALID); + throw new Exception(Exception::USER_INVALID_CODE); } $limit = $project->getAttribute('auths', [])['limit'] ?? 0; diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 9f035863eb..8ee0dca0c6 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -50,6 +50,7 @@ class Exception extends \Exception public const GENERAL_CURSOR_NOT_FOUND = 'general_cursor_not_found'; public const GENERAL_SERVER_ERROR = 'general_server_error'; public const GENERAL_PROTOCOL_UNSUPPORTED = 'general_protocol_unsupported'; + public const GENERAL_CODES_DISABLED = 'general_codes_disabled'; /** Users */ public const USER_COUNT_EXCEEDED = 'user_count_exceeded'; @@ -60,7 +61,7 @@ class Exception extends \Exception public const USER_PASSWORD_RESET_REQUIRED = 'user_password_reset_required'; public const USER_EMAIL_NOT_WHITELISTED = 'user_email_not_whitelisted'; public const USER_IP_NOT_WHITELISTED = 'user_ip_not_whitelisted'; - public const USER_CODE_INVALID = 'user_code_invalid'; + public const USER_INVALID_CODE = 'user_invalid_code'; public const USER_INVALID_CREDENTIALS = 'user_invalid_credentials'; public const USER_ANONYMOUS_CONSOLE_PROHIBITED = 'user_anonymous_console_prohibited'; public const USER_SESSION_ALREADY_EXISTS = 'user_session_already_exists'; @@ -179,6 +180,7 @@ class Exception extends \Exception public const DOMAIN_VERIFICATION_FAILED = 'domain_verification_failed'; protected $type = ''; + protected $errors = []; public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null) { diff --git a/tests/e2e/Services/Account/AccountConsoleClientTest.php b/tests/e2e/Services/Account/AccountConsoleClientTest.php index 4258004ecf..9cf0ba68b4 100644 --- a/tests/e2e/Services/Account/AccountConsoleClientTest.php +++ b/tests/e2e/Services/Account/AccountConsoleClientTest.php @@ -38,7 +38,7 @@ class AccountConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 401); - $this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID); + $this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE); $response = $this->client->call(Client::METHOD_POST, '/account/invite', array_merge([ 'origin' => 'http://localhost', @@ -52,7 +52,7 @@ class AccountConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 401); - $this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID); + $this->assertEquals($response['body']['type'], Exception::GENERAL_CODES_DISABLED); /** * Test for SUCCESS From 222421d8f3b65dbe82245bf2f7aa7f6286d678c5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Jan 2023 17:41:15 +0530 Subject: [PATCH 04/27] feat: update account codes --- tests/e2e/Services/Account/AccountConsoleClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Account/AccountConsoleClientTest.php b/tests/e2e/Services/Account/AccountConsoleClientTest.php index 9cf0ba68b4..69bb503428 100644 --- a/tests/e2e/Services/Account/AccountConsoleClientTest.php +++ b/tests/e2e/Services/Account/AccountConsoleClientTest.php @@ -52,7 +52,7 @@ class AccountConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 401); - $this->assertEquals($response['body']['type'], Exception::GENERAL_CODES_DISABLED); + $this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE); /** * Test for SUCCESS From 45b68fa44f6f5627055658bef9156804bc7b3af9 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 19 Jan 2023 16:08:05 +0000 Subject: [PATCH 05/27] Update Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 54aa327769..bba36995cb 100755 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,11 @@ WORKDIR /usr/local/src/console ARG VITE_GA_PROJECT ARG VITE_CONSOLE_MODE +ARG VITE_APPWRITE_GROWTH_ENDPOINT ENV VITE_GA_PROJECT=$VITE_GA_PROJECT ENV VITE_CONSOLE_MODE=$VITE_CONSOLE_MODE +ENV VITE_APPWRITE_GROWTH_ENDPOINT=$VITE_APPWRITE_GROWTH_ENDPOINT RUN npm ci RUN npm run build From c97e7ce328dc0f6fc3691d7d252ea9661e937494 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 02:26:33 +0530 Subject: [PATCH 06/27] feat: update scripts --- .env | 1 + Dockerfile | 2 +- bin/hamster | 3 + bin/stat | 3 - composer.json | 3 +- composer.lock | 222 ++++++++++++++------- docker-compose.yml | 1 + src/Appwrite/Platform/Services/Tasks.php | 4 +- src/Appwrite/Platform/Tasks/Hamster.php | 242 +++++++++++++++++++++++ src/Appwrite/Platform/Tasks/Stat.php | 212 -------------------- 10 files changed, 406 insertions(+), 287 deletions(-) create mode 100644 bin/hamster delete mode 100644 bin/stat create mode 100644 src/Appwrite/Platform/Tasks/Hamster.php delete mode 100644 src/Appwrite/Platform/Tasks/Stat.php diff --git a/.env b/.env index 51e6cc3705..6dcb013dff 100644 --- a/.env +++ b/.env @@ -44,6 +44,7 @@ _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= +_APP_HAMSTER_RECIPIENTS= _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 _APP_STORAGE_LIMIT=30000000 diff --git a/Dockerfile b/Dockerfile index 9a8e35f730..daeac6eb03 100755 --- a/Dockerfile +++ b/Dockerfile @@ -314,7 +314,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ - chmod +x /usr/local/bin/stat && \ + chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/vars && \ chmod +x /usr/local/bin/worker-audits && \ diff --git a/bin/hamster b/bin/hamster new file mode 100644 index 0000000000..dcc7ed308d --- /dev/null +++ b/bin/hamster @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php hamster $@ \ No newline at end of file diff --git a/bin/stat b/bin/stat deleted file mode 100644 index d7030a0958..0000000000 --- a/bin/stat +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php stat $@ \ No newline at end of file diff --git a/composer.json b/composer.json index 1d6d1be25b..f505291643 100644 --- a/composer.json +++ b/composer.json @@ -73,7 +73,8 @@ "phpmailer/phpmailer": "6.6.0", "chillerlan/php-qrcode": "4.3.3", "adhocore/jwt": "1.1.2", - "slickdeals/statsd": "3.1.0" + "slickdeals/statsd": "3.1.0", + "league/csv": "^9.0.0" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index c8b627849e..1324412b95 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4893e1c13630239fe6a20d1c652eb484", + "content-hash": "55eaba9ed1fd51ed74a18ee488c3fdd1", "packages": [ { "name": "adhocore/jwt", @@ -801,20 +801,21 @@ "issues": "https://github.com/influxdata/influxdb-php/issues", "source": "https://github.com/influxdata/influxdb-php/tree/1.15.2" }, + "abandoned": true, "time": "2020-12-26T17:45:17+00:00" }, { "name": "laravel/pint", - "version": "v1.2.0", + "version": "v1.2.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86" + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/1d276e4c803397a26cc337df908f55c2a4e90d86", - "reference": "1d276e4c803397a26cc337df908f55c2a4e90d86", + "url": "https://api.github.com/repos/laravel/pint/zipball/e60e2112ee779ce60f253695b273d1646a17d6f1", + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1", "shasum": "" }, "require": { @@ -826,10 +827,10 @@ }, "require-dev": { "friendsofphp/php-cs-fixer": "^3.11.0", - "illuminate/view": "^9.27", - "laravel-zero/framework": "^9.1.3", - "mockery/mockery": "^1.5.0", - "nunomaduro/larastan": "^2.2", + "illuminate/view": "^9.32.0", + "laravel-zero/framework": "^9.2.0", + "mockery/mockery": "^1.5.1", + "nunomaduro/larastan": "^2.2.0", "nunomaduro/termwind": "^1.14.0", "pestphp/pest": "^1.22.1" }, @@ -867,7 +868,91 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2022-09-13T15:07:15+00:00" + "time": "2022-11-29T16:25:20+00:00" + }, + { + "name": "league/csv", + "version": "9.8.0", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/9d2e0265c5d90f5dd601bc65ff717e05cec19b47", + "reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "php": "^7.4 || ^8.0" + }, + "require-dev": { + "ext-curl": "*", + "ext-dom": "*", + "friendsofphp/php-cs-fixer": "^v3.4.0", + "phpstan/phpstan": "^1.3.0", + "phpstan/phpstan-phpunit": "^1.0.0", + "phpstan/phpstan-strict-rules": "^1.1.0", + "phpunit/phpunit": "^9.5.11" + }, + "suggest": { + "ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes", + "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "CSV data manipulation made easy in PHP", + "homepage": "https://csv.thephpleague.com", + "keywords": [ + "convert", + "csv", + "export", + "filter", + "import", + "read", + "transform", + "write" + ], + "support": { + "docs": "https://csv.thephpleague.com", + "issues": "https://github.com/thephpleague/csv/issues", + "rss": "https://github.com/thephpleague/csv/releases.atom", + "source": "https://github.com/thephpleague/csv" + }, + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], + "time": "2022-01-04T00:13:07+00:00" }, { "name": "matomo/device-detector", @@ -1461,16 +1546,16 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.1.1", + "version": "v3.2.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918" + "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", - "reference": "07f1b9cc2ffee6aaafcf4b710fbc38ff736bd918", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3", + "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3", "shasum": "" }, "require": { @@ -1479,7 +1564,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.1-dev" + "dev-main": "3.3-dev" }, "thanks": { "name": "symfony/contracts", @@ -1508,7 +1593,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.1.1" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0" }, "funding": [ { @@ -1524,7 +1609,7 @@ "type": "tidelift" } ], - "time": "2022-02-25T11:15:52+00:00" + "time": "2022-11-25T10:21:52+00:00" }, { "name": "utopia-php/abuse", @@ -2574,16 +2659,16 @@ }, { "name": "utopia-php/storage", - "version": "0.13.0", + "version": "0.13.2", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "f34c010e4f8394a6b4aff70b6de55041d9a145d3" + "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/f34c010e4f8394a6b4aff70b6de55041d9a145d3", - "reference": "f34c010e4f8394a6b4aff70b6de55041d9a145d3", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/ad1c00f24ca56e73888acc2af3deee4919b1194b", + "reference": "ad1c00f24ca56e73888acc2af3deee4919b1194b", "shasum": "" }, "require": { @@ -2623,9 +2708,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.13.0" + "source": "https://github.com/utopia-php/storage/tree/0.13.2" }, - "time": "2022-11-17T15:10:18+00:00" + "time": "2022-12-20T11:11:35+00:00" }, { "name": "utopia-php/swoole", @@ -2905,30 +2990,30 @@ }, { "name": "doctrine/instantiator", - "version": "1.4.1", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "10dcfce151b967d20fde1b34ae6640712c3891bc" + "reference": "0a0fa9780f5d4e507415a065172d26a98d02047b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/10dcfce151b967d20fde1b34ae6640712c3891bc", - "reference": "10dcfce151b967d20fde1b34ae6640712c3891bc", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/0a0fa9780f5d4e507415a065172d26a98d02047b", + "reference": "0a0fa9780f5d4e507415a065172d26a98d02047b", "shasum": "" }, "require": { "php": "^7.1 || ^8.0" }, "require-dev": { - "doctrine/coding-standard": "^9", + "doctrine/coding-standard": "^9 || ^11", "ext-pdo": "*", "ext-phar": "*", "phpbench/phpbench": "^0.16 || ^1", "phpstan/phpstan": "^1.4", "phpstan/phpstan-phpunit": "^1", "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", - "vimeo/psalm": "^4.22" + "vimeo/psalm": "^4.30 || ^5.4" }, "type": "library", "autoload": { @@ -2955,7 +3040,7 @@ ], "support": { "issues": "https://github.com/doctrine/instantiator/issues", - "source": "https://github.com/doctrine/instantiator/tree/1.4.1" + "source": "https://github.com/doctrine/instantiator/tree/1.5.0" }, "funding": [ { @@ -2971,20 +3056,20 @@ "type": "tidelift" } ], - "time": "2022-03-03T08:28:38+00:00" + "time": "2022-12-30T00:15:36+00:00" }, { "name": "matthiasmullie/minify", - "version": "1.3.69", + "version": "1.3.70", "source": { "type": "git", "url": "https://github.com/matthiasmullie/minify.git", - "reference": "a61c949cccd086808063611ef9698eabe42ef22f" + "reference": "2807d9f9bece6877577ad44acb5c801bb3ae536b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/a61c949cccd086808063611ef9698eabe42ef22f", - "reference": "a61c949cccd086808063611ef9698eabe42ef22f", + "url": "https://api.github.com/repos/matthiasmullie/minify/zipball/2807d9f9bece6877577ad44acb5c801bb3ae536b", + "reference": "2807d9f9bece6877577ad44acb5c801bb3ae536b", "shasum": "" }, "require": { @@ -2993,9 +3078,10 @@ "php": ">=5.3.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "~2.0", - "matthiasmullie/scrapbook": "dev-master", - "phpunit/phpunit": ">=4.8" + "friendsofphp/php-cs-fixer": ">=2.0", + "matthiasmullie/scrapbook": ">=1.3", + "phpunit/phpunit": ">=4.8", + "squizlabs/php_codesniffer": ">=3.0" }, "suggest": { "psr/cache-implementation": "Cache implementation to use with Minify::cache" @@ -3018,12 +3104,12 @@ { "name": "Matthias Mullie", "email": "minify@mullie.eu", - "homepage": "http://www.mullie.eu", + "homepage": "https://www.mullie.eu", "role": "Developer" } ], "description": "CSS & JavaScript minifier, in PHP. Removes whitespace, strips comments, combines files (incl. @import statements and small assets in CSS files), and optimizes/shortens a few common programming patterns.", - "homepage": "http://www.minifier.org", + "homepage": "https://github.com/matthiasmullie/minify", "keywords": [ "JS", "css", @@ -3033,7 +3119,7 @@ ], "support": { "issues": "https://github.com/matthiasmullie/minify/issues", - "source": "https://github.com/matthiasmullie/minify/tree/1.3.69" + "source": "https://github.com/matthiasmullie/minify/tree/1.3.70" }, "funding": [ { @@ -3041,7 +3127,7 @@ "type": "github" } ], - "time": "2022-08-01T09:00:18+00:00" + "time": "2022-12-09T12:56:44+00:00" }, { "name": "matthiasmullie/path-converter", @@ -3157,16 +3243,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.2", + "version": "v4.15.3", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc" + "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", - "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/570e980a201d8ed0236b0a62ddf2c9cbb2034039", + "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039", "shasum": "" }, "require": { @@ -3207,9 +3293,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.3" }, - "time": "2022-11-12T15:38:23+00:00" + "time": "2023-01-16T22:05:37+00:00" }, { "name": "phar-io/manifest", @@ -3489,21 +3575,21 @@ }, { "name": "phpspec/prophecy", - "version": "v1.15.0", + "version": "v1.16.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "bbcd7380b0ebf3961ee21409db7b38bc31d69a13" + "reference": "be8cac52a0827776ff9ccda8c381ac5b71aeb359" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/bbcd7380b0ebf3961ee21409db7b38bc31d69a13", - "reference": "bbcd7380b0ebf3961ee21409db7b38bc31d69a13", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be8cac52a0827776ff9ccda8c381ac5b71aeb359", + "reference": "be8cac52a0827776ff9ccda8c381ac5b71aeb359", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.2", + "php": "^7.2 || 8.0.* || 8.1.* || 8.2.*", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" @@ -3550,22 +3636,22 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/v1.15.0" + "source": "https://github.com/phpspec/prophecy/tree/v1.16.0" }, - "time": "2021-12-08T12:19:24+00:00" + "time": "2022-11-29T15:06:56+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.19", + "version": "9.2.23", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559" + "reference": "9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c77b56b63e3d2031bd8997fcec43c1925ae46559", - "reference": "c77b56b63e3d2031bd8997fcec43c1925ae46559", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c", + "reference": "9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c", "shasum": "" }, "require": { @@ -3621,7 +3707,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.19" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.23" }, "funding": [ { @@ -3629,7 +3715,7 @@ "type": "github" } ], - "time": "2022-11-18T07:47:47+00:00" + "time": "2022-12-28T12:41:10+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5303,16 +5389,16 @@ }, { "name": "twig/twig", - "version": "v3.4.3", + "version": "v3.5.0", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "c38fd6b0b7f370c198db91ffd02e23b517426b58" + "reference": "3ffcf4b7d890770466da3b2666f82ac054e7ec72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/c38fd6b0b7f370c198db91ffd02e23b517426b58", - "reference": "c38fd6b0b7f370c198db91ffd02e23b517426b58", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/3ffcf4b7d890770466da3b2666f82ac054e7ec72", + "reference": "3ffcf4b7d890770466da3b2666f82ac054e7ec72", "shasum": "" }, "require": { @@ -5327,7 +5413,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "3.4-dev" + "dev-master": "3.5-dev" } }, "autoload": { @@ -5363,7 +5449,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.4.3" + "source": "https://github.com/twigphp/Twig/tree/v3.5.0" }, "funding": [ { @@ -5375,7 +5461,7 @@ "type": "tidelift" } ], - "time": "2022-09-28T08:42:51+00:00" + "time": "2022-12-27T12:28:18+00:00" } ], "aliases": [], diff --git a/docker-compose.yml b/docker-compose.yml index feac2c89d0..f831feca67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -134,6 +134,7 @@ services: - _APP_SMTP_SECURE - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD + - _APP_HAMSTER_RECIPIENTS - _APP_USAGE_STATS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 2cb0878589..00cf3f89c5 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,7 +12,7 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules; use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; -use Appwrite\Platform\Tasks\Stat; +use Appwrite\Platform\Tasks\Hamster; use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; @@ -28,7 +28,7 @@ class Tasks extends Service ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) - ->addAction(Stat::getName(), new Stat()) + ->addAction(Hamster::getName(), new Hamster()) ->addAction(Doctor::getName(), new Doctor()) ->addAction(Install::getName(), new Install()) ->addAction(Maintenance::getName(), new Maintenance()) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php new file mode 100644 index 0000000000..650afa7f94 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -0,0 +1,242 @@ + 'files.$all.count.total', + 'Buckets' => 'buckets.$all.count.total', + 'Databases' => 'databases.$all.count.total', + 'Documents' => 'documents.$all.count.total', + 'Collections' => 'collections.$all.count.total', + 'Storage' => 'project.$all.storage.size', + 'Requests' => 'project.$all.network.requests', + 'Bandwidth' => 'project.$all.network.bandwidth', + 'Users' => 'users.$all.count.total', + 'Sessions' => 'sessions.$all.requests.create', + 'Executions' => 'executions.$all.compute.total', + ]; + + protected string $directory = '/usr/local/dev'; + protected string $path; + + protected string $date; + + public static function getName(): string + { + return 'hamster'; + } + + public function __construct() + { + $this + ->desc('Get stats for project') + ->inject('register') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->callback(function (Registry $register, Group $pools, Cache $cache, Database $dbForConsole) { + $this->action($register, $pools, $cache, $dbForConsole); + }); + } + + private function getStats(Database $dbForConsole, Database $dbForProject, Document $project): array + { + $stats = []; + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Project Name */ + $stats['Project Name'] = $project->getAttribute('name'); + + /** Get Total Functions */ + $stats['Functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT); + + /** Get Total Deployments */ + $stats['Deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT); + + /** Get Total Members */ + $teamInternalId = $project->getAttribute('teamInternalId', null); + if ($teamInternalId) { + $stats['Members'] = $dbForConsole->count('memberships', [ + Query::equal('teamInternalId', [$teamInternalId]) + ], APP_LIMIT_COUNT); + } else { + $stats['Members'] = 0; + } + + /** Get Domains */ + $stats['Domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT); + + /** Get Usage stats */ + $range = '90d'; + $periods = [ + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + $metrics = array_values($this->usageStats); + Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $periods[$range]['limit']; + $period = $periods[$range]['period']; + + $requestDocs = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + // Calculate aggregate of each metric + $stats[$metric] = array_sum(array_column($stats[$metric], 'value')); + } + }); + + return $stats; + } + + public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void + { + Console::success('Getting stats...'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); + $csv->insertOne(array_values($statsPerProject)); + + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); + } + + $this->sendEmail($register); + } + + private function sendEmail(Registry $register) + { + /** @var \PHPMailer\PHPMailer\PHPMailer $mail */ + $mail = $register->get('smtp'); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_HAMSTER_RECIPIENTS', '')); + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Cloud Report for {$this->date}"; + $mail->Body = "Please find the daily cloud report atttached"; + + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Stat.php b/src/Appwrite/Platform/Tasks/Stat.php deleted file mode 100644 index 8ac5687bae..0000000000 --- a/src/Appwrite/Platform/Tasks/Stat.php +++ /dev/null @@ -1,212 +0,0 @@ -desc('Get stats for project') - ->callback(fn () => $this->action()); - } - - function getConnection(string $dsn): PDO - { - if (empty($dsn)) { - throw new Exception("Missing value for DSN connection"); - } - $dsn = new DSN($dsn); - $dsnHost = $dsn->getHost(); - $dsnPort = $dsn->getPort(); - $dsnUser = $dsn->getUser(); - $dsnPass = $dsn->getPassword(); - $dsnScheme = $dsn->getScheme(); - $dsnDatabase = $dsn->getPath(); - - $connection = new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_WARNING, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - )); - - return $connection; - } - - - function getStats(Database $dbForProject): array - { - $range = '90d'; - $periods = [ - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - - $metrics = [ - 'files.$all.count.total', - 'buckets.$all.count.total', - 'databases.$all.count.total', - 'documents.$all.count.total', - 'collections.$all.count.total', - 'project.$all.storage.size', - 'project.$all.network.requests', - 'project.$all.network.bandwidth', - 'users.$all.count.total', - 'sessions.$all.requests.create', - 'executions.$all.compute.total', - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - // Calculate aggregate of each metric - $stats[$metric . '.sum'] = array_sum(array_column($stats[$metric], 'value')); - } - }); - - // return only the ahhggregate values - return array_filter($stats, fn ($key) => strpos($key, '.sum') !== false, ARRAY_FILTER_USE_KEY); - } - - - public function action(): void - { - Console::success('Getting stats...'); - - $databases = [ - 'console' => [ - 'type' => 'database', - 'dsns' => '', - 'multiple' => false, - 'schemes' => ['mariadb', 'mysql'], - ], - 'projects' => [ - 'type' => 'database', - 'dsns' => '', - 'multiple' => true, - 'schemes' => ['mariadb', 'mysql'], - ], - ]; - - $dsns = explode(',', $databases['projects']['dsns']); - $projectdsns = []; - foreach ($dsns as &$dsn) { - $dsn = explode('=', $dsn); - $name = 'database' . '_' . $dsn[0]; - $dsn = $dsn[1] ?? ''; - $projectdsns[$name] = $dsn; - } - - $cache = new Cache(new None()); - $consoledsn = explode('=', $databases['console']['dsns']); - $consoledsn = $consoledsn[1] ?? ''; - $adapter = new MySQL($this->getConnection($consoledsn)); - $dbForConsole = new Database($adapter, $cache); - $dbForConsole->setDefaultDatabase('appwrite'); - $dbForConsole->setNamespace('console'); - - $totalProjects = $dbForConsole->count('projects') + 1; - Console::success("Iterating through : {$totalProjects} projects"); - - $app = new App('UTC'); - $console = $app->getResource('console'); - - $projects = [$console]; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - - $stats = []; - - while (!empty($projects)) { - foreach ($projects as $project) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; - } - Console::info("Getting stats for {$project->getId()}"); - - try { - // TODO: Iterate through all project DBs - $db = $project->getAttribute('database'); - $dsn = $projectdsns[$db] ?? ''; - $cache = new Cache(new None()); - $adapter = new MySQL($this->getConnection($dsn)); - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); - $statsPerProject = $this->getStats($dbForProject); - - foreach ($statsPerProject as $key => $value) { - $stats[$key] = isset($stats[$key]) ? $stats[$key] + $value : $value; - } - - } catch (\Throwable $th) { - throw $th; - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } - - } - - $sum = \count($projects); - - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); - - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); - } - - var_dump($stats); - } -} From a3cf72bc15d9d05cd271d907b1b37342200a37f8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 15:36:28 +0530 Subject: [PATCH 07/27] Update src/Appwrite/Platform/Tasks/Hamster.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 650afa7f94..cd85a0d5cc 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -65,7 +65,7 @@ class Hamster extends Action public function __construct() { $this - ->desc('Get stats for project') + ->desc('Get stats for projects') ->inject('register') ->inject('pools') ->inject('cache') From f6bef128e4391bbdc48c2da12c6946f33a50d9f3 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 15:36:34 +0530 Subject: [PATCH 08/27] Update src/Appwrite/Platform/Tasks/Hamster.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index cd85a0d5cc..17bbf99566 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -145,7 +145,7 @@ class Hamster extends Action public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void { - Console::success('Getting stats...'); + Console::info'Getting stats...'); /* Initialise new Utopia app */ $app = new App('UTC'); From ac2e7efb3ce3cb5c7abd1cf09be2748b39e7cfbc Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 16:29:24 +0530 Subject: [PATCH 09/27] feat: linter fixes --- src/Appwrite/Platform/Tasks/Hamster.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 650afa7f94..bc03c8d824 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -140,7 +140,7 @@ class Hamster extends Action } }); - return $stats; + return $stats; } public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void @@ -174,7 +174,7 @@ class Hamster extends Action if ($project->getId() === 'console') { continue; } - + Console::info("Getting stats for {$project->getId()}"); try { @@ -190,7 +190,6 @@ class Hamster extends Action $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); $csv->insertOne(array_values($statsPerProject)); - } catch (\Throwable $th) { throw $th; Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); From 73370ca74609572795c3652d5a75c558ed0f57f8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 17:13:39 +0530 Subject: [PATCH 10/27] feat: add hamster script --- src/Appwrite/Platform/Tasks/Hamster.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 9fef1695f6..bc03c8d824 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -65,7 +65,7 @@ class Hamster extends Action public function __construct() { $this - ->desc('Get stats for projects') + ->desc('Get stats for project') ->inject('register') ->inject('pools') ->inject('cache') @@ -145,7 +145,7 @@ class Hamster extends Action public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void { - Console::info'Getting stats...'); + Console::success('Getting stats...'); /* Initialise new Utopia app */ $app = new App('UTC'); From cdf9745a2a307eb47de8efc7d67d5ef3e8b33275 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 17:15:29 +0530 Subject: [PATCH 11/27] feat: add hamster script --- src/Appwrite/Platform/Tasks/Hamster.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index bc03c8d824..ebf6e1f8c7 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -65,7 +65,7 @@ class Hamster extends Action public function __construct() { $this - ->desc('Get stats for project') + ->desc('Get stats for projects') ->inject('register') ->inject('pools') ->inject('cache') @@ -145,7 +145,7 @@ class Hamster extends Action public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void { - Console::success('Getting stats...'); + Console::info('Getting stats...'); /* Initialise new Utopia app */ $app = new App('UTC'); From 304cb165531e9e3686899ba5b925631b5feeed69 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 17:28:17 +0530 Subject: [PATCH 12/27] feat: add hamster script --- dev/stats_2023-01-26.csv | 2 ++ src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 dev/stats_2023-01-26.csv diff --git a/dev/stats_2023-01-26.csv b/dev/stats_2023-01-26.csv new file mode 100644 index 0000000000..d18d40ddc0 --- /dev/null +++ b/dev/stats_2023-01-26.csv @@ -0,0 +1,2 @@ +"Project ID","Project Name",Functions,Deployments,Members,Domains,Files,Buckets,Databases,Documents,Collections,Storage,Requests,Bandwidth,Users,Sessions,Executions +63d18fd69683bde03f1c,"My First Project",0,0,2,0,0,0,0,0,0,0,2,27919,0,0,0 diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index ebf6e1f8c7..145cbd469d 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -52,7 +52,7 @@ class Hamster extends Action 'Executions' => 'executions.$all.compute.total', ]; - protected string $directory = '/usr/local/dev'; + protected string $directory = '/usr/local'; protected string $path; protected string $date; From fd389bc02f3338cf8711dd0b44c631547a5488f2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 17:28:30 +0530 Subject: [PATCH 13/27] feat: add hamster script --- dev/stats_2023-01-26.csv | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 dev/stats_2023-01-26.csv diff --git a/dev/stats_2023-01-26.csv b/dev/stats_2023-01-26.csv deleted file mode 100644 index d18d40ddc0..0000000000 --- a/dev/stats_2023-01-26.csv +++ /dev/null @@ -1,2 +0,0 @@ -"Project ID","Project Name",Functions,Deployments,Members,Domains,Files,Buckets,Databases,Documents,Collections,Storage,Requests,Bandwidth,Users,Sessions,Executions -63d18fd69683bde03f1c,"My First Project",0,0,2,0,0,0,0,0,0,0,2,27919,0,0,0 From b1f624bc0d7a8dc6d0d11d3d3b03a6141bc667f5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 18:24:32 +0530 Subject: [PATCH 14/27] feat: add hamster script --- src/Appwrite/Platform/Tasks/Hamster.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 145cbd469d..08646da983 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -193,6 +193,10 @@ class Hamster extends Action } catch (\Throwable $th) { throw $th; Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); } } From 45a2268910da76b556eb0ee66e1df58078c57d4d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Jan 2023 18:25:41 +0530 Subject: [PATCH 15/27] feat: add hamster script --- src/Appwrite/Platform/Tasks/Hamster.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 08646da983..801fff4537 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -214,6 +214,10 @@ class Hamster extends Action } $this->sendEmail($register); + + $pools + ->get('console') + ->reclaim(); } private function sendEmail(Registry $register) From 38d7401aa931528c6ff4ba797266e8dcb6961f04 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 6 Feb 2023 15:23:33 +0530 Subject: [PATCH 16/27] feat: automate hamster --- .env | 1 + app/console | 2 +- docker-compose.yml | 33 ++++++ src/Appwrite/Platform/Tasks/Hamster.php | 135 +++++++++++++----------- 4 files changed, 110 insertions(+), 61 deletions(-) diff --git a/.env b/.env index c8837dacfb..096bde5c52 100644 --- a/.env +++ b/.env @@ -46,6 +46,7 @@ _APP_SMTP_SECURE= _APP_SMTP_USERNAME= _APP_SMTP_PASSWORD= _APP_HAMSTER_RECIPIENTS= +_APP_HAMSTER_INTERVAL=86400 _APP_SMS_PROVIDER=sms://username:password@mock _APP_SMS_FROM=+123456789 _APP_STORAGE_LIMIT=30000000 diff --git a/app/console b/app/console index aea8c5f2bb..43891a526e 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit aea8c5f2bbe0836fc9fb4720b21d44fe01ac93d1 +Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1 diff --git a/docker-compose.yml b/docker-compose.yml index 45863846cd..fd8c425e48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -544,6 +544,39 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + appwrite-hamster: + entrypoint: hamster + <<: *x-logging + container_name: appwrite-hamster + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + environment: + - _APP_ENV + - _APP_WORKER_PER_CORE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS + - _APP_OPENSSL_KEY_V1 + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_HAMSTER_RECIPIENTS + - _APP_HAMSTER_INTERVAL + appwrite-maintenance: entrypoint: maintenance <<: *x-logging diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 801fff4537..4bbe8b0ccf 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -145,79 +145,94 @@ class Hamster extends Action public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void { - Console::info('Getting stats...'); - /* Initialise new Utopia app */ - $app = new App('UTC'); - $console = $app->getResource('console'); + Console::title('Cloud Hamster V1'); + Console::success(APP_NAME . ' cloud hamster process v1 has started'); - /** CSV stuff */ - $this->date = date('Y-m-d'); - $this->path = "{$this->directory}/stats_{$this->date}.csv"; - $csv = Writer::createFromPath($this->path, 'w'); - $csv->insertOne($this->columns); + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) - /** Database connections */ - $totalProjects = $dbForConsole->count('projects') + 1; - Console::success("Found a total of: {$totalProjects} projects"); + Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) { + + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Getting Cloud Usage Stats every {$interval} seconds"); + $loopStart = microtime(true); - $projects = [$console]; - $count = 0; - $limit = 30; - $sum = 30; - $offset = 0; - while (!empty($projects)) { - foreach ($projects as $project) { - /** - * Skip user projects with id 'console' - */ - if ($project->getId() === 'console') { - continue; + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects') + 1; + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 30; + $sum = 30; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); + $csv->insertOne(array_values($statsPerProject)); + } catch (\Throwable $th) { + throw $th; + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } } - Console::info("Getting stats for {$project->getId()}"); + $sum = \count($projects); - try { - $db = $project->getAttribute('database'); - $adapter = $pools - ->get($db) - ->pop() - ->getResource(); + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); - $dbForProject = new Database($adapter, $cache); - $dbForProject->setDefaultDatabase('appwrite'); - $dbForProject->setNamespace('_' . $project->getInternalId()); + $offset = $offset + $limit; + $count = $count + $sum; - $statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project); - $csv->insertOne(array_values($statsPerProject)); - } catch (\Throwable $th) { - throw $th; - Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); - } finally { - $pools - ->get($db) - ->reclaim(); - } + Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); } - $sum = \count($projects); + $this->sendEmail($register); - $projects = $dbForConsole->find('projects', [ - Query::limit($limit), - Query::offset($offset), - ]); + $pools + ->get('console') + ->reclaim(); - $offset = $offset + $limit; - $count = $count + $sum; - - Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...'); - } - - $this->sendEmail($register); - - $pools - ->get('console') - ->reclaim(); + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Cloud Stats took {$loopTook} seconds"); + }, $interval); } private function sendEmail(Registry $register) From 4672b31685e5e05276191e6a9bb1674f257e6385 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 6 Feb 2023 15:30:21 +0530 Subject: [PATCH 17/27] feat: linter --- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 4bbe8b0ccf..5e300e686c 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -152,7 +152,7 @@ class Hamster extends Action $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) { - + $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Getting Cloud Usage Stats every {$interval} seconds"); $loopStart = microtime(true); From 5c1adf1f687aa9e2d425ce8734e51ab115be1ddb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 6 Feb 2023 19:44:47 +0530 Subject: [PATCH 18/27] feat: add hamster background task --- src/Appwrite/Platform/Tasks/Hamster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 5e300e686c..a2b4793e5a 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -149,7 +149,7 @@ class Hamster extends Action Console::title('Cloud Hamster V1'); Console::success(APP_NAME . ' cloud hamster process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + $interval = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default) Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) { From b0d4d9183d8a5f1a9f3d9fd8198d1b7c3a8dde8c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 7 Feb 2023 20:07:35 +0530 Subject: [PATCH 19/27] feat: clear mail attachments --- src/Appwrite/Platform/Tasks/Hamster.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index a2b4793e5a..723f3e0e47 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -240,6 +240,13 @@ class Hamster extends Action /** @var \PHPMailer\PHPMailer\PHPMailer $mail */ $mail = $register->get('smtp'); + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + try { /** Addresses */ $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); From 4715bbcf57f64e3d1a5d2bf3d35db7f5292626ba Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 13 Feb 2023 23:29:35 +0000 Subject: [PATCH 20/27] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2554faba6c..32fee05810 100755 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ WORKDIR /usr/local/src/console ARG VITE_GA_PROJECT ARG VITE_CONSOLE_MODE -ARG VITE_APPWRITE_GROWTH_ENDPOINT +ARG VITE_APPWRITE_GROWTH_ENDPOINT=https://growth.appwrite.io/v1 ENV VITE_GA_PROJECT=$VITE_GA_PROJECT ENV VITE_CONSOLE_MODE=$VITE_CONSOLE_MODE From a1fca6ea2d89ee680fc9d579141792b12a805787 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 22 Feb 2023 12:41:32 +0530 Subject: [PATCH 21/27] feat: update homepage branch --- .gitmodules | 2 +- app/console | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f1693e9380..f4fc5d8511 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = db-pools-support + branch = feat-cloud diff --git a/app/console b/app/console index 43891a526e..77ba31859b 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1 +Subproject commit 77ba31859b2ef27738471f40d3e349b2afbfdb8a From ad2c068e75132948967f3eaf63fd1cb594e6dfb7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 22 Feb 2023 12:46:54 +0530 Subject: [PATCH 22/27] feat: update console mode --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fd8c425e48..a6b8af4d74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: DEBUG: false TESTING: true VERSION: dev - VITE_CONSOLE_MODE: self-hosted + VITE_CONSOLE_MODE: cloud ports: - 9501:80 networks: From 10b7503d62fd313926e6ec041f6775090e43279a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 22 Feb 2023 17:09:22 +0530 Subject: [PATCH 23/27] feat: support case insensitive emails --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 634ba6e0f2..d5d9539a5f 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -174,7 +174,7 @@ App::post('/v1/account') $whitelistEmails = $project->getAttribute('authWhitelistEmails'); $whitelistIPs = $project->getAttribute('authWhitelistIPs'); - if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails)) { + if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails) && !\in_array(strtoupper($email), $whitelistEmails)) { throw new Exception(Exception::USER_EMAIL_NOT_WHITELISTED); } From 334e3cc57dd1d35a3ea67e166131baa5a7082416 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 2 Mar 2023 08:13:42 +0000 Subject: [PATCH 24/27] feat: update console base branch --- .gitmodules | 2 +- app/console | 2 +- composer.lock | 153 ++++++++++++++++++++++---------------------------- 3 files changed, 70 insertions(+), 87 deletions(-) diff --git a/.gitmodules b/.gitmodules index f4fc5d8511..abda97c8ae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = feat-cloud + branch = main diff --git a/app/console b/app/console index 77ba31859b..668424a1cf 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 77ba31859b2ef27738471f40d3e349b2afbfdb8a +Subproject commit 668424a1cf7aa8ab03cb171eb184329609f2d4e3 diff --git a/composer.lock b/composer.lock index 1324412b95..e8a9000319 100644 --- a/composer.lock +++ b/composer.lock @@ -1546,16 +1546,16 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.2.0", + "version": "v3.2.1", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3" + "reference": "e2d1534420bd723d0ef5aec58a22c5fe60ce6f5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/e2d1534420bd723d0ef5aec58a22c5fe60ce6f5e", + "reference": "e2d1534420bd723d0ef5aec58a22c5fe60ce6f5e", "shasum": "" }, "require": { @@ -1593,7 +1593,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.1" }, "funding": [ { @@ -1609,7 +1609,7 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-03-01T10:25:55+00:00" }, { "name": "utopia-php/abuse", @@ -2224,22 +2224,23 @@ }, { "name": "utopia-php/logger", - "version": "0.3.0", + "version": "0.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/logger.git", - "reference": "079656cb5169ca9600861eda0b6819199e3d4a57" + "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/logger/zipball/079656cb5169ca9600861eda0b6819199e3d4a57", - "reference": "079656cb5169ca9600861eda0b6819199e3d4a57", + "url": "https://api.github.com/repos/utopia-php/logger/zipball/de623f1ec1c672c795d113dd25c5bf212f7ef4fc", + "reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc", "shasum": "" }, "require": { "php": ">=8.0" }, "require-dev": { + "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.3", "vimeo/psalm": "4.0.1" }, @@ -2253,20 +2254,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - }, - { - "name": "Matej Bačo", - "email": "matej@appwrite.io" - }, - { - "name": "Christy Jacob", - "email": "christy@appwrite.io" - } - ], "description": "Utopia Logger library is simple and lite library for logging information, such as errors or warnings. This library is aiming to be as simple and easy to learn and use.", "keywords": [ "appsignal", @@ -2284,22 +2271,22 @@ ], "support": { "issues": "https://github.com/utopia-php/logger/issues", - "source": "https://github.com/utopia-php/logger/tree/0.3.0" + "source": "https://github.com/utopia-php/logger/tree/0.3.1" }, - "time": "2022-03-18T10:56:57+00:00" + "time": "2023-02-10T15:52:50+00:00" }, { "name": "utopia-php/messaging", - "version": "0.1.0", + "version": "0.1.1", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "501272fad666f06bec8f130076862e7981a73f8c" + "reference": "a75d66ddd59b834ab500a4878a2c084e6572604a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/501272fad666f06bec8f130076862e7981a73f8c", - "reference": "501272fad666f06bec8f130076862e7981a73f8c", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/a75d66ddd59b834ab500a4878a2c084e6572604a", + "reference": "a75d66ddd59b834ab500a4878a2c084e6572604a", "shasum": "" }, "require": { @@ -2307,9 +2294,9 @@ "php": ">=8.0.0" }, "require-dev": { + "laravel/pint": "^1.2", "phpmailer/phpmailer": "6.6.*", - "phpunit/phpunit": "9.5.*", - "squizlabs/php_codesniffer": "^3.6" + "phpunit/phpunit": "9.5.*" }, "type": "library", "autoload": { @@ -2321,12 +2308,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Jake Barnby", - "email": "jake@appwrite.io" - } - ], "description": "A simple, light and advanced PHP messaging library", "keywords": [ "library", @@ -2338,9 +2319,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.1.0" + "source": "https://github.com/utopia-php/messaging/tree/0.1.1" }, - "time": "2022-09-29T11:22:48+00:00" + "time": "2023-02-07T05:42:46+00:00" }, { "name": "utopia-php/orchestration", @@ -3575,20 +3556,20 @@ }, { "name": "phpspec/prophecy", - "version": "v1.16.0", + "version": "v1.17.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be8cac52a0827776ff9ccda8c381ac5b71aeb359" + "reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be8cac52a0827776ff9ccda8c381ac5b71aeb359", - "reference": "be8cac52a0827776ff9ccda8c381ac5b71aeb359", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/15873c65b207b07765dbc3c95d20fdf4a320cbe2", + "reference": "15873c65b207b07765dbc3c95d20fdf4a320cbe2", "shasum": "" }, "require": { - "doctrine/instantiator": "^1.2", + "doctrine/instantiator": "^1.2 || ^2.0", "php": "^7.2 || 8.0.* || 8.1.* || 8.2.*", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", @@ -3596,6 +3577,7 @@ }, "require-dev": { "phpspec/phpspec": "^6.0 || ^7.0", + "phpstan/phpstan": "^1.9", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", @@ -3636,29 +3618,29 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/v1.16.0" + "source": "https://github.com/phpspec/prophecy/tree/v1.17.0" }, - "time": "2022-11-29T15:06:56+00:00" + "time": "2023-02-02T15:41:36+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.23", + "version": "9.2.25", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c" + "reference": "0e2b40518197a8c0d4b08bc34dfff1c99c508954" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c", - "reference": "9f1f0f9a2fbb680b26d1cf9b61b6eac43a6e4e9c", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/0e2b40518197a8c0d4b08bc34dfff1c99c508954", + "reference": "0e2b40518197a8c0d4b08bc34dfff1c99c508954", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.14", + "nikic/php-parser": "^4.15", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -3707,7 +3689,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.23" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.25" }, "funding": [ { @@ -3715,7 +3697,7 @@ "type": "github" } ], - "time": "2022-12-28T12:41:10+00:00" + "time": "2023-02-25T05:32:00+00:00" }, { "name": "phpunit/php-file-iterator", @@ -4427,16 +4409,16 @@ }, { "name": "sebastian/environment", - "version": "5.1.4", + "version": "5.1.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "1b5dff7bb151a4db11d49d90e5408e4e938270f7" + "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/1b5dff7bb151a4db11d49d90e5408e4e938270f7", - "reference": "1b5dff7bb151a4db11d49d90e5408e4e938270f7", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", + "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", "shasum": "" }, "require": { @@ -4478,7 +4460,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/environment/issues", - "source": "https://github.com/sebastianbergmann/environment/tree/5.1.4" + "source": "https://github.com/sebastianbergmann/environment/tree/5.1.5" }, "funding": [ { @@ -4486,7 +4468,7 @@ "type": "github" } ], - "time": "2022-04-03T09:37:03+00:00" + "time": "2023-02-03T06:03:51+00:00" }, { "name": "sebastian/exporter", @@ -4800,16 +4782,16 @@ }, { "name": "sebastian/recursion-context", - "version": "4.0.4", + "version": "4.0.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/recursion-context.git", - "reference": "cd9d8cf3c5804de4341c283ed787f099f5506172" + "reference": "e75bd0f07204fec2a0af9b0f3cfe97d05f92efc1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/cd9d8cf3c5804de4341c283ed787f099f5506172", - "reference": "cd9d8cf3c5804de4341c283ed787f099f5506172", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/e75bd0f07204fec2a0af9b0f3cfe97d05f92efc1", + "reference": "e75bd0f07204fec2a0af9b0f3cfe97d05f92efc1", "shasum": "" }, "require": { @@ -4848,10 +4830,10 @@ } ], "description": "Provides functionality to recursively process PHP variables", - "homepage": "http://www.github.com/sebastianbergmann/recursion-context", + "homepage": "https://github.com/sebastianbergmann/recursion-context", "support": { "issues": "https://github.com/sebastianbergmann/recursion-context/issues", - "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.4" + "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.5" }, "funding": [ { @@ -4859,7 +4841,7 @@ "type": "github" } ], - "time": "2020-10-26T13:17:30+00:00" + "time": "2023-02-03T06:07:39+00:00" }, { "name": "sebastian/resource-operations", @@ -4918,16 +4900,16 @@ }, { "name": "sebastian/type", - "version": "3.2.0", + "version": "3.2.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/type.git", - "reference": "fb3fe09c5f0bae6bc27ef3ce933a1e0ed9464b6e" + "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/fb3fe09c5f0bae6bc27ef3ce933a1e0ed9464b6e", - "reference": "fb3fe09c5f0bae6bc27ef3ce933a1e0ed9464b6e", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", + "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", "shasum": "" }, "require": { @@ -4962,7 +4944,7 @@ "homepage": "https://github.com/sebastianbergmann/type", "support": { "issues": "https://github.com/sebastianbergmann/type/issues", - "source": "https://github.com/sebastianbergmann/type/tree/3.2.0" + "source": "https://github.com/sebastianbergmann/type/tree/3.2.1" }, "funding": [ { @@ -4970,7 +4952,7 @@ "type": "github" } ], - "time": "2022-09-12T14:47:03+00:00" + "time": "2023-02-03T06:13:03+00:00" }, { "name": "sebastian/version", @@ -5027,16 +5009,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "3.7.1", + "version": "3.7.2", "source": { "type": "git", "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", - "reference": "1359e176e9307e906dc3d890bcc9603ff6d90619" + "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/1359e176e9307e906dc3d890bcc9603ff6d90619", - "reference": "1359e176e9307e906dc3d890bcc9603ff6d90619", + "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879", + "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879", "shasum": "" }, "require": { @@ -5072,14 +5054,15 @@ "homepage": "https://github.com/squizlabs/PHP_CodeSniffer", "keywords": [ "phpcs", - "standards" + "standards", + "static analysis" ], "support": { "issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues", "source": "https://github.com/squizlabs/PHP_CodeSniffer", "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki" }, - "time": "2022-06-18T07:21:10+00:00" + "time": "2023-02-22T23:07:41+00:00" }, { "name": "swoole/ide-helper", @@ -5389,16 +5372,16 @@ }, { "name": "twig/twig", - "version": "v3.5.0", + "version": "v3.5.1", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "3ffcf4b7d890770466da3b2666f82ac054e7ec72" + "reference": "a6e0510cc793912b451fd40ab983a1d28f611c15" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/3ffcf4b7d890770466da3b2666f82ac054e7ec72", - "reference": "3ffcf4b7d890770466da3b2666f82ac054e7ec72", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/a6e0510cc793912b451fd40ab983a1d28f611c15", + "reference": "a6e0510cc793912b451fd40ab983a1d28f611c15", "shasum": "" }, "require": { @@ -5449,7 +5432,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.5.0" + "source": "https://github.com/twigphp/Twig/tree/v3.5.1" }, "funding": [ { @@ -5461,7 +5444,7 @@ "type": "tidelift" } ], - "time": "2022-12-27T12:28:18+00:00" + "time": "2023-02-08T07:49:20+00:00" } ], "aliases": [], From 253dd5e58c8a96291c6403f1b1a73cc2fe7c2c4a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 7 Mar 2023 13:45:32 +0530 Subject: [PATCH 25/27] feat: update console branch --- .gitmodules | 2 +- app/console | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f4fc5d8511..abda97c8ae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = feat-cloud + branch = main diff --git a/app/console b/app/console index 77ba31859b..a0dfd89382 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 77ba31859b2ef27738471f40d3e349b2afbfdb8a +Subproject commit a0dfd89382ef55a5c8457bb965d1cc8c28d05746 From 46f07faa981ca1bb712538ca416aefd90b2b3aba Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 9 Mar 2023 12:07:47 +0000 Subject: [PATCH 26/27] feat: update Docekrfile to use appwrite:base image --- .gitmodules | 2 +- Dockerfile | 198 +------------------------------------------------- app/console | 2 +- composer.lock | 40 +++++----- 4 files changed, 23 insertions(+), 219 deletions(-) diff --git a/.gitmodules b/.gitmodules index abda97c8ae..6785e119cb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = main + branch = 2.2.2 diff --git a/Dockerfile b/Dockerfile index 32fee05810..abe9b7128f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -29,155 +29,7 @@ ENV VITE_APPWRITE_GROWTH_ENDPOINT=$VITE_APPWRITE_GROWTH_ENDPOINT RUN npm ci RUN npm run build -FROM php:8.0.18-cli-alpine3.15 as compile - -ARG DEBUG=false -ENV DEBUG=$DEBUG - -ENV PHP_REDIS_VERSION=5.3.7 \ - PHP_MONGODB_VERSION=1.13.0 \ - PHP_SWOOLE_VERSION=v4.8.10 \ - PHP_IMAGICK_VERSION=3.7.0 \ - PHP_YAML_VERSION=2.2.2 \ - PHP_MAXMINDDB_VERSION=v1.11.0 \ - PHP_MEMCACHED_VERSION=v3.2.0 \ - PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 " - -RUN \ - apk add --no-cache --virtual .deps \ - make \ - automake \ - autoconf \ - gcc \ - g++ \ - git \ - zlib-dev \ - brotli-dev \ - openssl-dev \ - yaml-dev \ - imagemagick \ - imagemagick-dev \ - libmaxminddb-dev \ - libmemcached-dev \ - zstd-dev - -RUN docker-php-ext-install sockets - -FROM compile AS redis -RUN \ - # Redis Extension - git clone --depth 1 --branch $PHP_REDIS_VERSION https://github.com/phpredis/phpredis.git && \ - cd phpredis && \ - phpize && \ - ./configure && \ - make && make install - -## Swoole Extension -FROM compile AS swoole -RUN \ - git clone --depth 1 --branch $PHP_SWOOLE_VERSION https://github.com/swoole/swoole-src.git && \ - cd swoole-src && \ - phpize && \ - ./configure --enable-sockets --enable-http2 --enable-openssl && \ - make && make install && \ - cd .. - -## Swoole Debugger setup -RUN if [ "$DEBUG" == "true" ]; then \ - cd /tmp && \ - apk add boost-dev && \ - git clone --depth 1 https://github.com/swoole/yasd && \ - cd yasd && \ - phpize && \ - ./configure && \ - make && make install && \ - cd ..;\ - fi - -## Imagick Extension -FROM compile AS imagick -RUN \ - git clone --depth 1 --branch $PHP_IMAGICK_VERSION https://github.com/imagick/imagick && \ - cd imagick && \ - phpize && \ - ./configure && \ - make && make install - -## YAML Extension -FROM compile AS yaml -RUN \ - git clone --depth 1 --branch $PHP_YAML_VERSION https://github.com/php/pecl-file_formats-yaml && \ - cd pecl-file_formats-yaml && \ - phpize && \ - ./configure && \ - make && make install - -## Maxminddb extension -FROM compile AS maxmind -RUN \ - git clone --depth 1 --branch $PHP_MAXMINDDB_VERSION https://github.com/maxmind/MaxMind-DB-Reader-php.git && \ - cd MaxMind-DB-Reader-php && \ - cd ext && \ - phpize && \ - ./configure && \ - make && make install - -# Mongodb Extension -FROM compile as mongodb -RUN \ - git clone --depth 1 --branch $PHP_MONGODB_VERSION https://github.com/mongodb/mongo-php-driver.git && \ - cd mongo-php-driver && \ - git submodule update --init && \ - phpize && \ - ./configure && \ - make && make install - -# Memcached Extension -FROM compile as memcached -RUN \ - git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \ - cd php-memcached && \ - phpize && \ - ./configure && \ - make && make install - -# Zstd Compression -FROM compile as zstd -RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ - && cd php-ext-zstd \ - && git checkout $PHP_ZSTD_VERSION \ - && phpize \ - && ./configure --with-libzstd \ - && make && make install - -# Rust Extensions Compile Image -FROM php:8.0.18-cli as rust_compile - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - -ENV PATH=/root/.cargo/bin:$PATH - -RUN apt-get update && apt-get install musl-tools build-essential clang-11 git -y -RUN rustup target add $(uname -m)-unknown-linux-musl - -# Install ZigBuild for easier cross-compilation -RUN curl https://ziglang.org/builds/zig-linux-$(uname -m)-0.10.0-dev.2674+d980c6a38.tar.xz --output /tmp/zig.tar.xz -RUN tar -xf /tmp/zig.tar.xz -C /tmp/ && cp -r /tmp/zig-linux-$(uname -m)-0.10.0-dev.2674+d980c6a38 /tmp/zig/ -ENV PATH=/tmp/zig:$PATH -RUN cargo install cargo-zigbuild -ENV RUSTFLAGS="-C target-feature=-crt-static" - -FROM rust_compile as scrypt - -WORKDIR /usr/local/lib/php/extensions/ - -RUN \ - git clone --depth 1 https://github.com/appwrite/php-scrypt.git && \ - cd php-scrypt && \ - cargo zigbuild --workspace --all-targets --target $(uname -m)-unknown-linux-musl --release && \ - mv target/$(uname -m)-unknown-linux-musl/release/libphp_scrypt.so target/libphp_scrypt.so - -FROM php:8.0.18-cli-alpine3.15 as final +FROM appwrite/base:0.1.0 as final LABEL maintainer="team@appwrite.io" @@ -231,38 +83,6 @@ ENV _APP_SERVER=swoole \ _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \ _APP_MAINTENANCE_INTERVAL=86400 -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN \ - apk update \ - && apk add --no-cache --virtual .deps \ - make \ - automake \ - autoconf \ - gcc \ - g++ \ - curl-dev \ - && apk add --no-cache \ - libstdc++ \ - certbot \ - rsync \ - brotli-dev \ - yaml-dev \ - imagemagick \ - imagemagick-dev \ - libmaxminddb-dev \ - certbot \ - docker-cli \ - libgomp \ - && docker-php-ext-install sockets opcache pdo_mysql \ - && apk del .deps \ - && rm -rf /var/cache/apk/* - -RUN \ - mkdir -p $DOCKER_CONFIG/cli-plugins \ - && ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then ARCH="armv7"; fi \ - && curl -SL https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-linux-$ARCH -o $DOCKER_CONFIG/cli-plugins/docker-compose \ - && chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose RUN \ if [ "$DEBUG" == "true" ]; then \ @@ -273,15 +93,6 @@ WORKDIR /usr/src/code COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor COPY --from=node /usr/local/src/console/build /usr/src/code/console -COPY --from=swoole /usr/local/lib/php/extensions/no-debug-non-zts-20200930/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yasd.so* /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=redis /usr/local/lib/php/extensions/no-debug-non-zts-20200930/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imagick.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ -COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ # Add Source Code COPY ./app /usr/src/code/app @@ -333,13 +144,6 @@ RUN chmod +x /usr/local/bin/doctor && \ RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ # Enable Extensions -RUN echo extension=swoole.so >> /usr/local/etc/php/conf.d/swoole.ini -RUN echo extension=redis.so >> /usr/local/etc/php/conf.d/redis.ini -RUN echo extension=imagick.so >> /usr/local/etc/php/conf.d/imagick.ini -RUN echo extension=yaml.so >> /usr/local/etc/php/conf.d/yaml.ini -RUN echo extension=maxminddb.so >> /usr/local/etc/php/conf.d/maxminddb.ini -RUN echo extension=libphp_scrypt.so >> /usr/local/etc/php/conf.d/libphp_scrypt.ini -RUN echo extension=zstd.so >> /usr/local/etc/php/conf.d/zstd.ini RUN if [ "$DEBUG" == "true" ]; then printf "zend_extension=yasd \nyasd.debug_mode=remote \nyasd.init_file=/usr/local/dev/yasd_init.php \nyasd.remote_port=9005 \nyasd.log_level=-1" >> /usr/local/etc/php/conf.d/yasd.ini; fi RUN if [ "$DEBUG" == "true" ]; then echo "opcache.enable=0" >> /usr/local/etc/php/conf.d/appwrite.ini; fi diff --git a/app/console b/app/console index a0dfd89382..cad6f3b1bf 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit a0dfd89382ef55a5c8457bb965d1cc8c28d05746 +Subproject commit cad6f3b1bfdae4d423ba6f0735ba2a5cd5a58551 diff --git a/composer.lock b/composer.lock index e8a9000319..4c23baa2b5 100644 --- a/composer.lock +++ b/composer.lock @@ -3165,16 +3165,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.11.0", + "version": "1.11.1", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614" + "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/14daed4296fae74d9e3201d2c4925d1acb7aa614", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", + "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", "shasum": "" }, "require": { @@ -3212,7 +3212,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.0" + "source": "https://github.com/myclabs/DeepCopy/tree/1.11.1" }, "funding": [ { @@ -3220,20 +3220,20 @@ "type": "tidelift" } ], - "time": "2022-03-03T13:19:32+00:00" + "time": "2023-03-08T13:26:56+00:00" }, { "name": "nikic/php-parser", - "version": "v4.15.3", + "version": "v4.15.4", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039" + "reference": "6bb5176bc4af8bcb7d926f88718db9b96a2d4290" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/570e980a201d8ed0236b0a62ddf2c9cbb2034039", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6bb5176bc4af8bcb7d926f88718db9b96a2d4290", + "reference": "6bb5176bc4af8bcb7d926f88718db9b96a2d4290", "shasum": "" }, "require": { @@ -3274,9 +3274,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.3" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.4" }, - "time": "2023-01-16T22:05:37+00:00" + "time": "2023-03-05T19:49:14+00:00" }, { "name": "phar-io/manifest", @@ -3624,16 +3624,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.25", + "version": "9.2.26", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "0e2b40518197a8c0d4b08bc34dfff1c99c508954" + "reference": "443bc6912c9bd5b409254a40f4b0f4ced7c80ea1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/0e2b40518197a8c0d4b08bc34dfff1c99c508954", - "reference": "0e2b40518197a8c0d4b08bc34dfff1c99c508954", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/443bc6912c9bd5b409254a40f4b0f4ced7c80ea1", + "reference": "443bc6912c9bd5b409254a40f4b0f4ced7c80ea1", "shasum": "" }, "require": { @@ -3655,8 +3655,8 @@ "phpunit/phpunit": "^9.3" }, "suggest": { - "ext-pcov": "*", - "ext-xdebug": "*" + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "type": "library", "extra": { @@ -3689,7 +3689,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.25" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.26" }, "funding": [ { @@ -3697,7 +3697,7 @@ "type": "github" } ], - "time": "2023-02-25T05:32:00+00:00" + "time": "2023-03-06T12:58:08+00:00" }, { "name": "phpunit/php-file-iterator", From a4b31f64726572a4ebfe1bd0405490cffa162018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 15 Mar 2023 09:20:25 +0100 Subject: [PATCH 27/27] Fix after merge --- app/config/collections.php | 15 +++- app/controllers/api/functions.php | 2 +- app/workers/builds.php | 12 +-- app/workers/deletes.php | 12 +-- app/workers/functions.php | 2 +- composer.lock | 89 +++++++++++++++++++- src/Appwrite/Migration/Version/V17.php | 12 +++ src/Appwrite/Usage/Stats.php | 2 + src/Appwrite/Utopia/Response/Model/Build.php | 6 ++ 9 files changed, 135 insertions(+), 17 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index d042dd17c1..454ed006eb 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2695,7 +2695,18 @@ $collections = [ 'filters' => [], ], [ - '$id' => 'deploymentInternalId', + '$id' => ID::custom('size'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('deploymentInternalId'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => Database::LENGTH_KEY, @@ -2739,7 +2750,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => ID::custom('outputPath'), + '$id' => ID::custom('path'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 2048, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 93db930783..74a2865043 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -1311,7 +1311,7 @@ App::post('/v1/functions/:functionId/executions') variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), path: $path, method: $method, diff --git a/app/workers/builds.php b/app/workers/builds.php index ddaf4f1b6d..b3a9974c92 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -100,14 +100,15 @@ class BuildsV1 extends Worker 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'status' => 'processing', - 'outputPath' => '', + 'path' => '', 'runtime' => $function->getAttribute('runtime'), 'source' => $deployment->getAttribute('path'), 'sourceType' => $device, 'stdout' => '', 'stderr' => '', 'endTime' => null, - 'duration' => 0 + 'duration' => 0, + 'size' => 0 ])); $deployment->setAttribute('buildId', $build->getId()); $deployment->setAttribute('buildInternalId', $build->getInternalId()); @@ -189,15 +190,14 @@ class BuildsV1 extends Worker ] ); - $endTime = new \DateTime(); - $endTime->setTimestamp($response['endTimeUnix']); + $endTime = DateTime::now(); /** Update the build document */ $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); - $build->setAttribute('endTime', DateTime::format($endTime)); + $build->setAttribute('endTime', $endTime); $build->setAttribute('duration', \intval(\ceil($response['duration']))); $build->setAttribute('status', 'ready'); - $build->setAttribute('outputPath', $response['path']); + $build->setAttribute('path', $response['path']); $build->setAttribute('size', $response['size']); $build->setAttribute('stderr', $response['stderr']); $build->setAttribute('stdout', $response['stdout']); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f296231187..904824b596 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -501,10 +501,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); } @@ -554,10 +554,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('builds', [ Query::equal('deploymentId', [$deploymentId]) ], $dbForProject, function (Document $document) use ($storageBuilds) { - if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) { - Console::success('Deleted build files: ' . $document->getAttribute('outputPath', '')); + if ($storageBuilds->delete($document->getAttribute('path', ''), true)) { + Console::success('Deleted build files: ' . $document->getAttribute('path', '')); } else { - Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', '')); + Console::error('Failed to delete build files: ' . $document->getAttribute('path', '')); } }); diff --git a/app/workers/functions.php b/app/workers/functions.php index a1c924b72d..2a1894baa7 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -173,7 +173,7 @@ Server::setResource('execute', function () { variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('outputPath', ''), + source: $build->getAttribute('path', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), path: $path, method: $method, diff --git a/composer.lock b/composer.lock index 24fb668be4..4bdf4a404d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e8e64d67c559b26264b6a89ebab1ab72", + "content-hash": "d388807afe22fb473b5136a75cb3d7e7", "packages": [ { "name": "adhocore/jwt", @@ -870,6 +870,93 @@ }, "time": "2022-11-29T16:25:20+00:00" }, + { + "name": "league/csv", + "version": "9.9.0", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "b4418ede47fbd88facc34e40a16c8ce9153b961b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/b4418ede47fbd88facc34e40a16c8ce9153b961b", + "reference": "b4418ede47fbd88facc34e40a16c8ce9153b961b", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "php": "^8.1.2" + }, + "require-dev": { + "doctrine/collections": "^2.1.2", + "ext-dom": "*", + "ext-xdebug": "*", + "friendsofphp/php-cs-fixer": "^v3.14.3", + "phpbench/phpbench": "^1.2.8", + "phpstan/phpstan": "^1.10.4", + "phpstan/phpstan-deprecation-rules": "^1.1.2", + "phpstan/phpstan-phpunit": "^1.3.10", + "phpstan/phpstan-strict-rules": "^1.5.0", + "phpunit/phpunit": "^10.0.14" + }, + "suggest": { + "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes", + "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "CSV data manipulation made easy in PHP", + "homepage": "https://csv.thephpleague.com", + "keywords": [ + "convert", + "csv", + "export", + "filter", + "import", + "read", + "transform", + "write" + ], + "support": { + "docs": "https://csv.thephpleague.com", + "issues": "https://github.com/thephpleague/csv/issues", + "rss": "https://github.com/thephpleague/csv/releases.atom", + "source": "https://github.com/thephpleague/csv" + }, + "funding": [ + { + "url": "https://github.com/sponsors/nyamsprod", + "type": "github" + } + ], + "time": "2023-03-11T15:57:12+00:00" + }, { "name": "matomo/device-detector", "version": "6.0.0", diff --git a/src/Appwrite/Migration/Version/V17.php b/src/Appwrite/Migration/Version/V17.php index c96c9e08a4..dac6b740c6 100644 --- a/src/Appwrite/Migration/Version/V17.php +++ b/src/Appwrite/Migration/Version/V17.php @@ -46,6 +46,18 @@ class V17 extends Migration $this->projectDB->setNamespace("_{$this->project->getInternalId()}"); switch ($id) { + case 'builds': + try { + /** + * Create 'size' attribute + */ + $this->createAttributeFromCollection($this->projectDB, $id, 'size'); + $this->projectDB->deleteCachedCollection($id); + } catch (\Throwable $th) { + Console::warning("'size' from {$id}: {$th->getMessage()}"); + } + + break; case 'files': try { /** diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index e6e0056664..38fdc123cf 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -184,6 +184,7 @@ class Stats $functionBuild = $this->params['builds.{scope}.compute'] ?? 0; $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms + $functionBuildSize = ($this->params['buildSize'] ?? 0); // bytes $functionBuildStatus = $this->params['buildStatus'] ?? ''; $functionCompute = $functionExecutionTime + $functionBuildTime; $functionTags = $tags . ',functionId=' . $functionId; @@ -207,6 +208,7 @@ class Stats if ($functionBuild >= 1) { $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus); $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime); + $this->statsd->count('builds.{scope}.storage.size' . $functionTags, $functionBuildSize); } if ($functionBuild + $functionExecution >= 1) { $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute); diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index b76f0ee083..d80c17645a 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -63,6 +63,12 @@ class Build extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The code size in bytes.', + 'default' => 0, + 'example' => 128, + ]) ; }