CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-03 13:41:01 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5223 from appwrite/feat-db-pools-sync

Browse source 
Feat db pools sync
This commit is contained in:
Christy Jacob 2023-03-24 18:21:43 +05:30 committed by GitHub
parent 8b6d523f57 b1c5457351
commit 109564c8b1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 871 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.env
View file

@ -41,6 +41,9 @@ _APP_SMTP_PORT=1025
_APP_SMTP_SECURE=
_APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_HAMSTER_INTERVAL=86400
_APP_HAMSTER_TIME=21:00
_APP_MIXPANEL_TOKEN=
_APP_SMS_PROVIDER=sms://username:password@mock
_APP_SMS_FROM=+123456789
_APP_STORAGE_LIMIT=30000000

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = feat-cloud
branch = 2.2.2

3
Dockerfile
View file

@ -111,7 +111,7 @@ RUN mkdir -p /storage/uploads && \
# Executables
RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/patch-create-missing-schedules && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/install && \
@ -121,6 +121,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/sdks && \
chmod +x /usr/local/bin/specs && \
chmod +x /usr/local/bin/ssl && \
chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/test && \
chmod +x /usr/local/bin/vars && \
chmod +x /usr/local/bin/worker-audits && \

35
app/config/collections.php
View file

@ -2311,17 +2311,6 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('scheduleUpdatedAt'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('timeout'),
'type' => Database::VAR_INTEGER,
@ -2616,6 +2605,17 @@ $collections = [
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('endTime'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('duration'),
'type' => Database::VAR_INTEGER,
@ -2672,7 +2672,7 @@ $collections = [
'filters' => [],
],
[
'$id' => ID::custom('path'),
'$id' => ID::custom('outputPath'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
@ -2682,17 +2682,6 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('size'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('stderr'),
'type' => Database::VAR_STRING,

9
app/config/errors.php
View file

@ -88,6 +88,11 @@ return [
'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.',
'code' => 500,
],
Exception::GENERAL_CODES_DISABLED => [
'name' => Exception::GENERAL_CODES_DISABLED,
'description' => 'Invitation codes are disabled on this server. Please contact the server administrator.',
'code' => 500,
],
/** User Errors */
Exception::USER_COUNT_EXCEEDED => [
@ -125,8 +130,8 @@ return [
'description' => 'Console registration is restricted to specific emails. Contact your administrator for more information.',
'code' => 401,
],
Exception::USER_CODE_INVALID => [
'name' => Exception::USER_CODE_INVALID,
Exception::USER_INVALID_CODE => [
'name' => Exception::USER_INVALID_CODE,
'description' => 'The specified code is not valid. Contact your administrator for more information.',
'code' => 401,
],

2
app/console

@ -1 +1 @@
Subproject commit fae048b91787d0b372c13caf27f14fc8b780ef60
Subproject commit 39727607a036534a323101fea876cbe68b05f4f0

8
app/controllers/api/account.php
View file

@ -81,8 +81,12 @@ App::post('/v1/account/invite')
$whitelistCodes = (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null)) : [];
if (empty($whitelistCodes)) {
throw new Exception(Exception::GENERAL_CODES_DISABLED);
}
if (!empty($whitelistCodes) && !\in_array($code, $whitelistCodes)) {
throw new Exception(Exception::USER_CODE_INVALID);
throw new Exception(Exception::USER_INVALID_CODE);
}
$limit = $project->getAttribute('auths', [])['limit'] ?? 0;
@ -168,7 +172,7 @@ App::post('/v1/account')
$whitelistEmails = $project->getAttribute('authWhitelistEmails');
$whitelistIPs = $project->getAttribute('authWhitelistIPs');
if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails)) {
if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails) && !\in_array(strtoupper($email), $whitelistEmails)) {
throw new Exception(Exception::USER_EMAIL_NOT_WHITELISTED);
}

103
app/controllers/api/functions.php
View file

@ -86,7 +86,6 @@ App::post('/v1/functions')
'events' => $events,
'schedule' => $schedule,
'scheduleInternalId' => '',
'scheduleUpdatedAt' => DateTime::now(),
'timeout' => $timeout,
'search' => implode(' ', [$functionId, $name, $runtime])
]));
@ -416,26 +415,16 @@ App::put('/v1/functions/:functionId')
'name' => $name,
'events' => $events,
'schedule' => $schedule,
'scheduleUpdatedAt' => DateTime::now(),
'timeout' => $timeout,
'enabled' => $enabled,
'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]),
])));
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
/**
* In case we want to clear the schedule
*/
if (!empty($function->getAttribute('deployment'))) {
$schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt'));
}
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$eventsInstance->setParam('functionId', $function->getId());
@ -488,19 +477,14 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId')
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [
'deploymentInternalId' => $deployment->getInternalId(),
'deployment' => $deployment->getId()
'deployment' => $deployment->getId(),
])));
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$active = !empty($function->getAttribute('schedule'));
if ($active) {
$schedule->setAttribute('resourceUpdatedAt', datetime::now());
}
$schedule->setAttribute('active', $active);
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$events
@ -543,12 +527,9 @@ App::delete('/v1/functions/:functionId')
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('active', false)
;
->setAttribute('active', false);
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$deletes
@ -749,22 +730,6 @@ App::post('/v1/functions/:functionId/deployments')
}
}
/**
* TODO Should we update also the function collection with the scheduleUpdatedAt attr?
*/
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$active = !empty($function->getAttribute('schedule'));
if ($active) {
$schedule->setAttribute('resourceUpdatedAt', datetime::now());
}
$schedule->setAttribute('active', $active);
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$metadata = null;
$events
@ -1163,7 +1128,7 @@ App::post('/v1/functions/:functionId/executions')
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $build->getAttribute('path', ''),
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);
@ -1347,7 +1312,8 @@ App::post('/v1/functions/:functionId/variables')
->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', false)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@ -1376,8 +1342,22 @@ App::post('/v1/functions/:functionId/variables')
throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($variable, Response::MODEL_VARIABLE);
@ -1463,7 +1443,8 @@ App::put('/v1/functions/:functionId/variables/:variableId')
->param('value', null, new Text(8192), 'Variable value. Max length: 8192 chars.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -1492,8 +1473,22 @@ App::put('/v1/functions/:functionId/variables/:variableId')
throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response->dynamic($variable, Response::MODEL_VARIABLE);
});
@ -1513,7 +1508,8 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
->param('variableId', '', new UID(), 'Variable unique ID.', false)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@ -1530,7 +1526,22 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
}
$dbForProject->deleteDocument('variables', $variable->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response->noContent();
});

18
app/workers/builds.php
View file

@ -104,13 +104,13 @@ class BuildsV1 extends Worker
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'status' => 'processing',
'path' => '',
'size' => 0,
'outputPath' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path'),
'sourceType' => $device,
'stdout' => '',
'stderr' => '',
'endTime' => null,
'duration' => 0
]));
$deployment->setAttribute('buildId', $build->getId());
@ -192,12 +192,14 @@ class BuildsV1 extends Worker
]
);
$endTime = new \DateTime();
$endTime->setTimestamp($response['endTimeUnix']);
/** Update the build document */
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
$build->setAttribute('endTime', DateTime::format($endTime));
$build->setAttribute('duration', \intval($response['duration']));
$build->setAttribute('status', $response['status']);
$build->setAttribute('path', $response['path']);
$build->setAttribute('size', $response['size']);
$build->setAttribute('outputPath', $response['outputPath']);
$build->setAttribute('stderr', $response['stderr']);
$build->setAttribute('stdout', $response['stdout']);
@ -206,8 +208,6 @@ class BuildsV1 extends Worker
Console::success("Build id: $buildId created");
$function->setAttribute('scheduleUpdatedAt', DateTime::now());
/** Set auto deploy */
if ($deployment->getAttribute('activate') === true) {
$function->setAttribute('deploymentInternalId', $deployment->getInternalId());
@ -218,7 +218,7 @@ class BuildsV1 extends Worker
/** Update function schedule */
$dbForConsole = $this->getConsoleDB();
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule->setAttribute('resourceUpdatedAt', $function->getAttribute('scheduleUpdatedAt'));
$schedule->setAttribute('resourceUpdatedAt', DateTime::now());
$schedule
->setAttribute('schedule', $function->getAttribute('schedule'))
@ -229,7 +229,7 @@ class BuildsV1 extends Worker
} catch (\Throwable $th) {
$endTime = DateTime::now();
$interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', $interval->format('%s') + 0);
$build->setAttribute('status', 'failed');
$build->setAttribute('stderr', $th->getMessage());

12
app/workers/deletes.php
View file

@ -527,10 +527,10 @@ class DeletesV1 extends Worker
$this->deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) {
if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
} else {
Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
}
});
}
@ -576,10 +576,10 @@ class DeletesV1 extends Worker
$this->deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds) {
if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
} else {
Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
}
});

4
app/workers/functions.php
View file

@ -93,7 +93,7 @@ Server::setResource('execute', function () {
'response' => '',
'stderr' => '',
'duration' => 0.0,
'search' => implode(' ', [$functionId, $executionId]),
'search' => implode(' ', [$function->getId(), $executionId]),
]));
// TODO: @Meldiron Trigger executions.create event here
@ -145,7 +145,7 @@ Server::setResource('execute', function () {
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $build->getAttribute('path', ''),
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);

3
bin/hamster Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php hamster $@

3
bin/patch-create-missing-schedules
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-create-missing-schedules $@

3
bin/patch-delete-schedule-updated-at-attribute Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@

7
composer.json
View file

@ -44,10 +44,10 @@
"appwrite/php-clamav": "1.1.*",
"appwrite/php-runtimes": "0.11.*",
"utopia-php/abuse": "0.18.*",
"utopia-php/analytics": "0.2.*",
"utopia-php/audit": "0.20.*",
"utopia-php/analytics": "0.10.2",
"utopia-php/cache": "0.8.*",
"utopia-php/cli": "0.14.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.30.*",
"utopia-php/queue": "0.5.*",
@ -72,7 +72,8 @@
"phpmailer/phpmailer": "6.6.0",
"chillerlan/php-qrcode": "4.3.3",
"adhocore/jwt": "1.1.2",
"webonyx/graphql-php": "14.11.*"
"webonyx/graphql-php": "14.11.*",
"slickdeals/statsd": "3.1.0"
},
"repositories": [
{

263
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "60f1cefcf6d210ed231acb4a99f2f20c",
"content-hash": "1614e987617e0a710448662d1d9cc7ef",
"packages": [
{
"name": "adhocore/jwt",
@ -998,6 +998,62 @@
},
"time": "2020-04-16T16:39:50+00:00"
},
{
"name": "slickdeals/statsd",
"version": "3.1.0",
"source": {
"type": "git",
"url": "https://github.com/Slickdeals/statsd-php.git",
"reference": "225588a0a079e145359049f6e5e23eedb1b4c17f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Slickdeals/statsd-php/zipball/225588a0a079e145359049f6e5e23eedb1b4c17f",
"reference": "225588a0a079e145359049f6e5e23eedb1b4c17f",
"shasum": ""
},
"require": {
"php": ">= 7.3 || ^8"
},
"replace": {
"domnikl/statsd": "self.version"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.0",
"phpunit/phpunit": "^9",
"vimeo/psalm": "^4.6"
},
"type": "library",
"autoload": {
"psr-4": {
"Domnikl\\Statsd\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Dominik Liebler",
"email": "liebler.dominik@gmail.com"
}
],
"description": "a PHP client for statsd",
"homepage": "https://github.com/Slickdeals/statsd-php",
"keywords": [
"Metrics",
"monitoring",
"statistics",
"statsd",
"udp"
],
"support": {
"issues": "https://github.com/Slickdeals/statsd-php/issues",
"source": "https://github.com/Slickdeals/statsd-php/tree/3.1.0"
},
"time": "2021-06-04T20:33:46+00:00"
},
{
"name": "symfony/polyfill-php80",
"version": "v1.27.0",
@ -1133,24 +1189,25 @@
},
{
"name": "utopia-php/analytics",
"version": "0.2.0",
"version": "0.10.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/analytics.git",
"reference": "adfc2d057a7f6ab618a77c8a20ed3e35485ff416"
"reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/analytics/zipball/adfc2d057a7f6ab618a77c8a20ed3e35485ff416",
"reference": "adfc2d057a7f6ab618a77c8a20ed3e35485ff416",
"url": "https://api.github.com/repos/utopia-php/analytics/zipball/14c805114736f44c26d6d24b176a2f8b93d86a1f",
"reference": "14c805114736f44c26d6d24b176a2f8b93d86a1f",
"shasum": ""
},
"require": {
"php": ">=7.4"
"php": ">=8.0",
"utopia-php/cli": "^0.15.0"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.0.1"
"laravel/pint": "dev-main",
"phpunit/phpunit": "^9.3"
},
"type": "library",
"autoload": {
@ -1162,16 +1219,6 @@
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
},
{
"name": "Torsten Dittmann",
"email": "torsten@appwrite.io"
}
],
"description": "A simple library to track events & users.",
"keywords": [
"analytics",
@ -1182,9 +1229,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/analytics/issues",
"source": "https://github.com/utopia-php/analytics/tree/0.2.0"
"source": "https://github.com/utopia-php/analytics/tree/0.10.2"
},
"time": "2021-03-23T21:33:07+00:00"
"time": "2023-03-22T12:01:09+00:00"
},
{
"name": "utopia-php/audit",
@ -1286,16 +1333,16 @@
},
{
"name": "utopia-php/cli",
"version": "0.14.0",
"version": "0.15.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cli.git",
"reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086"
"reference": "ccb7c8125ffe0254fef8f25744bfa376eb7bd0ea"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/c30ef985a4e739758a0d95eb0706b357b6d8c086",
"reference": "c30ef985a4e739758a0d95eb0706b357b6d8c086",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/ccb7c8125ffe0254fef8f25744bfa376eb7bd0ea",
"reference": "ccb7c8125ffe0254fef8f25744bfa376eb7bd0ea",
"shasum": ""
},
"require": {
@ -1303,8 +1350,10 @@
"utopia-php/framework": "0.*.*"
},
"require-dev": {
"laravel/pint": "1.2.*",
"phpunit/phpunit": "^9.3",
"squizlabs/php_codesniffer": "^3.6"
"squizlabs/php_codesniffer": "^3.6",
"vimeo/psalm": "4.0.1"
},
"type": "library",
"autoload": {
@ -1316,12 +1365,6 @@
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
}
],
"description": "A simple CLI library to manage command line applications",
"keywords": [
"cli",
@ -1333,9 +1376,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/cli/issues",
"source": "https://github.com/utopia-php/cli/tree/0.14.0"
"source": "https://github.com/utopia-php/cli/tree/0.15.0"
},
"time": "2022-10-09T10:19:07+00:00"
"time": "2023-03-01T05:55:14+00:00"
},
{
"name": "utopia-php/config",
@ -1853,23 +1896,24 @@
},
{
"name": "utopia-php/orchestration",
"version": "0.9.0",
"version": "0.9.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/orchestration.git",
"reference": "1d4f66684b8c4927f31b695817eae6d84aafd172"
"reference": "55f43513b3f940a3f4f9c2cde7682d0c2581beb0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/1d4f66684b8c4927f31b695817eae6d84aafd172",
"reference": "1d4f66684b8c4927f31b695817eae6d84aafd172",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/55f43513b3f940a3f4f9c2cde7682d0c2581beb0",
"reference": "55f43513b3f940a3f4f9c2cde7682d0c2581beb0",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/cli": "0.14.*"
"utopia-php/cli": "0.15.*"
},
"require-dev": {
"laravel/pint": "^1.2",
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.0.1"
},
@ -1896,34 +1940,34 @@
],
"support": {
"issues": "https://github.com/utopia-php/orchestration/issues",
"source": "https://github.com/utopia-php/orchestration/tree/0.9.0"
"source": "https://github.com/utopia-php/orchestration/tree/0.9.1"
},
"time": "2022-11-09T17:38:00+00:00"
"time": "2023-03-17T15:05:06+00:00"
},
{
"name": "utopia-php/platform",
"version": "0.3.2",
"version": "0.3.3",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/platform.git",
"reference": "a53997f1882dc16ff67968aa5f436364c8c8043e"
"reference": "a9e7a501f33e0da59779782359a747cb8d34cf6f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/platform/zipball/a53997f1882dc16ff67968aa5f436364c8c8043e",
"reference": "a53997f1882dc16ff67968aa5f436364c8c8043e",
"url": "https://api.github.com/repos/utopia-php/platform/zipball/a9e7a501f33e0da59779782359a747cb8d34cf6f",
"reference": "a9e7a501f33e0da59779782359a747cb8d34cf6f",
"shasum": ""
},
"require": {
"ext-json": "*",
"ext-redis": "*",
"php": ">=8.0",
"utopia-php/cli": "0.14.*",
"utopia-php/cli": "0.15.*",
"utopia-php/framework": "0.26.*"
},
"require-dev": {
"phpunit/phpunit": "^9.3",
"squizlabs/php_codesniffer": "^3.6"
"laravel/pint": "1.2.*",
"phpunit/phpunit": "^9.3"
},
"type": "library",
"autoload": {
@ -1945,9 +1989,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/platform/issues",
"source": "https://github.com/utopia-php/platform/tree/0.3.2"
"source": "https://github.com/utopia-php/platform/tree/0.3.3"
},
"time": "2023-02-14T04:51:46+00:00"
"time": "2023-03-07T08:52:22+00:00"
},
{
"name": "utopia-php/pools",
@ -2055,21 +2099,21 @@
},
{
"name": "utopia-php/queue",
"version": "0.5.1",
"version": "0.5.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1"
"reference": "310271c5cd477541208d7fa74a4dea64df8e04a0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/4b39e1f232b2e619b0d7fb4004f2356df334ddc1",
"reference": "4b39e1f232b2e619b0d7fb4004f2356df334ddc1",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/310271c5cd477541208d7fa74a4dea64df8e04a0",
"reference": "310271c5cd477541208d7fa74a4dea64df8e04a0",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/cli": "0.14.*",
"utopia-php/cli": "0.15.*",
"utopia-php/framework": "0.*.*"
},
"require-dev": {
@ -2110,9 +2154,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/0.5.1"
"source": "https://github.com/utopia-php/queue/tree/0.5.2"
},
"time": "2022-11-16T19:47:26+00:00"
"time": "2023-03-07T08:54:10+00:00"
},
{
"name": "utopia-php/registry",
@ -2564,6 +2608,49 @@
},
"time": "2023-02-03T05:44:59+00:00"
},
{
"name": "doctrine/deprecations",
"version": "v1.0.0",
"source": {
"type": "git",
"url": "https://github.com/doctrine/deprecations.git",
"reference": "0e2a4f1f8cdfc7a92ec3b01c9334898c806b30de"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/deprecations/zipball/0e2a4f1f8cdfc7a92ec3b01c9334898c806b30de",
"reference": "0e2a4f1f8cdfc7a92ec3b01c9334898c806b30de",
"shasum": ""
},
"require": {
"php": "^7.1|^8.0"
},
"require-dev": {
"doctrine/coding-standard": "^9",
"phpunit/phpunit": "^7.5|^8.5|^9.5",
"psr/log": "^1|^2|^3"
},
"suggest": {
"psr/log": "Allows logging deprecations via PSR-3 logger implementation"
},
"type": "library",
"autoload": {
"psr-4": {
"Doctrine\\Deprecations\\": "lib/Doctrine/Deprecations"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "A small layer on top of trigger_error(E_USER_DEPRECATED) or PSR-3 logging with options to disable all deprecations or selectively for packages.",
"homepage": "https://www.doctrine-project.org/",
"support": {
"issues": "https://github.com/doctrine/deprecations/issues",
"source": "https://github.com/doctrine/deprecations/tree/v1.0.0"
},
"time": "2022-05-02T15:47:09+00:00"
},
{
"name": "doctrine/instantiator",
"version": "1.5.0",
@ -3096,24 +3183,27 @@
},
{
"name": "phpdocumentor/type-resolver",
"version": "1.6.2",
"version": "1.7.0",
"source": {
"type": "git",
"url": "https://github.com/phpDocumentor/TypeResolver.git",
"reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d"
"reference": "1534aea9bde19a5c85c5d1e1f834ab63f4c5dcf5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d",
"reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d",
"url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/1534aea9bde19a5c85c5d1e1f834ab63f4c5dcf5",
"reference": "1534aea9bde19a5c85c5d1e1f834ab63f4c5dcf5",
"shasum": ""
},
"require": {
"doctrine/deprecations": "^1.0",
"php": "^7.4 || ^8.0",
"phpdocumentor/reflection-common": "^2.0"
"phpdocumentor/reflection-common": "^2.0",
"phpstan/phpdoc-parser": "^1.13"
},
"require-dev": {
"ext-tokenizer": "*",
"phpbench/phpbench": "^1.2",
"phpstan/extension-installer": "^1.1",
"phpstan/phpstan": "^1.8",
"phpstan/phpstan-phpunit": "^1.1",
@ -3145,9 +3235,9 @@
"description": "A PSR-5 based resolver of Class names, Types and Structural Element Names",
"support": {
"issues": "https://github.com/phpDocumentor/TypeResolver/issues",
"source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2"
"source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.0"
},
"time": "2022-10-14T12:47:21+00:00"
"time": "2023-03-12T10:13:29+00:00"
},
{
"name": "phpspec/prophecy",
@ -3217,6 +3307,51 @@
},
"time": "2023-02-02T15:41:36+00:00"
},
{
"name": "phpstan/phpdoc-parser",
"version": "1.16.1",
"source": {
"type": "git",
"url": "https://github.com/phpstan/phpdoc-parser.git",
"reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e27e92d939e2e3636f0a1f0afaba59692c0bf571",
"reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571",
"shasum": ""
},
"require": {
"php": "^7.2 || ^8.0"
},
"require-dev": {
"php-parallel-lint/php-parallel-lint": "^1.2",
"phpstan/extension-installer": "^1.0",
"phpstan/phpstan": "^1.5",
"phpstan/phpstan-phpunit": "^1.1",
"phpstan/phpstan-strict-rules": "^1.0",
"phpunit/phpunit": "^9.5",
"symfony/process": "^5.2"
},
"type": "library",
"autoload": {
"psr-4": {
"PHPStan\\PhpDocParser\\": [
"src/"
]
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "PHPDoc parser with support for nullable, intersection and generic types",
"support": {
"issues": "https://github.com/phpstan/phpdoc-parser/issues",
"source": "https://github.com/phpstan/phpdoc-parser/tree/1.16.1"
},
"time": "2023-02-07T18:11:17+00:00"
},
{
"name": "phpunit/php-code-coverage",
"version": "9.2.26",
@ -5068,5 +5203,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.2.0"
"plugin-api-version": "2.3.0"
}

38
docker-compose.yml
View file

@ -54,7 +54,7 @@ services:
TESTING: true
VERSION: dev
VITE_CONSOLE_MODE: cloud
ports:
ports:
- 9501:80
networks:
- appwrite
@ -542,6 +542,40 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-hamster:
entrypoint: hamster
<<: *x-logging
container_name: appwrite-hamster
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_HAMSTER_INTERVAL
- _APP_HAMSTER_TIME
- _APP_MIXPANEL_TOKEN
appwrite-maintenance:
entrypoint: maintenance
<<: *x-logging
@ -655,7 +689,7 @@ services:
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.2.0
image: openruntimes/executor:0.1.6
networks:
- appwrite
- runtimes

4
src/Appwrite/Extend/Exception.php
View file

@ -51,6 +51,7 @@ class Exception extends \Exception
public const GENERAL_CURSOR_NOT_FOUND = 'general_cursor_not_found';
public const GENERAL_SERVER_ERROR = 'general_server_error';
public const GENERAL_PROTOCOL_UNSUPPORTED = 'general_protocol_unsupported';
public const GENERAL_CODES_DISABLED = 'general_codes_disabled';
/** Users */
public const USER_COUNT_EXCEEDED = 'user_count_exceeded';
@ -61,7 +62,7 @@ class Exception extends \Exception
public const USER_PASSWORD_RESET_REQUIRED = 'user_password_reset_required';
public const USER_EMAIL_NOT_WHITELISTED = 'user_email_not_whitelisted';
public const USER_IP_NOT_WHITELISTED = 'user_ip_not_whitelisted';
public const USER_CODE_INVALID = 'user_code_invalid';
public const USER_INVALID_CODE = 'user_invalid_code';
public const USER_INVALID_CREDENTIALS = 'user_invalid_credentials';
public const USER_ANONYMOUS_CONSOLE_PROHIBITED = 'user_anonymous_console_prohibited';
public const USER_SESSION_ALREADY_EXISTS = 'user_session_already_exists';
@ -184,6 +185,7 @@ class Exception extends \Exception
public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries';
protected $type = '';
protected $errors = [];
public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
{

21
src/Appwrite/Migration/Version/V17.php
View file

@ -80,17 +80,6 @@ class V17 extends Migration
} catch (\Throwable $th) {
Console::warning("'mimeType' from {$id}: {$th->getMessage()}");
}
break;
case 'builds':
try {
/**
* Create 'size' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'size');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'size' from {$id}: {$th->getMessage()}");
}
try {
/**
@ -167,6 +156,16 @@ class V17 extends Migration
} catch (\Throwable $th) {
Console::warning("'scheduleInternalId' from {$id}: {$th->getMessage()}");
}
try {
/**
* Delete 'scheduleUpdatedAt' attribute
*/
$this->projectDB->deleteAttribute($id, 'scheduleUpdatedAt');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'scheduleUpdatedAt' from {$id}: {$th->getMessage()}");
}
break;
case 'deployments':

5
src/Appwrite/Platform/Services/Tasks.php
View file

@ -12,6 +12,9 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
@ -25,10 +28,12 @@ class Tasks extends Service
->addAction(Version::getName(), new Version())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
->addAction(Doctor::getName(), new Doctor())
->addAction(Install::getName(), new Install())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute())
->addAction(Schedule::getName(), new Schedule())
->addAction(Migrate::getName(), new Migrate())
->addAction(SDKs::getName(), new SDKs())

420
src/Appwrite/Platform/Tasks/Hamster.php Normal file
View file

@ -0,0 +1,420 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Analytics\Adapter\Mixpanel;
use Utopia\Analytics\Event;
use Utopia\Database\Document;
use Utopia\Pools\Group;
class Hamster extends Action
{
private array $metrics = [
'usage_files' => 'files.$all.count.total',
'usage_buckets' => 'buckets.$all.count.total',
'usage_databases' => 'databases.$all.count.total',
'usage_documents' => 'documents.$all.count.total',
'usage_collections' => 'collections.$all.count.total',
'usage_storage' => 'project.$all.storage.size',
'usage_requests' => 'project.$all.network.requests',
'usage_bandwidth' => 'project.$all.network.bandwidth',
'usage_users' => 'users.$all.count.total',
'usage_sessions' => 'sessions.email.requests.create',
'usage_executions' => 'executions.$all.compute.total',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
protected Mixpanel $mixpanel;
public static function getName(): string
{
return 'hamster';
}
public function __construct()
{
$this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', ''));
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole) {
$this->action($pools, $cache, $dbForConsole);
});
}
private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
{
$this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
Console::info("Skipping project console");
return;
}
Console::log("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = [];
$statsPerProject['time'] = microtime(true);
/** Get Project ID */
$statsPerProject['project_id'] = $project->getId();
/** Get project created time */
$statsPerProject['project_created'] = $project->getAttribute('$createdAt');
/** Get Project Name */
$statsPerProject['project_name'] = $project->getAttribute('name');
/** Get Total Functions */
$statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
/** Get Total Deployments */
$statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
/** Get Total Teams */
$statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
], APP_LIMIT_COUNT);
} else {
$statsPerProject['custom_organization_members'] = 0;
}
/** Get Email and Name of the project owner */
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
throw new Exception('Membership not found. Skipping project : ' . $project->getId());
}
$userInternalId = $membership->getAttribute('userInternalId', null);
if ($userInternalId) {
$user = $dbForConsole->findOne('users', [
Query::equal('_id', [$userInternalId]),
]);
$statsPerProject['email'] = $user->getAttribute('email', null);
$statsPerProject['name'] = $user->getAttribute('name', null);
}
}
/** Get Domains */
$statsPerProject['custom_domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
/** Get Platforms */
$platforms = $dbForConsole->find('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
$statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'web';
}));
$statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'android';
}));
$statsPerProject['custom_platforms_iOS'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'apple');
}));
$statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'flutter');
}));
/** Get Usage $statsPerProject */
$periods = [
'infinity' => [
'period' => '1d',
'limit' => 90,
],
'24h' => [
'period' => '1h',
'limit' => 24,
],
];
Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
foreach ($this->metrics as $key => $metric) {
foreach ($periods as $periodKey => $periodValue) {
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$statsPerProject[$key . '_' . $periodKey] = [];
foreach ($requestDocs as $requestDoc) {
$statsPerProject[$key . '_' . $periodKey][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
// Calculate aggregate of each metric
$statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
}
}
});
if (isset($statsPerProject['email'])) {
/** Send data to mixpanel */
$res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
'name' => $statsPerProject['name'],
'email' => $statsPerProject['email']
]);
if (!$res) {
Console::error('Failed to create user profile for project: ' . $project->getId());
}
$event = new Event();
$event
->setName('Project Daily Usage')
->setProps($statsPerProject);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
Console::error('Failed to create event for project: ' . $project->getId());
}
}
} catch (Exception $e) {
Console::error('Failed to send stats for project: ' . $project->getId());
Console::error($e->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
});
}
public function action(Group $pools, Cache $cache, Database $dbForConsole): void
{
Console::title('Cloud Hamster V1');
Console::success(APP_NAME . ' cloud hamster process has started');
$sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
$jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
$now = new \DateTime();
$now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$next = new \DateTime($now->format("Y-m-d $jobInitTime"));
$next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$delay = $next->getTimestamp() - $now->getTimestamp();
/**
* If time passed for the target day.
*/
if ($delay <= 0) {
$next->add(\DateInterval::createFromDateString('1 days'));
$delay = $next->getTimestamp() - $now->getTimestamp();
}
Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds");
$loopStart = microtime(true);
/* Initialise new Utopia app */
$app = new App('UTC');
Console::info('Getting stats for all projects');
$this->getStatsPerProject($pools, $cache, $dbForConsole);
Console::success('Completed getting stats for all projects');
Console::info('Getting stats for all organizations');
$this->getStatsPerOrganization($dbForConsole);
Console::success('Completed getting stats for all organizations');
Console::info('Getting stats for all users');
$this->getStatsPerUser($dbForConsole);
Console::success('Completed getting stats for all users');
$pools
->get('console')
->reclaim();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
}, $sleep, $delay);
}
protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback)
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $dbForConsole->find($collection, \array_merge([
Query::limit($limit),
Query::offset($count)
]));
$sum = count($results);
Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
call_user_func($callback, $dbForConsole, $document);
$count++;
}
}
$executionEnd = \microtime(true);
Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
protected function getStatsPerOrganization(Database $dbForConsole)
{
$this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
try {
$statsPerOrganization = [];
/** Organization name */
$statsPerOrganization['name'] = $document->getAttribute('name');
/** Get Email and of the organization owner */
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$document->getInternalId()]),
]);
if (!$membership || $membership->isEmpty()) {
throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
}
$userInternalId = $membership->getAttribute('userInternalId', null);
if ($userInternalId) {
$user = $dbForConsole->findOne('users', [
Query::equal('_id', [$userInternalId]),
]);
$statsPerOrganization['email'] = $user->getAttribute('email', null);
}
/** Organization Creation Date */
$statsPerOrganization['created'] = $document->getAttribute('$createdAt');
/** Number of team members */
$statsPerOrganization['members'] = $document->getAttribute('total');
/** Number of projects in this organization */
$statsPerOrganization['projects'] = $dbForConsole->count('projects', [
Query::equal('teamId', [$document->getId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerOrganization['email'])) {
throw new Exception('Email not found. Skipping organization : ' . $document->getId());
}
$event = new Event();
$event
->setName('Organization Daily Usage')
->setProps($statsPerOrganization);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new Exception('Failed to create event for organization : ' . $document->getId());
}
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
protected function getStatsPerUser(Database $dbForConsole)
{
$this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
try {
$statsPerUser = [];
/** Organization name */
$statsPerUser['name'] = $document->getAttribute('name');
/** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
$statsPerUser['email'] = $document->getAttribute('email');
/** Organization Creation Date */
$statsPerUser['created'] = $document->getAttribute('$createdAt');
/** Number of teams this user is a part of */
$statsPerUser['memberships'] = $dbForConsole->count('memberships', [
Query::equal('userInternalId', [$document->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerUser['email'])) {
throw new Exception('User has no email: ' . $document->getId());
}
/** Send data to mixpanel */
$event = new Event();
$event
->setName('User Daily Usage')
->setProps($statsPerUser);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new Exception('Failed to create user profile for user: ' . $document->getId());
}
} catch (Exception $e) {
Console::error($e->getMessage());
}
});
}
}

74
src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php Normal file
View file

@ -0,0 +1,74 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Pools\Group;
class PatchDeleteScheduleUpdatedAtAttribute extends Action
{
public static function getName(): string
{
return 'patch-delete-schedule-updated-at-attribute';
}
public function __construct()
{
$this
->desc('Ensure function collections do not have scheduleUpdatedAt attribute')
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchDeleteScheduleUpdatedAtAttribute V1');
Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
try {
/**
* Delete 'scheduleUpdatedAt' attribute
*/
$dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt');
$dbForProject->deleteCachedCollection('functions');
Console::success("'scheduleUpdatedAt' deleted.");
} catch (\Throwable $th) {
Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}");
}
$pools->reclaim();
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}

12
src/Appwrite/Utopia/Response/Model/Build.php
View file

@ -51,18 +51,18 @@ class Build extends Model
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('endTime', [
'type' => self::TYPE_DATETIME,
'description' => 'The time the build was finished in ISO 8601 format.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('duration', [
'type' => self::TYPE_INTEGER,
'description' => 'The build duration in seconds.',
'default' => 0,
'example' => 0,
])
->addRule('size', [
'type' => self::TYPE_INTEGER,
'description' => 'The code size in bytes.',
'default' => 0,
'example' => 128,
])
;
}

4
tests/e2e/Services/Account/AccountConsoleClientTest.php
View file

@ -39,7 +39,7 @@ class AccountConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 401);
$this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID);
$this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE);
$response = $this->client->call(Client::METHOD_POST, '/account/invite', array_merge([
'origin' => 'http://localhost',
@ -53,7 +53,7 @@ class AccountConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 401);
$this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID);
$this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE);
/**
* Test for SUCCESS