CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-16 10:42:34 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'refactor-usage-sn' into refactor-usage-sn-1.5.x

Browse source
This commit is contained in:
Matej Bačo 2024-04-02 18:42:17 +02:00
parent 61f04d5340 a792f3de1d
commit d091b37a15
47 changed files with 423 additions and 2677 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
Dockerfile
View file

@ -104,30 +104,9 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-hamster && \
chmod +x /usr/local/bin/worker-usage && \
chmod +x /usr/local/bin/worker-usage-dump
# Cloud Executabless
RUN chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/delete-orphaned-projects && \
chmod +x /usr/local/bin/get-migration-stats && \
chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/delete-orphaned-projects && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/get-migration-stats && \
chmod +x /usr/local/bin/create-inf-metric
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

3
app/cli.php
View file

@ -131,9 +131,6 @@ CLI::setResource('queue', function (Group $pools) {
CLI::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
CLI::setResource('queueForHamster', function (Connection $queue) {
return new Hamster($queue);
}, ['queue']);
CLI::setResource('queueForDeletes', function (Connection $queue) {
return new Delete($queue);
}, ['queue']);

2
app/config/errors.php
View file

@ -705,7 +705,7 @@ return [
Exception::PROJECT_PROVIDER_UNSUPPORTED => [
'name' => Exception::PROJECT_PROVIDER_UNSUPPORTED,
'description' => 'The chosen OAuth provider is unsupported. Please check the <a href="/docs/client/account?sdk=web-default#accountCreateOAuth2Session">Create OAuth2 Session docs</a> for the complete list of supported OAuth providers.',
'code' => 501,
'code' => 400,
],
Exception::PROJECT_INVALID_SUCCESS_URL => [
'name' => Exception::PROJECT_INVALID_SUCCESS_URL,

2
app/controllers/api/account.php
View file

@ -2613,7 +2613,7 @@ App::patch('/v1/account/email')
// Makes sure this email is not already used in another identity
$identityWithMatchingEmail = $dbForProject->findOne('identities', [
Query::equal('providerEmail', [$email]),
Query::notEqual('userId', $user->getId()),
Query::notEqual('userInternalId', $user->getInternalId()),
]);
if ($identityWithMatchingEmail !== false && !$identityWithMatchingEmail->isEmpty()) {
throw new Exception(Exception::GENERAL_BAD_REQUEST); /** Return a generic bad request to prevent exposing existing accounts */

75
app/controllers/api/avatars.php
View file

@ -24,6 +24,9 @@ use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\URL;
use Utopia\Validator\WhiteList;
use chillerlan\QRCode\QRCode;
use chillerlan\QRCode\QROptions;
use Utopia\Fetch\Client;
$avatarCallback = function (string $type, string $code, int $width, int $height, int $quality, Response $response) {
@ -284,14 +287,21 @@ App::get('/v1/avatars/image')
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
$fetch = @\file_get_contents($url);
$client = new Client();
try {
$res = $client
->setAllowRedirects(false)
->fetch($url);
} catch (\Throwable) {
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
if (!$fetch) {
if ($res->getStatusCode() !== 200) {
throw new Exception(Exception::AVATAR_IMAGE_NOT_FOUND);
}
try {
$image = new Image($fetch);
$image = new Image($res->getBody());
} catch (\Throwable $exception) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to parse image');
}
@ -340,31 +350,27 @@ App::get('/v1/avatars/favicon')
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
$curl = \curl_init();
$client = new Client();
try {
$res = $client
->setAllowRedirects(false)
->setUserAgent(\sprintf(
APP_USERAGENT,
App::getEnv('_APP_VERSION', 'UNKNOWN'),
App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)
))
->fetch($url);
} catch (\Throwable) {
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
\curl_setopt_array($curl, [
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 3,
CURLOPT_URL => $url,
CURLOPT_USERAGENT => \sprintf(
APP_USERAGENT,
System::getEnv('_APP_VERSION', 'UNKNOWN'),
System::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS', APP_EMAIL_SECURITY)
),
]);
$html = \curl_exec($curl);
\curl_close($curl);
if (!$html) {
if ($res->getStatusCode() !== 200) {
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
$doc = new DOMDocument();
$doc->strictErrorChecking = false;
@$doc->loadHTML($html);
@$doc->loadHTML($res->getBody());
$links = $doc->getElementsByTagName('link');
$outputHref = '';
@ -419,9 +425,22 @@ App::get('/v1/avatars/favicon')
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
if ('ico' == $outputExt) { // Skip crop, Imagick isn\'t supporting icon files
$data = @\file_get_contents($outputHref, false);
$client = new Client();
try {
$res = $client
->setAllowRedirects(false)
->fetch($outputHref);
} catch (\Throwable) {
throw new Exception(Exception::AVATAR_REMOTE_URL_FAILED);
}
if ($res->getStatusCode() !== 200) {
throw new Exception(Exception::AVATAR_ICON_NOT_FOUND);
}
$data = $res->getBody();
if ('ico' == $outputExt) { // Skip crop, Imagick isn\'t supporting icon files
if (empty($data) || (\mb_substr($data, 0, 5) === '<html') || \mb_substr($data, 0, 5) === '<!doc') {
throw new Exception(Exception::AVATAR_ICON_NOT_FOUND, 'Favicon not found');
}
@ -431,13 +450,7 @@ App::get('/v1/avatars/favicon')
->file($data);
}
$fetch = @\file_get_contents($outputHref, false);
if (!$fetch) {
throw new Exception(Exception::AVATAR_ICON_NOT_FOUND);
}
$image = new Image($fetch);
$image = new Image($data);
$image->crop((int) $width, (int) $height);
$output = (empty($output)) ? $type : $output;
$data = $image->output($output, $quality);

50
app/controllers/api/databases.php
View file

@ -24,7 +24,6 @@ use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Conflict as ConflictException;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Database\Exception\Limit as LimitException;
use Utopia\Database\Exception\Query as QueryException;
use Utopia\Database\Exception\Restricted as RestrictedException;
use Utopia\Database\Exception\Structure as StructureException;
use Utopia\Database\Helpers\ID;
@ -666,16 +665,10 @@ App::put('/v1/databases/:databaseId')
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
try {
$database = $dbForProject->updateDocument('databases', $databaseId, $database
->setAttribute('name', $name)
->setAttribute('enabled', $enabled)
->setAttribute('search', implode(' ', [$databaseId, $name])));
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (StructureException $exception) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage());
}
$database = $dbForProject->updateDocument('databases', $databaseId, $database
->setAttribute('name', $name)
->setAttribute('enabled', $enabled)
->setAttribute('search', implode(' ', [$databaseId, $name])));
$queueForEvents->setParam('databaseId', $database->getId());
@ -1036,19 +1029,14 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
$enabled ??= $collection->getAttribute('enabled', true);
try {
$collection = $dbForProject->updateDocument('database_' . $database->getInternalId(), $collectionId, $collection
->setAttribute('name', $name)
->setAttribute('$permissions', $permissions)
->setAttribute('documentSecurity', $documentSecurity)
->setAttribute('enabled', $enabled)
->setAttribute('search', implode(' ', [$collectionId, $name])));
$dbForProject->updateCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $permissions, $documentSecurity);
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (StructureException $exception) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, 'Bad structure. ' . $exception->getMessage());
}
$collection = $dbForProject->updateDocument('database_' . $database->getInternalId(), $collectionId, $collection
->setAttribute('name', $name)
->setAttribute('$permissions', $permissions)
->setAttribute('documentSecurity', $documentSecurity)
->setAttribute('enabled', $enabled)
->setAttribute('search', implode(' ', [$collectionId, $name])));
$dbForProject->updateCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $permissions, $documentSecurity);
$queueForEvents
->setContext('database', $database)
@ -3599,16 +3587,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
}
$dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) {
try {
$dbForProject->deleteDocument(
'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(),
$documentId
);
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (RestrictedException) {
throw new Exception(Exception::DOCUMENT_DELETE_RESTRICTED);
}
$dbForProject->deleteDocument(
'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(),
$documentId
);
});
// Add $collectionId and $databaseId for all documents

3
app/controllers/api/health.php
View file

@ -856,8 +856,7 @@ App::get('/v1/health/queue/failed/:name')
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,
Event::MESSAGING_QUEUE_NAME,
Event::MIGRATIONS_QUEUE_NAME,
Event::HAMSTER_CLASS_NAME
Event::MIGRATIONS_QUEUE_NAME
]), 'The name of the queue')
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md')

6
app/controllers/api/projects.php
View file

@ -5,6 +5,7 @@ use Appwrite\Event\Delete;
use Appwrite\Event\Mail;
use Appwrite\Event\Validator\Event;
use Appwrite\Extend\Exception;
use Appwrite\Hooks\Hooks;
use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Origin;
use Appwrite\Template\Template;
@ -77,7 +78,8 @@ App::post('/v1/projects')
->inject('dbForConsole')
->inject('cache')
->inject('pools')
->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) {
->inject('hooks')
->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools, Hooks $hooks) {
$team = $dbForConsole->getDocument('teams', $teamId);
@ -224,6 +226,8 @@ App::post('/v1/projects')
$dbForProject->createCollection($key, $attributes, $indexes);
}
$hooks->trigger('afterProjectCreation', [ $project, $pools, $cache ]);
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($project, Response::MODEL_PROJECT);

190
app/controllers/api/storage.php
View file

@ -588,111 +588,95 @@ App::post('/v1/storage/buckets/:bucketId/files')
$openSSLIV = \bin2hex($iv);
}
try {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getInternalId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $fileSize,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
if ($file->isEmpty()) {
$doc = new Document([
'$id' => $fileId,
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getInternalId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
'sizeOriginal' => $fileSize,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
} else {
$file = $file
->setAttribute('$permissions', $permissions)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV)
->setAttribute('metadata', $metadata)
->setAttribute('chunksUploaded', $chunksUploaded);
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
} else {
$file = $file
->setAttribute('$permissions', $permissions)
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
->setAttribute('openSSLIV', $openSSLIV)
->setAttribute('metadata', $metadata)
->setAttribute('chunksUploaded', $chunksUploaded);
/**
* Validate create permission and skip authorization in updateDocument
* Without this, the file creation will fail when user doesn't have update permission
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we validate create permission instead of update
*/
$validator = new Authorization(Database::PERMISSION_CREATE);
if (!$validator->isValid($bucket->getCreate())) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
/**
* Validate create permission and skip authorization in updateDocument
* Without this, the file creation will fail when user doesn't have update permission
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we validate create permission instead of update
*/
$validator = new Authorization(Database::PERMISSION_CREATE);
if (!$validator->isValid($bucket->getCreate())) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (StructureException $exception) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage());
} catch (DuplicateException) {
throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS);
$file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
}
} else {
try {
if ($file->isEmpty()) {
$doc = new Document([
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getInternalId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
if ($file->isEmpty()) {
$doc = new Document([
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getInternalId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName]),
'metadata' => $metadata,
]);
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
} else {
$file = $file
->setAttribute('chunksUploaded', $chunksUploaded)
->setAttribute('metadata', $metadata);
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
} else {
$file = $file
->setAttribute('chunksUploaded', $chunksUploaded)
->setAttribute('metadata', $metadata);
/**
* Validate create permission and skip authorization in updateDocument
* Without this, the file creation will fail when user doesn't have update permission
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we validate create permission instead of update
*/
$validator = new Authorization(Database::PERMISSION_CREATE);
if (!$validator->isValid($bucket->getCreate())) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
/**
* Validate create permission and skip authorization in updateDocument
* Without this, the file creation will fail when user doesn't have update permission
* However as with chunk upload even if we are updating, we are essentially creating a file
* adding it's new chunk so we validate create permission instead of update
*/
$validator = new Authorization(Database::PERMISSION_CREATE);
if (!$validator->isValid($bucket->getCreate())) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (StructureException $exception) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage());
} catch (DuplicateException) {
throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS);
$file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
}
}
@ -1554,11 +1538,7 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
}
if ($fileSecurity && !$valid) {
try {
$file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
} else {
$file = Authorization::skip(fn () => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
}
@ -1643,11 +1623,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
;
if ($fileSecurity && !$valid) {
try {
$deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId);
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId);
} else {
$deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId));
}

2
app/controllers/api/teams.php
View file

@ -948,7 +948,7 @@ App::patch('/v1/teams/:teamId/memberships/:membershipId/status')
throw new Exception(Exception::MEMBERSHIP_NOT_FOUND);
}
$team = Authorization::skip(fn () => $dbForProject->getDocument('teams', $teamId));
$team = Authorization::skip(fn() => $dbForProject->getDocument('teams', $teamId));
if ($team->isEmpty()) {
throw new Exception(Exception::TEAM_NOT_FOUND);

2
app/controllers/api/vcs.php
View file

@ -102,7 +102,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId
$latestCommentId = '';
if (!empty($providerPullRequestId)) {
if (!empty($providerPullRequestId) && $function->getAttribute('providerSilentMode', false) === false) {
$latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::equal('providerPullRequestId', [$providerPullRequestId]),

122
app/controllers/general.php
View file

@ -157,7 +157,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
$permissions = $function->getAttribute('execute');
if (!(\in_array('any', $permissions)) && (\in_array('guests', $permissions))) {
if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) {
throw new AppwriteException(AppwriteException::USER_UNAUTHORIZED, 'To execute function using domain, execute permissions must include "any" or "guests"');
}
@ -616,6 +616,64 @@ App::error()
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
$version = System::getEnv('_APP_VERSION', 'UNKNOWN');
$route = $utopia->getRoute();
$class = \get_class($error);
$code = $error->getCode();
$message = $error->getMessage();
$file = $error->getFile();
$line = $error->getLine();
$trace = $error->getTrace();
if (php_sapi_name() === 'cli') {
Console::error('[Error] Timestamp: ' . date('c', time()));
if ($route) {
Console::error('[Error] Method: ' . $route->getMethod());
Console::error('[Error] URL: ' . $route->getPath());
}
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $message);
Console::error('[Error] File: ' . $file);
Console::error('[Error] Line: ' . $line);
}
switch ($class) {
case 'Utopia\Exception':
$error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error);
switch ($code) {
case 400:
$error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID);
break;
case 404:
$error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND);
break;
}
break;
case 'Utopia\Database\Exception\Conflict':
$error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error);
break;
case 'Utopia\Database\Exception\Timeout':
$error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error);
break;
case 'Utopia\Database\Exception\Query':
$error = new AppwriteException(AppwriteException::GENERAL_QUERY_INVALID, $error->getMessage(), previous: $error);
break;
case 'Utopia\Database\Exception\Structure':
$error = new AppwriteException(AppwriteException::DOCUMENT_INVALID_STRUCTURE, $error->getMessage(), previous: $error);
break;
case 'Utopia\Database\Exception\Duplicate':
$error = new AppwriteException(AppwriteException::DOCUMENT_ALREADY_EXISTS);
break;
case 'Utopia\Database\Exception\Restricted':
$error = new AppwriteException(AppwriteException::DOCUMENT_DELETE_RESTRICTED);
break;
case 'Utopia\Database\Exception\Authorization':
$error = new AppwriteException(AppwriteException::USER_UNAUTHORIZED);
break;
}
$code = $error->getCode();
$message = $error->getMessage();
if ($error instanceof AppwriteException) {
$publish = $error->isPublishable();
@ -623,11 +681,28 @@ App::error()
$publish = $error->getCode() === 0 || $error->getCode() >= 500;
}
if ($logger && ($publish || $error->getCode() === 0)) {
if ($error->getCode() >= 400 && $error->getCode() < 500) {
// Register error logger
$providerName = App::getEnv('_APP_EXPERIMENT_LOGGING_PROVIDER', '');
$providerConfig = App::getEnv('_APP_EXPERIMENT_LOGGING_CONFIG', '');
if (!(empty($providerName) || empty($providerConfig))) {
if (!Logger::hasProvider($providerName)) {
throw new Exception("Logging provider not supported. Logging is disabled");
}
$classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
$adapter = new $classname($providerConfig);
$logger = new Logger($adapter);
$publish = true;
}
}
if ($logger && $publish) {
try {
/** @var Utopia\Database\Document $user */
$user = $utopia->getResource('user');
} catch (\Throwable $th) {
} catch (\Throwable) {
// All good, user is optional information for logger
}
@ -666,47 +741,6 @@ App::error()
Console::info('Log pushed with status code: ' . $responseCode);
}
$code = $error->getCode();
$message = $error->getMessage();
$file = $error->getFile();
$line = $error->getLine();
$trace = $error->getTrace();
if (php_sapi_name() === 'cli') {
Console::error('[Error] Timestamp: ' . date('c', time()));
if ($route) {
Console::error('[Error] Method: ' . $route->getMethod());
Console::error('[Error] URL: ' . $route->getPath());
}
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $message);
Console::error('[Error] File: ' . $file);
Console::error('[Error] Line: ' . $line);
}
/** Handle Utopia Errors */
if ($error instanceof Utopia\Exception) {
$error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error);
switch ($code) {
case 400:
$error->setType(AppwriteException::GENERAL_ARGUMENT_INVALID);
break;
case 404:
$error->setType(AppwriteException::GENERAL_ROUTE_NOT_FOUND);
break;
}
} elseif ($error instanceof Utopia\Database\Exception\Conflict) {
$error = new AppwriteException(AppwriteException::DOCUMENT_UPDATE_CONFLICT, previous: $error);
$code = $error->getCode();
$message = $error->getMessage();
} elseif ($error instanceof Utopia\Database\Exception\Timeout) {
$error = new AppwriteException(AppwriteException::DATABASE_TIMEOUT, previous: $error);
$code = $error->getCode();
$message = $error->getMessage();
}
/** Wrap all exceptions inside Appwrite\Extend\Exception */
if (!($error instanceof AppwriteException)) {
$error = new AppwriteException(AppwriteException::GENERAL_UNKNOWN, $message, $code, $error);

10
app/init.php
View file

@ -1187,7 +1187,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
}
if (APP_MODE_ADMIN === $mode) {
if ($user->find('teamId', $project->getAttribute('teamId'), 'memberships')) {
if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) {
Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users.
} else {
$user = new Document([]);
@ -1441,7 +1441,9 @@ function getDevice($root): Device
case Storage::DEVICE_S3:
return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE::DEVICE_DO_SPACES:
return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
$device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
$device->setHttpVersion(S3::HTTP_VERSION_1_1);
return $device;
case Storage::DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LINODE:
@ -1470,7 +1472,9 @@ function getDevice($root): Device
$doSpacesRegion = System::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = System::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
$device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
$device->setHttpVersion(S3::HTTP_VERSION_1_1);
return $device;
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = System::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');

133
app/realtime.php
View file

@ -38,74 +38,83 @@ require_once __DIR__ . '/init.php';
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
function getConsoleDB(): Database
{
global $register;
// Allows overriding
if (!function_exists("getConsoleDB")) {
function getConsoleDB(): Database
{
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools');
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools');
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', '_console');
return $database;
}
function getProjectDB(Document $project): Database
{
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools');
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database
->setNamespace('_' . $project->getInternalId())
->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId());
return $database;
}
function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
$database = new Database($dbAdapter, getCache());
$database
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', '_console');
return $database;
}
}
// Allows overriding
if (!function_exists("getProjectDB")) {
function getProjectDB(Document $project): Database
{
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools');
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database
->setNamespace('_' . $project->getInternalId())
->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId());
return $database;
}
}
// Allows overriding
if (!function_exists("getCache")) {
function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}
}
$realtime = new Realtime();

47
app/views/general/error.phtml
View file

@ -59,7 +59,7 @@ $title = $this->getParam('title', '')
<link rel="stylesheet" href="/fonts/main.css" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta http-equiv="content-security-policy" content="">
<title><?php echo $this->print($title, self::FILTER_ESCAPE); ?></title>
<title><?php echo $this->print($title); ?></title>
<style>
@media(min-width:768px) {
@ -74,11 +74,11 @@ $title = $this->getParam('title', '')
<div class="container u-margin-block-start-24">
<article class="card u-padding-16">
<div class="u-flex u-flex-vertical u-gap-16">
<h1 class="heading-level-4 u-trim-1">Error <?php echo $this->print($code, self::FILTER_ESCAPE); ?></h1>
<p class="text"><?php echo $this->print($message, self::FILTER_ESCAPE); ?></p>
<h1 class="heading-level-4 u-trim-1">Error <?php echo $this->print($code); ?></h1>
<p class="text"><?php echo $this->print($message); ?></p>
<div class="u-flex u-flex-vertical u-gap-8">
<p class="text">Type</p>
<p><code class="inline-code"><?php echo $this->print($type, self::FILTER_ESCAPE); ?></code></p>
<p><code class="inline-code"><?php echo $this->print($type); ?></code></p>
</div>
<?php if ($development) : ?>
<h2 class="heading-level-5 u-trim-1">Error Trace</h2>
@ -111,45 +111,6 @@ $title = $this->getParam('title', '')
</div>
</article>
</div>
<!--section>
<h1>Error <?php echo $this->print($code, self::FILTER_ESCAPE); ?></h1>
<p><?php echo $this->print($message, self::FILTER_ESCAPE); ?></p>
<small>Error ID: <?php echo $this->print($errorID, self::FILTER_ESCAPE); ?></small>
<?php if (!empty($projectURL)) : ?>
<hr />
<p><a href="<?php echo $this->print($projectURL, self::FILTER_ESCAPE); ?>" rel="noopener">Back to <?php echo $this->print($projectName, self::FILTER_ESCAPE); ?></a></p>
<?php endif; ?>
<?php if ($development) : ?>
<div>
<h2>Error Trace</h2>
<?php foreach ($trace as $log) : ?>
<table>
<?php foreach ($log as $key => $value) : ?>
<tr>
<td style="width: 120px"><?php echo $this->print($key, self::FILTER_ESCAPE); ?></td>
<td>
<?php if (is_array($value)) : ?>
<?php echo $this->print(var_export($value, true), self::FILTER_ESCAPE); ?>
<?php else : ?>
<?php echo $this->print($value, self::FILTER_ESCAPE); ?>
<?php endif; ?>
</td>
</tr>
<?php endforeach; ?>
</table>
<?php endforeach; ?>
</div>
<?php endif; ?>
</section-->
<script type="text/javascript">
const app = (JSON.parse(localStorage.getItem('appwrite')) ?? {});
const theme = app.theme ?? 'auto';

4
app/worker.php
View file

@ -194,10 +194,6 @@ Server::setResource('queueForMigrations', function (Connection $queue) {
return new Migration($queue);
}, ['queue']);
Server::setResource('queueForHamster', function (Connection $queue) {
return new Hamster($queue);
}, ['queue']);
Server::setResource('logger', function (Registry $register) {
return $register->get('logger');
}, ['register']);

3
bin/calc-tier-stats
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-tier-stats $@

3
bin/calc-users-stats
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-users-stats $@

3
bin/clear-card-cache
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php clear-card-cache $@

3
bin/create-inf-metric
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php create-inf-metric $@

3
bin/delete-orphaned-projects
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php delete-orphaned-projects $@

3
bin/get-migration-stats
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php get-migration-stats $@

3
bin/hamster
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php hamster $@

3
bin/patch-delete-project-collections
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-project-collections $@

3
bin/patch-delete-schedule-updated-at-attribute
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@

3
bin/patch-recreate-repositories-documents
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@

3
bin/volume-sync
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php volume-sync $@

3
bin/worker-hamster
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/worker.php hamster $@

1
composer.json
View file

@ -54,6 +54,7 @@
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.*",
"utopia-php/framework": "0.33.*",
"utopia-php/fetch": "0.2.*",
"utopia-php/image": "0.6.*",
"utopia-php/locale": "0.4.*",
"utopia-php/logger": "0.3.*",

63
docker-compose.yml
View file

@ -53,7 +53,7 @@ services:
DEBUG: false
TESTING: true
VERSION: dev
ports:
ports:
- 9501:80
networks:
- appwrite
@ -188,7 +188,9 @@ services:
- _APP_MESSAGE_EMAIL_TEST_DSN
- _APP_MESSAGE_PUSH_TEST_DSN
- _APP_CONSOLE_COUNTRIES_DENYLIST
- _APP_EXPERIMENT_LOGGING_PROVIDER
- _APP_EXPERIMENT_LOGGING_CONFIG
appwrite-realtime:
entrypoint: realtime
<<: *x-logging
@ -788,63 +790,6 @@ services:
environment:
- _APP_ASSISTANT_OPENAI_API_KEY
appwrite-worker-hamster:
entrypoint: worker-hamster
<<: *x-logging
container_name: appwrite-worker-hamster
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_MIXPANEL_TOKEN
appwrite-hamster-scheduler:
entrypoint: hamster
<<: *x-logging
container_name: appwrite-hamster-scheduler
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_HAMSTER_TIME
- _APP_HAMSTER_INTERVAL
openruntimes-executor:
container_name: openruntimes-executor
hostname: appwrite-executor

3
src/Appwrite/Event/Event.php
View file

@ -45,9 +45,6 @@ class Event
public const MIGRATIONS_QUEUE_NAME = 'v1-migrations';
public const MIGRATIONS_CLASS_NAME = 'MigrationsV1';
public const HAMSTER_QUEUE_NAME = 'v1-hamster';
public const HAMSTER_CLASS_NAME = 'HamsterV1';
protected string $queue = '';
protected string $class = '';
protected string $event = '';

157
src/Appwrite/Event/Hamster.php
View file

@ -1,157 +0,0 @@
<?php
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class Hamster extends Event
{
protected string $type = '';
protected ?Document $project = null;
protected ?Document $organization = null;
protected ?Document $user = null;
public const TYPE_PROJECT = 'project';
public const TYPE_ORGANISATION = 'organisation';
public const TYPE_USER = 'user';
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
$this
->setQueue(Event::HAMSTER_QUEUE_NAME)
->setClass(Event::HAMSTER_CLASS_NAME);
}
/**
* Sets the type for the hamster event.
*
* @param string $type
* @return self
*/
public function setType(string $type): self
{
$this->type = $type;
return $this;
}
/**
* Returns the set type for the hamster event.
*
* @return string
*/
public function getType(): string
{
return $this->type;
}
/**
* Sets the project for the hamster event.
*
* @param Document $project
*/
public function setProject(Document $project): self
{
$this->project = $project;
return $this;
}
/**
* Returns the set project for the hamster event.
*
* @return Document
*/
public function getProject(): Document
{
return $this->project;
}
/**
* Sets the organization for the hamster event.
*
* @param Document $organization
*/
public function setOrganization(Document $organization): self
{
$this->organization = $organization;
return $this;
}
/**
* Returns the set organization for the hamster event.
*
* @return string
*/
public function getOrganization(): Document
{
return $this->organization;
}
/**
* Sets the user for the hamster event.
*
* @param Document $user
*/
public function setUser(Document $user): self
{
$this->user = $user;
return $this;
}
/**
* Returns the set user for the hamster event.
*
* @return Document
*/
public function getUser(): Document
{
return $this->user;
}
/**
* Executes the function event and sends it to the functions worker.
*
* @return string|bool
* @throws \InvalidArgumentException
*/
public function trigger(): string|bool
{
if ($this->paused) {
return false;
}
$client = new Client($this->queue, $this->connection);
$events = $this->getEvent() ? Event::generateEvents($this->getEvent(), $this->getParams()) : null;
return $client->enqueue([
'type' => $this->type,
'project' => $this->project,
'organization' => $this->organization,
'user' => $this->user,
'events' => $events,
]);
}
/**
* Generate a function event from a base event
*
* @param Event $event
*
* @return self
*
*/
public function from(Event $event): self
{
$this->event = $event->getEvent();
$this->params = $event->getParams();
return $this;
}
}

7
src/Appwrite/Hooks/Hooks.php
View file

@ -16,11 +16,14 @@ class Hooks
/**
* @param mixed[] $params
* @return mixed
*/
public function trigger(string $name, array $params = [])
public function trigger(string $name, array $params = []): mixed
{
if (isset(self::$hooks[$name])) {
call_user_func_array(self::$hooks[$name], $params);
return call_user_func_array(self::$hooks[$name], $params);
}
return null;
}
}

407
src/Appwrite/Platform/Tasks/CalcTierStats.php
View file

@ -1,407 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\System\System;
use Utopia\Validator\Text;
class CalcTierStats extends Action
{
/*
* Csv cols headers
*/
private array $columns = [
'Project ID',
'Organization ID',
'Organization Email',
'Organization Members',
'Teams',
'Users',
'Requests',
'Bandwidth',
'Domains',
'Api keys',
'Webhooks',
'Platforms',
'Buckets',
'Files',
'Storage (bytes)',
'Max File Size (bytes)',
'Databases',
'Functions',
'Deployments',
'Executions',
'Migrations',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
private array $usageStats = [
'network.requests' => 'Requests',
'network.inbound' => 'Inbound',
'network.outbound' => 'Outbound',
];
public static function getName(): string
{
return 'calc-tier-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->param('after', '', new Text(36), 'After cursor', true)
->param('projectId', '', new Text(36), 'Select project to validate', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('getProjectDB')
->inject('register')
->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) {
$this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register);
});
}
public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void
{
//docker compose exec -t appwrite calc-tier-stats
Console::title('Cloud free tier stats calculation V1');
Console::success(APP_NAME . ' cloud free tier stats calculation has started');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
if (!empty($projectId)) {
try {
console::log("Project " . $projectId);
$project = $dbForConsole->getDocument('projects', $projectId);
$dbForProject = call_user_func($getProjectDB, $project);
$data = $this->getData($project, $dbForConsole, $dbForProject);
$csv->insertOne($data);
$this->sendMail($register);
return;
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
}
$queries = [];
if (!empty($after)) {
Console::info("Iterating remaining projects after project with ID {$after}");
$project = $dbForConsole->getDocument('projects', $after);
$queries = [Query::cursorAfter($project)];
} else {
Console::info("Iterating all projects");
}
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) {
$projectId = $project->getId();
console::log("Project " . $projectId);
try {
$dbForProject = call_user_func($getProjectDB, $project);
$data = $this->getData($project, $dbForConsole, $dbForProject);
$csv->insertOne($data);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
});
$this->sendMail($register);
}
private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
{
$limit = 1000;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
if ($latestDocument != null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$results = $database->find('projects', $newQueries);
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
private function sendMail(Registry $register): void
{
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', System::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (\Throwable $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array
{
$stats['Project ID'] = $project->getId();
$stats['Organization ID'] = $project->getAttribute('teamId', null);
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
Console::error('Membership not found. Skipping project : ' . $project->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$stats['Organization Email'] = $user->getAttribute('email', null);
}
} else {
Console::error("Email was not found for this Organization ID :{$teamInternalId}");
}
/** Get Total Members */
if ($teamInternalId) {
$stats['Organization Members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
]);
} else {
$stats['Organization Members'] = 0;
}
/** Get Total internal Teams */
try {
$stats['Teams'] = $dbForProject->count('teams', []);
} catch (\Throwable) {
$stats['Teams'] = 0;
}
/** Get Total users */
try {
$stats['Users'] = $dbForProject->count('users', []);
} catch (\Throwable) {
$stats['Users'] = 0;
}
/** Get Usage stats */
$range = '30d';
$periods = [
'30d' => [
'period' => '1d',
'limit' => 30,
]
];
$tmp = [];
$metrics = $this->usageStats;
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
foreach ($metrics as $metric => $name) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('metric', [$metric]),
Query::equal('period', [$period]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$tmp[$metric] = [];
foreach ($requestDocs as $requestDoc) {
if (empty($requestDoc)) {
continue;
}
$tmp[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$tmp[$metric] = array_reverse($tmp[$metric]);
$tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
}
});
foreach ($tmp as $key => $value) {
$stats[$metrics[$key]] = $value;
}
/**
* Workaround to combine network.inbound+network.outbound as network.
*/
$stats['Bandwidth'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0);
unset($stats['Inbound']);
unset($stats['Outbound']);
try {
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Domains'] = 0;
}
try {
/** Get Api keys */
$stats['Api keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Api keys'] = 0;
}
try {
/** Get Webhooks */
$stats['Webhooks'] = $dbForConsole->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Webhooks'] = 0;
}
try {
/** Get Platforms */
$stats['Platforms'] = $dbForConsole->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
$stats['Platforms'] = 0;
}
/** Get Files & Buckets */
$filesCount = 0;
$filesSum = 0;
$maxFileSize = 0;
$counter = 0;
try {
$buckets = $dbForProject->find('buckets', []);
foreach ($buckets as $bucket) {
$file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
if (empty($file)) {
continue;
}
$filesSum += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
$filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
$maxFileSize = $file->getAttribute('sizeOriginal');
}
$counter++;
}
} catch (\Throwable $t) {
Console::error("Error while counting buckets: {$project->getId()}");
}
$stats['Buckets'] = $counter;
$stats['Files'] = $filesCount;
$stats['Storage (bytes)'] = $filesSum;
$stats['Max File Size (bytes)'] = $maxFileSize;
try {
/** Get Total Functions */
$stats['Databases'] = $dbForProject->count('databases', []);
} catch (\Throwable) {
$stats['Databases'] = 0;
}
/** Get Total Functions */
try {
$stats['Functions'] = $dbForProject->count('functions', []);
} catch (\Throwable) {
$stats['Functions'] = 0;
}
/** Get Total Deployments */
try {
$stats['Deployments'] = $dbForProject->count('deployments', []);
} catch (\Throwable) {
$stats['Deployments'] = 0;
}
/** Get Total Executions */
try {
$stats['Executions'] = $dbForProject->count('executions', []);
} catch (\Throwable) {
$stats['Executions'] = 0;
}
/** Get Total Migrations */
try {
$stats['Migrations'] = $dbForProject->count('migrations', []);
} catch (\Throwable) {
$stats['Migrations'] = 0;
}
return array_values($stats);
}
}

409
src/Appwrite/Platform/Tasks/CreateInfMetric.php
View file

@ -1,409 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Validator\Text;
class CreateInfMetric extends Action
{
public static function getName(): string
{
return 'create-inf-metric';
}
public function __construct()
{
$this
->desc('Create infinity stats metric')
->param('after', '', new Text(36), 'After cursor', true)
->param('projectId', '', new Text(36), 'Select project to validate', true)
->inject('getProjectDB')
->inject('dbForConsole')
->callback(function (string $after, string $projectId, callable $getProjectDB, Database $dbForConsole) {
$this->action($after, $projectId, $getProjectDB, $dbForConsole);
});
}
/**
* @throws Exception
* @throws Exception\Timeout
* @throws Exception\Query
*/
public function action(string $after, string $projectId, callable $getProjectDB, Database $dbForConsole): void
{
Console::title('Create infinity metric V1');
Console::success(APP_NAME . ' Create infinity metric started');
if (!empty($projectId)) {
try {
$project = $dbForConsole->getDocument('projects', $projectId);
$dbForProject = call_user_func($getProjectDB, $project);
$this->getUsageData($dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
} else {
$queries = [];
if (!empty($after)) {
Console::info("Iterating remaining projects after project with ID {$after}");
$project = $dbForConsole->getDocument('projects', $after);
$queries = [Query::cursorAfter($project)];
} else {
Console::info("Iterating all projects");
}
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB) {
$projectId = $project->getId();
try {
$dbForProject = call_user_func($getProjectDB, $project);
$this->getUsageData($dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
});
}
}
/**
* @param Database $database
* @param string $collection
* @param array $queries
* @param callable|null $callback
* @return void
* @throws Exception
* @throws Exception\Query
* @throws Exception\Timeout
*/
private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
{
$limit = 1000;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
if ($latestDocument != null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$results = $database->find($collection, $newQueries);
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
/**
* @param Database $dbForProject
* @param Document $project
* @return void
*/
private function getUsageData(Database $dbForProject, Document $project): void
{
try {
$this->network($dbForProject);
$this->sessions($dbForProject);
$this->users($dbForProject);
$this->teams($dbForProject);
$this->databases($dbForProject);
$this->functions($dbForProject);
$this->storage($dbForProject);
} catch (\Throwable $th) {
var_dump($th->getMessage());
}
Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
}
/**
* @param Database $dbForProject
* @param string $metric
* @param int|float $value
* @return void
* @throws Exception
* @throws Exception\Authorization
* @throws Exception\Conflict
* @throws Exception\Restricted
* @throws Exception\Structure
*/
private function createInfMetric(database $dbForProject, string $metric, int|float $value): void
{
try {
$id = \md5("_inf_{$metric}");
$dbForProject->deleteDocument('stats', $id);
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'metric' => $metric,
'period' => 'inf',
'value' => (int)$value,
'time' => null,
'region' => 'default',
]));
} catch (Duplicate $th) {
console::log("Error while creating inf metric: duplicate id {$metric} {$id}");
}
}
/**
* @param Database $dbForProject
* @param string $metric
* @return int|float
* @throws Exception
*/
protected function getFromMetric(database $dbForProject, string $metric): int|float
{
return $dbForProject->sum('stats', 'value', [
Query::equal('metric', [
$metric,
]),
Query::equal('period', ['1d']),
]);
}
/**
* @param Database $dbForProject
* @throws Exception
* @throws Exception\Authorization
* @throws Exception\Conflict
* @throws Exception\Restricted
* @throws Exception\Structure
*/
private function network(database $dbForProject)
{
$this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound'));
$this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound'));
$this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests'));
}
/**
* @throws Exception\Authorization
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception\Timeout
* @throws Exception\Structure
* @throws Exception
* @throws Exception\Query
*/
private function storage(database $dbForProject)
{
$bucketsCount = 0;
$filesCount = 0;
$filesStorageSum = 0;
$buckets = $dbForProject->find('buckets');
foreach ($buckets as $bucket) {
$files = $dbForProject->count('bucket_' . $bucket->getInternalId());
$this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files);
$filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal');
$this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage);
$bucketsCount++;
$filesCount += $files;
$filesStorageSum += $filesStorage;
}
$this->createInfMetric($dbForProject, 'buckets', $bucketsCount);
$this->createInfMetric($dbForProject, 'files', $filesCount);
$this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum);
}
/**
* @throws Exception\Authorization
* @throws Exception\Timeout
* @throws Exception\Restricted
* @throws Exception\Structure
* @throws Exception\Conflict
* @throws Exception
* @throws Exception\Query
*/
private function functions(Database $dbForProject)
{
$functionsCount = 0;
$deploymentsCount = 0;
$buildsCount = 0;
$buildsStorageSum = 0;
$buildsComputeSum = 0;
$executionsCount = 0;
$executionsComputeSum = 0;
$deploymentsStorageSum = 0;
//functions
$functions = $dbForProject->find('functions');
foreach ($functions as $function) {
//deployments
$deployments = $dbForProject->find('deployments', [
Query::equal('resourceType', ['functions']),
Query::equal('resourceInternalId', [$function->getInternalId()]),
]);
$deploymentCount = 0;
$deploymentStorageSum = 0;
foreach ($deployments as $deployment) {
//builds
$builds = $dbForProject->count('builds', [
Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
]);
$buildsCompute = $dbForProject->sum('builds', 'duration', [
Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
]);
$buildsStorage = $dbForProject->sum('builds', 'size', [
Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
]);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage);
$buildsCount += $builds;
$buildsComputeSum += $buildsCompute;
$buildsStorageSum += $buildsStorage;
$deploymentCount++;
$deploymentsCount++;
$deploymentsStorageSum += $deployment['size'];
$deploymentStorageSum += $deployment['size'];
}
$this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount);
$this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum);
//executions
$executions = $dbForProject->count('executions', [
Query::equal('functionInternalId', [$function->getInternalId()]),
]);
$executionsCompute = $dbForProject->sum('executions', 'duration', [
Query::equal('functionInternalId', [$function->getInternalId()]),
]);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions);
$this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000);
$executionsCount += $executions;
$executionsComputeSum += $executionsCompute;
$functionsCount++;
}
$this->createInfMetric($dbForProject, 'functions', $functionsCount);
$this->createInfMetric($dbForProject, 'deployments', $deploymentsCount);
$this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum);
$this->createInfMetric($dbForProject, 'builds', $buildsCount);
$this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000);
$this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum);
$this->createInfMetric($dbForProject, 'executions', $executionsCount);
$this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000);
}
/**
* @throws Exception\Authorization
* @throws Exception\Timeout
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
* @throws Exception\Query
*/
private function databases(Database $dbForProject)
{
$databasesCount = 0;
$collectionsCount = 0;
$documentsCount = 0;
$databases = $dbForProject->find('databases');
foreach ($databases as $database) {
$collectionCount = 0;
$collections = $dbForProject->find('database_' . $database->getInternalId());
foreach ($collections as $collection) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents);
$documentsCount += $documents;
$collectionCount++;
$collectionsCount++;
}
$this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount);
$this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount);
$databasesCount++;
}
$this->createInfMetric($dbForProject, 'collections', $collectionsCount);
$this->createInfMetric($dbForProject, 'databases', $databasesCount);
$this->createInfMetric($dbForProject, 'documents', $documentsCount);
}
/**
* @throws Exception\Authorization
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
*/
private function users(Database $dbForProject)
{
$users = $dbForProject->count('users');
$this->createInfMetric($dbForProject, 'users', $users);
}
/**
* @throws Exception\Authorization
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
*/
private function sessions(Database $dbForProject)
{
$users = $dbForProject->count('sessions');
$this->createInfMetric($dbForProject, 'sessions', $users);
}
/**
* @throws Exception\Authorization
* @throws Exception\Structure
* @throws Exception\Restricted
* @throws Exception\Conflict
* @throws Exception
*/
private function teams(Database $dbForProject)
{
$teams = $dbForProject->count('teams');
$this->createInfMetric($dbForProject, 'teams', $teams);
}
}

161
src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php
View file

@ -1,161 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\Validator\Boolean;
class DeleteOrphanedProjects extends Action
{
public static function getName(): string
{
return 'delete-orphaned-projects';
}
public function __construct()
{
$this
->desc('Delete orphaned projects')
->param('commit', false, new Boolean(true), 'Commit project deletion', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($commit, $pools, $cache, $dbForConsole, $register);
});
}
public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
Console::title('Delete orphaned projects V1');
Console::success(APP_NAME . ' Delete orphaned projects started');
/** @var array $collections */
$collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
$collectionsConfig = array_merge([
'audit' => [
'$id' => ID::custom('audit'),
'$collection' => Database::METADATA
],
'abuse' => [
'$id' => ID::custom('abuse'),
'$collection' => Database::METADATA
]
], $collectionsConfig);
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
$projects = [$console];
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$orphans = 1;
$cnt = 0;
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$collectionsCreated = 0;
$cnt++;
if ($dbForProject->exists($dbForProject->getDatabase(), Database::METADATA)) {
$collectionsCreated = $dbForProject->count(Database::METADATA);
}
$msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')';
if ($collectionsCreated >= count($collectionsConfig)) {
Console::log($msg . ' ignoring....');
continue;
}
Console::log($msg);
if ($collectionsCreated > 0) {
$collections = $dbForProject->find(Database::METADATA, []);
foreach ($collections as $collection) {
if ($commit) {
$dbForProject->deleteCollection($collection->getId());
$dbForConsole->purgeCachedCollection($collection->getId());
}
Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')');
}
}
if ($commit) {
$dbForConsole->deleteDocument('projects', $project->getId());
$dbForConsole->purgeCachedDocument('projects', $project->getId());
if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
try {
$dbForProject->deleteCollection(Database::METADATA);
$dbForProject->purgeCachedCollection(Database::METADATA);
} catch (\Throwable $th) {
Console::warning('Metadata collection does not exist');
}
}
}
Console::info('--Deleting project no (' . $project->getInternalId() . ')');
$orphans++;
} catch (\Throwable $th) {
Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans');
}
}

187
src/Appwrite/Platform/Tasks/GetMigrationStats.php
View file

@ -1,187 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use League\Csv\CannotInsertRecord;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\System\System;
class GetMigrationStats extends Action
{
/*
* Csv cols headers
*/
private array $columns = [
'Project ID',
'$id',
'$createdAt',
'status',
'stage',
'source'
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'get-migration-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
/**
* @throws \Utopia\Exception
* @throws CannotInsertRecord
*/
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite get-migration-stats
Console::title('Migration stats calculation V1');
Console::success(APP_NAME . ' Migration stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/migration_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 100;
$sum = 100;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Migration details */
$migrations = $dbForProject->find('migrations', [
Query::limit(500)
]);
$migrations = array_map(function ($migration) use ($project) {
return [
$project->getId(),
$migration->getAttribute('$id'),
$migration->getAttribute('$createdAt'),
$migration->getAttribute('status'),
$migration->getAttribute('stage'),
$migration->getAttribute('source'),
];
}, $migrations);
if (!empty($migrations)) {
$csv->insertAll($migrations);
}
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', System::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Migration Report for {$this->date}";
$mail->Body = "Please find the migration report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (\Throwable $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}

157
src/Appwrite/Platform/Tasks/Hamster.php
View file

@ -1,157 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\Hamster as EventHamster;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\System\System;
class Hamster extends Action
{
public static function getName(): string
{
return 'hamster';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('queueForHamster')
->inject('dbForConsole')
->callback(function (EventHamster $queueForHamster, Database $dbForConsole) {
$this->action($queueForHamster, $dbForConsole);
});
}
public function action(EventHamster $queueForHamster, Database $dbForConsole): void
{
Console::title('Cloud Hamster V1');
Console::success(APP_NAME . ' cloud hamster process has started');
$sleep = (int) System::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
$jobInitTime = System::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
$now = new \DateTime();
$now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$next = new \DateTime($now->format("Y-m-d $jobInitTime"));
$next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
$delay = $next->getTimestamp() - $now->getTimestamp();
/**
* If time passed for the target day.
*/
if ($delay <= 0) {
$next->add(\DateInterval::createFromDateString('1 days'));
$delay = $next->getTimestamp() - $now->getTimestamp();
}
Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
Console::loop(function () use ($queueForHamster, $dbForConsole, $sleep) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Queuing Cloud Usage Stats every {$sleep} seconds");
$loopStart = microtime(true);
Console::info('Queuing stats for all projects');
$this->getStatsPerProject($queueForHamster, $dbForConsole, $loopStart);
Console::success('Completed queuing stats for all projects');
Console::info('Queuing stats for all organizations');
$this->getStatsPerOrganization($queueForHamster, $dbForConsole, $loopStart);
Console::success('Completed queuing stats for all organizations');
Console::info('Queuing stats for all users');
$this->getStatsPerUser($queueForHamster, $dbForConsole, $loopStart);
Console::success('Completed queuing stats for all users');
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
}, $sleep, $delay);
}
protected function calculateByGroup(string $collection, Database $database, callable $callback)
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([
Query::limit($limit),
Query::offset($count)
]));
$sum = count($results);
Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
call_user_func($callback, $database, $document);
$count++;
}
}
$executionEnd = \microtime(true);
Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
protected function getStatsPerOrganization(EventHamster $hamster, Database $dbForConsole, float $loopStart)
{
$this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $organization) use ($hamster, $loopStart) {
try {
$organization->setAttribute('$time', $loopStart);
$hamster
->setType(EventHamster::TYPE_ORGANISATION)
->setOrganization($organization)
->trigger();
} catch (\Throwable $e) {
Console::error($e->getMessage());
}
});
}
private function getStatsPerProject(EventHamster $hamster, Database $dbForConsole, float $loopStart)
{
$this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($hamster, $loopStart) {
try {
$project->setAttribute('$time', $loopStart);
$hamster
->setType(EventHamster::TYPE_PROJECT)
->setProject($project)
->trigger();
} catch (\Throwable $e) {
Console::error($e->getMessage());
}
});
}
protected function getStatsPerUser(EventHamster $hamster, Database $dbForConsole, float $loopStart)
{
$this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $user) use ($hamster, $loopStart) {
try {
$user->setAttribute('$time', $loopStart);
$hamster
->setType(EventHamster::TYPE_USER)
->setUser($user)
->trigger();
} catch (\Throwable $e) {
Console::error($e->getMessage());
}
});
}
}

169
src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php
View file

@ -1,169 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Validator\Text;
class PatchRecreateRepositoriesDocuments extends Action
{
public static function getName(): string
{
return 'patch-recreate-repositories-documents';
}
public function __construct()
{
$this
->desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.')
->param('after', '', new Text(36), 'After cursor', true)
->param('projectId', '', new Text(36), 'Select project to validate', true)
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB));
}
public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void
{
Console::info("Starting the patch");
$startTime = microtime(true);
if (!empty($projectId)) {
try {
$project = $dbForConsole->getDocument('projects', $projectId);
$dbForProject = call_user_func($getProjectDB, $project);
$this->recreateRepositories($dbForConsole, $dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
} else {
$queries = [];
if (!empty($after)) {
Console::info("Iterating remaining projects after project with ID {$after}");
$project = $dbForConsole->getDocument('projects', $after);
$queries = [Query::cursorAfter($project)];
} else {
Console::info("Iterating all projects");
}
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) {
$projectId = $project->getId();
try {
$dbForProject = call_user_func($getProjectDB, $project);
$this->recreateRepositories($dbForConsole, $dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
});
}
$endTime = microtime(true);
$timeTaken = $endTime - $startTime;
$hours = (int)($timeTaken / 3600);
$timeTaken -= $hours * 3600;
$minutes = (int)($timeTaken / 60);
$timeTaken -= $minutes * 60;
$seconds = (int)$timeTaken;
$milliseconds = ($timeTaken - $seconds) * 1000;
Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)");
}
protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
{
$limit = 1000;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
if ($latestDocument != null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$results = $database->find($collection, $newQueries);
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void
{
$projectId = $project->getId();
Console::log("Running patch for project {$projectId}");
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) {
$isConnected = !empty($function->getAttribute('providerRepositoryId', ''));
if ($isConnected) {
$repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', ''));
if ($repository->isEmpty()) {
$projectId = $project->getId();
$functionId = $function->getId();
Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}");
$repository = $dbForConsole->createDocument('repositories', new Document([
'$id' => ID::unique(),
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'installationId' => $function->getAttribute('installationId', ''),
'installationInternalId' => $function->getAttribute('installationInternalId', ''),
'projectId' => $project->getId(),
'projectInternalId' => $project->getInternalId(),
'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''),
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceType' => 'function',
'providerPullRequestIds' => []
]));
$function = $dbForProject->updateDocument('functions', $function->getId(), $function
->setAttribute('repositoryId', $repository->getId())
->setAttribute('repositoryInternalId', $repository->getInternalId()));
$this->foreachDocument($dbForProject, 'deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', ['functions'])
], function (Document $deployment) use ($dbForProject, $repository) {
$dbForProject->updateDocument('deployments', $deployment->getId(), $deployment
->setAttribute('repositoryId', $repository->getId())
->setAttribute('repositoryInternalId', $repository->getInternalId()));
});
}
}
});
}
}

59
src/Appwrite/Platform/Tasks/VolumeSync.php
View file

@ -1,59 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Platform\Action;
use Utopia\Validator\Integer;
use Utopia\Validator\Text;
class VolumeSync extends Action
{
public static function getName(): string
{
return 'volume-sync';
}
public function __construct()
{
$this
->desc('Runs rsync to sync certificates between the storage mount and traefik.')
->param('source', null, new Text(255), 'Source path to sync from.', false)
->param('destination', null, new Text(255), 'Destination path to sync to.', false)
->param('interval', null, new Integer(true), 'Interval to run rsync', false)
->callback(fn ($source, $destination, $interval) => $this->action($source, $destination, $interval));
}
public function action(string $source, string $destination, int $interval)
{
Console::title('RSync V1');
Console::success(APP_NAME . ' rsync process v1 has started');
if (!file_exists($source)) {
Console::error('Source directory does not exist. Exiting ... ');
Console::exit(0);
}
Console::loop(function () use ($interval, $source, $destination) {
$time = DateTime::now();
Console::info("[{$time}] Executing rsync every {$interval} seconds");
Console::info("Syncing between $source and $destination");
if (!file_exists($source)) {
Console::error('Source directory does not exist. Skipping ... ');
return;
}
$stdin = "";
$stdout = "";
$stderr = "";
Console::execute("rsync -av $source $destination", $stdin, $stdout, $stderr);
Console::success($stdout);
Console::error($stderr);
}, $interval);
}
}

81
src/Appwrite/Platform/Workers/Functions.php
View file

@ -198,6 +198,70 @@ class Functions extends Action
}
}
/**
* @param string $message
* @param Document $function
* @param string $trigger
* @param string $path
* @param string $method
* @param Document $user
* @param string|null $jwt
* @param string|null $event
* @throws Exception
*/
private function fail(
string $message,
Database $dbForProject,
Document $function,
string $trigger,
string $path,
string $method,
Document $user,
string $jwt = null,
string $event = null,
): void {
$headers['x-appwrite-trigger'] = $trigger;
$headers['x-appwrite-event'] = $event ?? '';
$headers['x-appwrite-user-id'] = $user->getId() ?? '';
$headers['x-appwrite-user-jwt'] = $jwt ?? '';
$headersFiltered = [];
foreach ($headers as $key => $value) {
if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_REQUEST)) {
$headersFiltered[] = ['name' => $key, 'value' => $value];
}
}
$executionId = ID::unique();
$execution = new Document([
'$id' => $executionId,
'$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))],
'functionInternalId' => $function->getInternalId(),
'functionId' => $function->getId(),
'deploymentInternalId' => '',
'deploymentId' => '',
'trigger' => $trigger,
'status' => 'failed',
'responseStatusCode' => 0,
'responseHeaders' => [],
'requestPath' => $path,
'requestMethod' => $method,
'requestHeaders' => $headersFiltered,
'errors' => $message,
'logs' => '',
'duration' => 0.0,
'search' => implode(' ', [$function->getId(), $executionId]),
]);
if ($function->getAttribute('logging')) {
$execution = $dbForProject->createDocument('executions', $execution);
}
if ($execution->isEmpty()) {
throw new Exception('Failed to create execution');
}
}
/**
* @param Log $log
* @param Database $dbForProject
@ -251,11 +315,15 @@ class Functions extends Action
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if ($deployment->getAttribute('resourceId') !== $functionId) {
throw new Exception('Deployment not found. Create deployment before trying to execute a function');
$errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
$this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found. Create deployment before trying to execute a function');
$errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
$this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
$buildId = $deployment->getAttribute('buildId', '');
@ -265,11 +333,15 @@ class Functions extends Action
/** Check if build has exists */
$build = $dbForProject->getDocument('builds', $buildId);
if ($build->isEmpty()) {
throw new Exception('Build not found');
$errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.';
$this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
if ($build->getAttribute('status') !== 'ready') {
throw new Exception('Build not ready');
$errorMessage = 'The execution could not be completed because the build is not ready. Please wait for the build to complete and try again.';
$this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event);
return;
}
/** Check if runtime is supported */
@ -287,7 +359,6 @@ class Functions extends Action
$headers['x-appwrite-user-id'] = $user->getId() ?? '';
$headers['x-appwrite-user-jwt'] = $jwt ?? '';
/** Create execution or update execution status */
/** Create execution or update execution status */
$execution = $dbForProject->getDocument('executions', $executionId ?? '');
if ($execution->isEmpty()) {

494
src/Appwrite/Platform/Workers/Hamster.php
View file

@ -1,494 +0,0 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Hamster as EventHamster;
use Appwrite\Network\Validator\Origin;
use Utopia\Analytics\Adapter\Mixpanel;
use Utopia\Analytics\Event as AnalyticsEvent;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Action;
use Utopia\Pools\Group;
use Utopia\Queue\Message;
use Utopia\System\System;
class Hamster extends Action
{
private array $metrics = [
'usage_files' => 'files',
'usage_buckets' => 'buckets',
'usage_databases' => 'databases',
'usage_documents' => 'documents',
'usage_collections' => 'collections',
'usage_storage' => 'files.storage',
'usage_requests' => 'network.requests',
'usage_inbound' => 'network.inbound',
'usage_outbound' => 'network.outbound',
'usage_users' => 'users',
'usage_sessions' => 'sessions',
'usage_executions' => 'executions',
];
protected Mixpanel $mixpanel;
public static function getName(): string
{
return 'hamster';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this
->desc('Hamster worker')
->inject('message')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(fn (Message $message, Group $pools, Cache $cache, Database $dbForConsole) => $this->action($message, $pools, $cache, $dbForConsole));
}
/**
* @param Message $message
* @param Group $pools
* @param Cache $cache
* @param Database $dbForConsole
*
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Group $pools, Cache $cache, Database $dbForConsole): void
{
$token = System::getEnv('_APP_MIXPANEL_TOKEN', '');
if (empty($token)) {
throw new \Exception('Missing MixPanel Token');
}
$this->mixpanel = new Mixpanel($token);
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new \Exception('Missing payload');
}
$type = $payload['type'] ?? '';
switch ($type) {
case EventHamster::TYPE_PROJECT:
$this->getStatsForProject(new Document($payload['project']), $pools, $cache, $dbForConsole);
break;
case EventHamster::TYPE_ORGANISATION:
$this->getStatsForOrganization(new Document($payload['organization']), $dbForConsole);
break;
case EventHamster::TYPE_USER:
$this->getStatsPerUser(new Document($payload['user']), $dbForConsole);
break;
}
}
/**
* @param Document $project
* @param Group $pools
* @param Cache $cache
* @param Database $dbForConsole
* @throws \Utopia\Database\Exception
*/
private function getStatsForProject(Document $project, Group $pools, Cache $cache, Database $dbForConsole): void
{
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
Console::info("Skipping project console");
return;
}
Console::log("Getting stats for Project {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = [];
$statsPerProject['time'] = $project->getAttribute('$time');
/** Get Project ID */
$statsPerProject['project_id'] = $project->getId();
/** Get project created time */
$statsPerProject['project_created'] = $project->getAttribute('$createdAt');
/** Get Project Name */
$statsPerProject['project_name'] = $project->getAttribute('name');
/** Total Project Variables */
$statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT);
/** Total Migrations */
$statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
/** Get Custom SMTP */
$smtp = $project->getAttribute('smtp', null);
if ($smtp) {
$statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled';
/** Get Custom Templates Count */
$templates = array_keys($project->getAttribute('templates', []));
$statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) {
return str_contains($template, 'email');
});
$statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) {
return str_contains($template, 'sms');
});
}
/** Get total relationship attributes */
$statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [
Query::equal('type', ['relationship'])
], APP_LIMIT_COUNT);
/** Get Total Functions */
$statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
$statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
Query::equal('runtime', [$runtime]),
], APP_LIMIT_COUNT);
}
/** Get Total Deployments */
$statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
$statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [
Query::equal('type', ['manual'])
], APP_LIMIT_COUNT);
$statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [
Query::equal('type', ['vcs'])
], APP_LIMIT_COUNT);
/** Get VCS repos connected */
$statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [
Query::equal('projectInternalId', [$project->getInternalId()])
], APP_LIMIT_COUNT);
/** Get Total Teams */
$statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
], APP_LIMIT_COUNT);
} else {
$statsPerProject['custom_organization_members'] = 0;
}
/** Get Email and Name of the project owner */
if ($teamInternalId) {
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$teamInternalId]),
]);
if (!$membership || $membership->isEmpty()) {
throw new \Exception('Membership not found. Skipping project : ' . $project->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerProject['email'] = $user->getAttribute('email', null);
$statsPerProject['name'] = $user->getAttribute('name', null);
}
}
/** Add billing information to the project */
$organization = $dbForConsole->findOne('teams', [
Query::equal('$internalId', [$teamInternalId])
]);
$billing = $this->getBillingDetails($organization);
$statsPerProject['billing_plan'] = $billing['billing_plan'] ?? null;
$statsPerProject['billing_start_date'] = $billing['billing_start_date'] ?? null;
/** Get Domains */
$statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Platforms */
$platforms = $dbForConsole->find('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
$statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'web';
}));
$statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
return $platform['type'] === 'android';
}));
$statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'apple');
}));
$statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
return str_contains($platform['type'], 'flutter');
}));
$flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
foreach ($flutterPlatforms as $flutterPlatform) {
$statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
return $platform['type'] === $flutterPlatform;
}));
}
$statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
/** Get Usage $statsPerProject */
$periods = [
'infinity' => [
'period' => '1d',
'limit' => 90,
],
'24h' => [
'period' => '1h',
'limit' => 24,
],
];
Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
foreach ($this->metrics as $key => $metric) {
foreach ($periods as $periodKey => $periodValue) {
$limit = $periodValue['limit'];
$period = $periodValue['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$statsPerProject[$key . '_' . $periodKey] = [];
foreach ($requestDocs as $requestDoc) {
$statsPerProject[$key . '_' . $periodKey][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
// Calculate aggregate of each metric
$statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
}
}
});
/**
* Workaround to combine network.Inbound+network.outbound as bandwidth.
*/
$statsPerProject["usage_bandwidth_infinity"] = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"];
$statsPerProject["usage_bandwidth_24h"] = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"];
unset($statsPerProject["usage_outbound_24h"]);
unset($statsPerProject["usage_inbound_24h"]);
unset($statsPerProject["usage_outbound_infinity"]);
unset($statsPerProject["usage_inbound_infinity"]);
if (isset($statsPerProject['email'])) {
/** Send data to mixpanel */
$res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
'name' => $statsPerProject['name'],
'email' => $statsPerProject['email']
]);
if (!$res) {
Console::error('Failed to create user profile for project: ' . $project->getId());
}
}
$event = new AnalyticsEvent();
$event
->setName('Project Daily Usage')
->setProps($statsPerProject);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
Console::error('Failed to create event for project: ' . $project->getId());
}
} catch (\Throwable $e) {
Console::error('Failed to send stats for project: ' . $project->getId());
Console::error($e->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
/**
* @param Document $organization
* @param Database $dbForConsole
*/
private function getStatsForOrganization(Document $organization, Database $dbForConsole): void
{
Console::log("Getting stats for Organization {$organization->getId()}");
try {
$statsPerOrganization = [];
$statsPerOrganization['time'] = $organization->getAttribute('$time');
/** Organization name */
$statsPerOrganization['name'] = $organization->getAttribute('name');
/** Get Email and of the organization owner */
$membership = $dbForConsole->findOne('memberships', [
Query::equal('teamInternalId', [$organization->getInternalId()]),
]);
if (!$membership || $membership->isEmpty()) {
throw new \Exception('Membership not found. Skipping organization : ' . $organization->getId());
}
$userId = $membership->getAttribute('userId', null);
if ($userId) {
$user = $dbForConsole->getDocument('users', $userId);
$statsPerOrganization['email'] = $user->getAttribute('email', null);
}
/** Add billing information */
$billing = $this->getBillingDetails($organization);
$statsPerOrganization['billing_plan'] = $billing['billing_plan'] ?? null;
$statsPerOrganization['billing_start_date'] = $billing['billing_start_date'] ?? null;
$statsPerOrganization['marked_for_deletion'] = $billing['markedForDeletion'] ?? 0;
$statsPerOrganization['billing_plan_downgrade'] = $billing['billing_plan_downgrade'] ?? null;
/** Organization Creation Date */
$statsPerOrganization['created'] = $organization->getAttribute('$createdAt');
/** Number of team members */
$statsPerOrganization['members'] = $organization->getAttribute('total');
/** Number of projects in this organization */
$statsPerOrganization['projects'] = $dbForConsole->count('projects', [
Query::equal('teamId', [$organization->getId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerOrganization['email'])) {
throw new \Exception('Email not found. Skipping organization : ' . $organization->getId());
}
$event = new AnalyticsEvent();
$event
->setName('Organization Daily Usage')
->setProps($statsPerOrganization);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new \Exception('Failed to create event for organization : ' . $organization->getId());
}
} catch (\Throwable $e) {
Console::error($e->getMessage());
}
}
protected function getStatsPerUser(Document $user, Database $dbForConsole)
{
Console::log("Getting stats for User {$user->getId()}");
try {
$statsPerUser = [];
$statsPerUser['time'] = $user->getAttribute('$time');
/** Add billing information */
$organization = $dbForConsole->findOne('teams', [
Query::equal('userInternalId', [$user->getInternalId()])
]);
$billing = $this->getBillingDetails($organization);
$statsPerUser['billing_plan'] = $billing['billing_plan'] ?? null;
$statsPerUser['billing_start_date'] = $billing['billing_start_date'] ?? null;
/** Organization name */
$statsPerUser['name'] = $user->getAttribute('name');
/** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
$statsPerUser['email'] = $user->getAttribute('email');
/** Organization Creation Date */
$statsPerUser['created'] = $user->getAttribute('$createdAt');
/** Number of teams this user is a part of */
$statsPerUser['memberships'] = $dbForConsole->count('memberships', [
Query::equal('userInternalId', [$user->getInternalId()]),
Query::limit(APP_LIMIT_COUNT)
]);
if (!isset($statsPerUser['email'])) {
throw new \Exception('User has no email: ' . $user->getId());
}
/** Send data to mixpanel */
$event = new AnalyticsEvent();
$event
->setName('User Daily Usage')
->setProps($statsPerUser);
$res = $this->mixpanel->createEvent($event);
if (!$res) {
throw new \Exception('Failed to create user profile for user: ' . $user->getId());
}
} catch (\Throwable $e) {
Console::error($e->getMessage());
}
}
private function getBillingDetails(bool|Document $team): array
{
$billing = [];
if (!empty($team) && !$team->isEmpty()) {
$billingPlan = $team->getAttribute('billingPlan', null);
$billingPlanDowngrade = $team->getAttribute('billingPlanDowngrade', null);
if (!empty($billingPlan) && empty($billingPlanDowngrade)) {
$billing['billing_plan'] = $billingPlan;
}
if (in_array($billingPlan, ['tier-1', 'tier-2'])) {
$billingStartDate = $team->getAttribute('billingStartDate', null);
$billing['billing_start_date'] = $billingStartDate;
}
$billing['marked_for_deletion'] = $team->getAttribute('markedForDeletion', 0);
$billing['billing_plan_downgrade'] = $billingPlanDowngrade;
}
return $billing;
}
}

19
src/Appwrite/Vcs/Comment.php
View file

@ -67,9 +67,6 @@ class Comment
];
}
//TODO: Update link to documentation
$text .= "**Your function has been automatically deployed.** Learn more about Appwrite [Function Deployments](https://appwrite.io/docs/functions).\n\n";
foreach ($projects as $projectId => $project) {
$text .= "**{$project['name']}** `{$projectId}`\n\n";
$text .= "| Function | ID | Status | Action |\n";
@ -79,6 +76,18 @@ class Comment
$hostname = System::getEnv('_APP_DOMAIN');
foreach ($project['functions'] as $functionId => $function) {
if ($function['status'] === 'waiting' || $function['status'] === 'processing' || $function['status'] === 'building') {
$text .= "**Your function deployment is in progress. Please check back in a few minutes for the updated status.**\n\n";
} elseif ($function['status'] === 'ready') {
$text .= "**Your function has been successfully deployed.**\n\n";
} else {
$text .= "**Your function deployment has failed. Please check the logs for more details and retry.**\n\n";
}
$text .= "Project name: **{$project['name']}** \nProject ID: `{$projectId}`\n\n";
$text .= "| Function | ID | Status | Action |\n";
$text .= "| :- | :- | :- | :- |\n";
$generateImage = function (string $status) use ($protocol, $hostname) {
$extention = $status === 'building' ? 'gif' : 'png';
$imagesUrl = $protocol . '://' . $hostname . '/images/vcs/';
@ -106,7 +115,9 @@ class Comment
$text .= "\n\n";
}
//TODO: Update did you know section
$functionUrl = $protocol . '://' . $hostname . '/console/project-' . $projectId . '/functions/function-' . $functionId;
$text .= "Only deployments on the production branch are activated automatically. If you'd like to activate this deployment, navigate to [your deployments]($functionUrl). Learn more about Appwrite [Function deployments](https://appwrite.io/docs/functions).\n\n";
$tip = $this->tips[array_rand($this->tips)];
$text .= "> **💡 Did you know?** \n " . $tip . "\n\n";

2
tests/e2e/Services/Databases/DatabasesBase.php
View file

@ -4730,7 +4730,7 @@ trait DatabasesBase
], $this->getHeaders()), [
'documentId' => ID::unique(),
'data' => [
'longtext' => file_get_contents('tests/resources/longtext.txt'),
'longtext' => file_get_contents(__DIR__ . '/../../../resources/longtext.txt'),
],
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),

6
tests/e2e/Services/Locale/LocaleBase.php
View file

@ -227,9 +227,9 @@ trait LocaleBase
/**
* Test for SUCCESS
*/
$languages = require('app/config/locale/codes.php');
$defaultCountries = require('app/config/locale/countries.php');
$defaultContinents = require('app/config/locale/continents.php');
$languages = require(__DIR__ . '/../../../../app/config/locale/codes.php');
$defaultCountries = require(__DIR__ . '/../../../../app/config/locale/countries.php');
$defaultContinents = require(__DIR__ . '/../../../../app/config/locale/continents.php');
foreach ($languages as $lang) {
$response = $this->client->call(Client::METHOD_GET, '/locale/countries', [

8
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
View file

@ -1061,7 +1061,7 @@ class ProjectsConsoleClientTest extends Scope
public function testUpdateProjectAuthStatus($data): array
{
$id = $data['projectId'] ?? '';
$auth = require('app/config/auth.php');
$auth = require(__DIR__ . '/../../../../app/config/auth.php');
$originalEmail = uniqid() . 'user@localhost.test';
$originalPassword = 'password';
@ -1897,7 +1897,7 @@ class ProjectsConsoleClientTest extends Scope
$this->assertNotEmpty($project['body']['$id']);
$id = $project['body']['$id'];
$services = require('app/config/services.php');
$services = require(__DIR__ . '/../../../../app/config/services.php');
/**
* Test for Disabled
@ -1971,7 +1971,7 @@ class ProjectsConsoleClientTest extends Scope
{
$id = $data['projectId'];
$services = require('app/config/services.php');
$services = require(__DIR__ . '/../../../../app/config/services.php');
/**
* Test for Disabled
@ -2045,7 +2045,7 @@ class ProjectsConsoleClientTest extends Scope
{
$id = $data['projectId'];
$services = require('app/config/services.php');
$services = require(__DIR__ . '/../../../../app/config/services.php');
/**
* Test for Disabled

2
tests/unit/Event/EventTest.php
View file

@ -30,7 +30,7 @@ class EventTest extends TestCase
$dsn = System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis);
$dsn = explode('=', $dsn);
$dsn = $dsn[0] ?? '';
$dsn = $dsn[1] ?? '';
$dsn = new DSN($dsn);
$connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
$this->queue = 'v1-tests' . uniqid();