Merge pull request #7660 from appwrite/refactor-usage-sn
Refactor usage sn
This commit is contained in:
commit
9ea34ce068
2
.env
2
.env
|
@ -78,7 +78,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
|
|||
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
|
||||
_APP_MAINTENANCE_RETENTION_ABUSE=86400
|
||||
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
|
||||
_APP_USAGE_AGGREGATION_INTERVAL=30
|
||||
_APP_USAGE_AGGREGATION_INTERVAL=20
|
||||
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
|
||||
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
|
||||
_APP_USAGE_STATS=enabled
|
||||
|
|
10
.github/workflows/publish.yml
vendored
10
.github/workflows/publish.yml
vendored
|
@ -17,6 +17,12 @@ jobs:
|
|||
fetch-depth: 2
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
|
@ -35,11 +41,11 @@ jobs:
|
|||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
VITE_APPWRITE_GROWTH_ENDPOINT=https://growth.appwrite.io/v1
|
||||
VITE_GA_PROJECT=G-L7G2B6PLDS
|
||||
VITE_CONSOLE_MODE=cloud
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
|
|
|
@ -34,28 +34,6 @@ $commonCollections = [
|
|||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => 'resourceType',
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 255,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('mimeType'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => 'accessedAt',
|
||||
'type' => Database::VAR_DATETIME,
|
||||
|
@ -1349,10 +1327,10 @@ $commonCollections = [
|
|||
]
|
||||
],
|
||||
|
||||
'stats' => [
|
||||
'stats_v2' => [
|
||||
'$collection' => ID::custom(Database::METADATA),
|
||||
'$id' => ID::custom('stats'),
|
||||
'name' => 'stats',
|
||||
'$id' => ID::custom('stats_v2'),
|
||||
'name' => 'stats_v2',
|
||||
'attributes' => [
|
||||
[
|
||||
'$id' => ID::custom('metric'),
|
||||
|
|
|
@ -31,7 +31,7 @@ return [
|
|||
Exception::GENERAL_SERVICE_DISABLED => [
|
||||
'name' => Exception::GENERAL_SERVICE_DISABLED,
|
||||
'description' => 'The requested service is disabled. You can enable the service from the Appwrite console.',
|
||||
'code' => 503,
|
||||
'code' => 403,
|
||||
],
|
||||
Exception::GENERAL_UNAUTHORIZED_SCOPE => [
|
||||
'name' => Exception::GENERAL_UNAUTHORIZED_SCOPE,
|
||||
|
|
|
@ -1791,7 +1791,7 @@ App::get('/v1/account/logs')
|
|||
}
|
||||
|
||||
$response->dynamic(new Document([
|
||||
'total' => $audit->countLogsByUser($user->getInternalId()),
|
||||
'total' => $audit->countLogsByUser($user->getId()),
|
||||
'logs' => $output,
|
||||
]), Response::MODEL_LOG_LIST);
|
||||
});
|
||||
|
|
|
@ -76,7 +76,7 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro
|
|||
}
|
||||
|
||||
if (empty($gitHubSession)) {
|
||||
throw new Exception(Exception::USER_SESSION_NOT_FOUND, 'GitHub session not found.');
|
||||
throw new Exception(Exception::GENERAL_UNKNOWN, 'GitHub session not found.');
|
||||
}
|
||||
|
||||
$provider = $gitHubSession->getAttribute('provider', '');
|
||||
|
|
|
@ -3590,7 +3590,7 @@ App::get('/v1/databases/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -3598,7 +3598,7 @@ App::get('/v1/databases/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -3674,7 +3674,7 @@ App::get('/v1/databases/:databaseId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -3682,7 +3682,7 @@ App::get('/v1/databases/:databaseId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -3760,7 +3760,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -3768,7 +3768,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -484,7 +484,7 @@ App::get('/v1/functions/:functionId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -492,7 +492,7 @@ App::get('/v1/functions/:functionId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -576,7 +576,7 @@ App::get('/v1/functions/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -584,7 +584,7 @@ App::get('/v1/functions/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -659,60 +659,6 @@ App::get('/v1/health/queue/functions')
|
|||
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
|
||||
}, ['response']);
|
||||
|
||||
App::get('/v1/health/queue/usage')
|
||||
->desc('Get usage queue')
|
||||
->groups(['api', 'health'])
|
||||
->label('scope', 'health.read')
|
||||
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
|
||||
->label('sdk.namespace', 'health')
|
||||
->label('sdk.method', 'getQueueUsage')
|
||||
->label('sdk.description', '/docs/references/health/get-queue-usage.md')
|
||||
->label('sdk.response.code', Response::STATUS_CODE_OK)
|
||||
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
|
||||
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
|
||||
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
|
||||
->inject('queue')
|
||||
->inject('response')
|
||||
->action(function (int|string $threshold, Connection $queue, Response $response) {
|
||||
$threshold = \intval($threshold);
|
||||
|
||||
$client = new Client(Event::USAGE_QUEUE_NAME, $queue);
|
||||
$size = $client->getQueueSize();
|
||||
|
||||
if ($size >= $threshold) {
|
||||
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
|
||||
}
|
||||
|
||||
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
|
||||
});
|
||||
|
||||
App::get('/v1/health/queue/usage-dump')
|
||||
->desc('Get usage dump queue')
|
||||
->groups(['api', 'health'])
|
||||
->label('scope', 'health.read')
|
||||
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
|
||||
->label('sdk.namespace', 'health')
|
||||
->label('sdk.method', 'getQueueUsage')
|
||||
->label('sdk.description', '/docs/references/health/get-queue-usage-dump.md')
|
||||
->label('sdk.response.code', Response::STATUS_CODE_OK)
|
||||
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
|
||||
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
|
||||
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
|
||||
->inject('queue')
|
||||
->inject('response')
|
||||
->action(function (int|string $threshold, Connection $queue, Response $response) {
|
||||
$threshold = \intval($threshold);
|
||||
|
||||
$client = new Client(Event::USAGE_DUMP_QUEUE_NAME, $queue);
|
||||
$size = $client->getQueueSize();
|
||||
|
||||
if ($size >= $threshold) {
|
||||
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
|
||||
}
|
||||
|
||||
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
|
||||
});
|
||||
|
||||
App::get('/v1/health/storage/local')
|
||||
->desc('Get local storage')
|
||||
->groups(['api', 'health'])
|
||||
|
@ -756,6 +702,47 @@ App::get('/v1/health/storage/local')
|
|||
$response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
|
||||
});
|
||||
|
||||
App::get('/v1/health/storage')
|
||||
->desc('Get storage')
|
||||
->groups(['api', 'health'])
|
||||
->label('scope', 'health.read')
|
||||
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
|
||||
->label('sdk.namespace', 'health')
|
||||
->label('sdk.method', 'getStorage')
|
||||
->label('sdk.description', '/docs/references/health/get-storage.md')
|
||||
->label('sdk.response.code', Response::STATUS_CODE_OK)
|
||||
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
|
||||
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
|
||||
->inject('response')
|
||||
->inject('deviceFiles')
|
||||
->inject('deviceFunctions')
|
||||
->inject('deviceBuilds')
|
||||
->action(function (Response $response, Device $deviceFiles, Device $deviceFunctions, Device $deviceBuilds) {
|
||||
$devices = [$deviceFiles, $deviceFunctions, $deviceBuilds];
|
||||
$checkStart = \microtime(true);
|
||||
|
||||
foreach ($devices as $device) {
|
||||
if (!$device->write($device->getPath('health.txt'), 'test', 'text/plain')) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed writing test file to ' . $device->getRoot());
|
||||
}
|
||||
|
||||
if ($device->read($device->getPath('health.txt')) !== 'test') {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed reading test file from ' . $device->getRoot());
|
||||
}
|
||||
|
||||
if (!$device->delete($device->getPath('health.txt'))) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed deleting test file from ' . $device->getRoot());
|
||||
}
|
||||
}
|
||||
|
||||
$output = [
|
||||
'status' => 'pass',
|
||||
'ping' => \round((\microtime(true) - $checkStart) / 1000)
|
||||
];
|
||||
|
||||
$response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
|
||||
});
|
||||
|
||||
App::get('/v1/health/anti-virus')
|
||||
->desc('Get antivirus')
|
||||
->groups(['api', 'health'])
|
||||
|
@ -809,7 +796,6 @@ App::get('/v1/health/queue/failed/:name')
|
|||
Event::MAILS_QUEUE_NAME,
|
||||
Event::FUNCTIONS_QUEUE_NAME,
|
||||
Event::USAGE_QUEUE_NAME,
|
||||
Event::USAGE_DUMP_QUEUE_NAME,
|
||||
Event::WEBHOOK_CLASS_NAME,
|
||||
Event::CERTIFICATES_QUEUE_NAME,
|
||||
Event::BUILDS_QUEUE_NAME,
|
||||
|
|
|
@ -73,7 +73,7 @@ App::get('/v1/project/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
|
||||
foreach ($metrics['total'] as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -81,7 +81,7 @@ App::get('/v1/project/usage')
|
|||
}
|
||||
|
||||
foreach ($metrics['period'] as $metric) {
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::greaterThanEqual('time', $firstDay),
|
||||
|
@ -116,7 +116,7 @@ App::get('/v1/project/usage')
|
|||
$id = $function->getId();
|
||||
$name = $function->getAttribute('name');
|
||||
$metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
|
||||
$value = $dbForProject->findOne('stats', [
|
||||
$value = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -132,7 +132,7 @@ App::get('/v1/project/usage')
|
|||
$id = $bucket->getId();
|
||||
$name = $bucket->getAttribute('name');
|
||||
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
|
||||
$value = $dbForProject->findOne('stats', [
|
||||
$value = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
|
|
@ -1467,7 +1467,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
|
|||
if ($deviceDeleted) {
|
||||
$queueForDeletes
|
||||
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
|
||||
->setResourceType('bucket/' . $bucket->getId())
|
||||
->setResource('file/' . $fileId)
|
||||
;
|
||||
|
||||
|
@ -1525,7 +1524,7 @@ App::get('/v1/storage/usage')
|
|||
$total = [];
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -1533,7 +1532,7 @@ App::get('/v1/storage/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
@ -1610,7 +1609,7 @@ App::get('/v1/storage/:bucketId/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
|
||||
foreach ($metrics as $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -1618,7 +1617,7 @@ App::get('/v1/storage/:bucketId/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -615,9 +615,6 @@ App::get('/v1/users/:userId/logs')
|
|||
|
||||
$output[$i] = new Document([
|
||||
'event' => $log['event'],
|
||||
'userId' => ID::custom($log['data']['userId']),
|
||||
'userEmail' => $log['data']['userEmail'] ?? null,
|
||||
'userName' => $log['data']['userName'] ?? null,
|
||||
'ip' => $log['ip'],
|
||||
'time' => $log['time'],
|
||||
'osCode' => $os['osCode'],
|
||||
|
@ -646,7 +643,7 @@ App::get('/v1/users/:userId/logs')
|
|||
}
|
||||
|
||||
$response->dynamic(new Document([
|
||||
'total' => $audit->countLogsByUser($user->getInternalId()),
|
||||
'total' => $audit->countLogsByUser($user->getId()),
|
||||
'logs' => $output,
|
||||
]), Response::MODEL_LOG_LIST);
|
||||
});
|
||||
|
@ -1266,7 +1263,7 @@ App::get('/v1/users/usage')
|
|||
|
||||
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
|
||||
foreach ($metrics as $count => $metric) {
|
||||
$result = $dbForProject->findOne('stats', [
|
||||
$result = $dbForProject->findOne('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', ['inf'])
|
||||
]);
|
||||
|
@ -1274,7 +1271,7 @@ App::get('/v1/users/usage')
|
|||
$stats[$metric]['total'] = $result['value'] ?? 0;
|
||||
$limit = $days['limit'];
|
||||
$period = $days['period'];
|
||||
$results = $dbForProject->find('stats', [
|
||||
$results = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -9,9 +9,8 @@ use Utopia\Logger\Logger;
|
|||
use Utopia\Logger\Log;
|
||||
use Utopia\Logger\Log\User;
|
||||
use Swoole\Http\Request as SwooleRequest;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Pools\Group;
|
||||
use Appwrite\Utopia\Request;
|
||||
use MaxMind\Db\Reader;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Appwrite\Utopia\View;
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
|
@ -19,6 +18,8 @@ use Utopia\Config\Config;
|
|||
use Utopia\Domains\Domain;
|
||||
use Appwrite\Auth\Auth;
|
||||
use Appwrite\Event\Certificate;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Usage;
|
||||
use Appwrite\Network\Validator\Origin;
|
||||
use Appwrite\Utopia\Response\Filters\V11 as ResponseV11;
|
||||
use Appwrite\Utopia\Response\Filters\V12 as ResponseV12;
|
||||
|
@ -31,6 +32,7 @@ use Utopia\Database\Database;
|
|||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Validator\Hostname;
|
||||
use Appwrite\Utopia\Request\Filters\V12 as RequestV12;
|
||||
|
@ -38,6 +40,7 @@ use Appwrite\Utopia\Request\Filters\V13 as RequestV13;
|
|||
use Appwrite\Utopia\Request\Filters\V14 as RequestV14;
|
||||
use Appwrite\Utopia\Request\Filters\V15 as RequestV15;
|
||||
use Appwrite\Utopia\Request\Filters\V16 as RequestV16;
|
||||
use Executor\Executor;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
|
||||
|
@ -45,7 +48,7 @@ Config::setParam('domainVerification', false);
|
|||
Config::setParam('cookieDomain', 'localhost');
|
||||
Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);
|
||||
|
||||
function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleRequest, Request $request, Response $response)
|
||||
function router(App $utopia, Database $dbForConsole, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Reader $geodb)
|
||||
{
|
||||
$utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml');
|
||||
|
||||
|
@ -117,59 +120,218 @@ function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleReques
|
|||
$path .= '?' . $query;
|
||||
}
|
||||
|
||||
|
||||
$body = $swooleRequest->getContent() ?? '';
|
||||
$method = $swooleRequest->server['request_method'];
|
||||
|
||||
$requestHeaders = $request->getHeaders();
|
||||
|
||||
$body = \json_encode([
|
||||
'async' => false,
|
||||
'body' => $swooleRequest->getContent() ?? '',
|
||||
'method' => $swooleRequest->server['request_method'],
|
||||
'path' => $path,
|
||||
'headers' => $requestHeaders
|
||||
$project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId));
|
||||
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
|
||||
|
||||
if ($function->isEmpty() || !$function->getAttribute('enabled')) {
|
||||
throw new AppwriteException(AppwriteException::FUNCTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
|
||||
|
||||
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null;
|
||||
|
||||
if (\is_null($runtime)) {
|
||||
throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
||||
}
|
||||
|
||||
$deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $function->getAttribute('deployment', '')));
|
||||
|
||||
if ($deployment->getAttribute('resourceId') !== $function->getId()) {
|
||||
throw new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function');
|
||||
}
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
throw new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function');
|
||||
}
|
||||
|
||||
/** Check if build has completed */
|
||||
$build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')));
|
||||
if ($build->isEmpty()) {
|
||||
throw new AppwriteException(AppwriteException::BUILD_NOT_FOUND);
|
||||
}
|
||||
|
||||
if ($build->getAttribute('status') !== 'ready') {
|
||||
throw new AppwriteException(AppwriteException::BUILD_NOT_READY);
|
||||
}
|
||||
|
||||
$permissions = $function->getAttribute('execute');
|
||||
|
||||
if (!(\in_array('any', $permissions)) && (\in_array('guests', $permissions))) {
|
||||
throw new AppwriteException(AppwriteException::USER_UNAUTHORIZED, 'To execute function using domain, execute permissions must include "any" or "guests"');
|
||||
}
|
||||
|
||||
$headers = \array_merge([], $requestHeaders);
|
||||
$headers['x-appwrite-trigger'] = 'http';
|
||||
$headers['x-appwrite-user-id'] = '';
|
||||
$headers['x-appwrite-user-jwt'] = '';
|
||||
$headers['x-appwrite-country-code'] = '';
|
||||
$headers['x-appwrite-continent-code'] = '';
|
||||
$headers['x-appwrite-continent-eu'] = 'false';
|
||||
|
||||
$ip = $headers['x-real-ip'] ?? '';
|
||||
if (!empty($ip)) {
|
||||
$record = $geodb->get($ip);
|
||||
|
||||
if ($record) {
|
||||
$eu = Config::getParam('locale-eu');
|
||||
|
||||
$headers['x-appwrite-country-code'] = $record['country']['iso_code'] ?? '';
|
||||
$headers['x-appwrite-continent-code'] = $record['continent']['code'] ?? '';
|
||||
$headers['x-appwrite-continent-eu'] = (\in_array($record['country']['iso_code'], $eu)) ? 'true' : 'false';
|
||||
}
|
||||
}
|
||||
|
||||
$headersFiltered = [];
|
||||
foreach ($headers as $key => $value) {
|
||||
if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_REQUEST)) {
|
||||
$headersFiltered[] = ['name' => $key, 'value' => $value];
|
||||
}
|
||||
}
|
||||
|
||||
$executionId = ID::unique();
|
||||
|
||||
$execution = new Document([
|
||||
'$id' => $executionId,
|
||||
'$permissions' => [],
|
||||
'functionInternalId' => $function->getInternalId(),
|
||||
'functionId' => $function->getId(),
|
||||
'deploymentInternalId' => $deployment->getInternalId(),
|
||||
'deploymentId' => $deployment->getId(),
|
||||
'trigger' => 'http', // http / schedule / event
|
||||
'status' => 'processing', // waiting / processing / completed / failed
|
||||
'responseStatusCode' => 0,
|
||||
'responseHeaders' => [],
|
||||
'requestPath' => $path,
|
||||
'requestMethod' => $method,
|
||||
'requestHeaders' => $headersFiltered,
|
||||
'errors' => '',
|
||||
'logs' => '',
|
||||
'duration' => 0.0,
|
||||
'search' => implode(' ', [$functionId, $executionId]),
|
||||
]);
|
||||
|
||||
$headers = [
|
||||
'Content-Type: application/json',
|
||||
'Content-Length: ' . \strlen($body),
|
||||
'X-Appwrite-Project: ' . $projectId
|
||||
];
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
->setParam('executionId', $execution->getId())
|
||||
->setContext('function', $function);
|
||||
|
||||
$ch = \curl_init();
|
||||
\curl_setopt($ch, CURLOPT_URL, "http://localhost/v1/functions/{$functionId}/executions");
|
||||
\curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
|
||||
\curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
|
||||
\curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
|
||||
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
// \curl_setopt($ch, CURLOPT_HEADER, true);
|
||||
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
||||
\curl_setopt($ch, CURLOPT_TIMEOUT, 30);
|
||||
$durationStart = \microtime(true);
|
||||
|
||||
$executionResponse = \curl_exec($ch);
|
||||
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$error = \curl_error($ch);
|
||||
$errNo = \curl_errno($ch);
|
||||
$vars = [];
|
||||
|
||||
\curl_close($ch);
|
||||
|
||||
if ($errNo !== 0) {
|
||||
throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, "Internal error: " . $error);
|
||||
// V2 vars
|
||||
if ($version === 'v2') {
|
||||
$vars = \array_merge($vars, [
|
||||
'APPWRITE_FUNCTION_TRIGGER' => $headers['x-appwrite-trigger'] ?? '',
|
||||
'APPWRITE_FUNCTION_DATA' => $body ?? '',
|
||||
'APPWRITE_FUNCTION_USER_ID' => $headers['x-appwrite-user-id'] ?? '',
|
||||
'APPWRITE_FUNCTION_JWT' => $headers['x-appwrite-user-jwt'] ?? ''
|
||||
]);
|
||||
}
|
||||
|
||||
if ($statusCode >= 400) {
|
||||
$error = \json_decode($executionResponse, true)['message'];
|
||||
throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, "Execution error: " . $error);
|
||||
// Shared vars
|
||||
foreach ($function->getAttribute('varsProject', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
|
||||
$execution = \json_decode($executionResponse, true);
|
||||
// Function vars
|
||||
foreach ($function->getAttribute('vars', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
|
||||
$contentType = 'text/plain';
|
||||
foreach ($execution['responseHeaders'] as $header) {
|
||||
if (\strtolower($header['name']) === 'content-type') {
|
||||
$contentType = $header['value'];
|
||||
// Appwrite vars
|
||||
$vars = \array_merge($vars, [
|
||||
'APPWRITE_FUNCTION_ID' => $functionId,
|
||||
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
|
||||
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
|
||||
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
|
||||
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
|
||||
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
|
||||
]);
|
||||
|
||||
/** Execute function */
|
||||
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
|
||||
try {
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$command = $runtime['startCommand'];
|
||||
$command = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"';
|
||||
$executionResponse = $executor->createExecution(
|
||||
projectId: $project->getId(),
|
||||
deploymentId: $deployment->getId(),
|
||||
body: \strlen($body) > 0 ? $body : null,
|
||||
variables: $vars,
|
||||
timeout: $function->getAttribute('timeout', 0),
|
||||
image: $runtime['image'],
|
||||
source: $build->getAttribute('path', ''),
|
||||
entrypoint: $deployment->getAttribute('entrypoint', ''),
|
||||
version: $version,
|
||||
path: $path,
|
||||
method: $method,
|
||||
headers: $headers,
|
||||
runtimeEntrypoint: $command,
|
||||
requestTimeout: 30
|
||||
);
|
||||
|
||||
$headersFiltered = [];
|
||||
foreach ($executionResponse['headers'] as $key => $value) {
|
||||
if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_RESPONSE)) {
|
||||
$headersFiltered[] = ['name' => $key, 'value' => $value];
|
||||
}
|
||||
}
|
||||
|
||||
$response->setHeader($header['name'], $header['value']);
|
||||
/** Update execution status */
|
||||
$status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed';
|
||||
$execution->setAttribute('status', $status);
|
||||
$execution->setAttribute('responseStatusCode', $executionResponse['statusCode']);
|
||||
$execution->setAttribute('responseHeaders', $headersFiltered);
|
||||
$execution->setAttribute('logs', $executionResponse['logs']);
|
||||
$execution->setAttribute('errors', $executionResponse['errors']);
|
||||
$execution->setAttribute('duration', $executionResponse['duration']);
|
||||
} catch (\Throwable $th) {
|
||||
$durationEnd = \microtime(true);
|
||||
|
||||
$execution
|
||||
->setAttribute('duration', $durationEnd - $durationStart)
|
||||
->setAttribute('status', 'failed')
|
||||
->setAttribute('responseStatusCode', 500)
|
||||
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
|
||||
Console::error($th->getMessage());
|
||||
} finally {
|
||||
$queueForUsage
|
||||
->addMetric(METRIC_EXECUTIONS, 1)
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
|
||||
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
|
||||
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
|
||||
;
|
||||
}
|
||||
|
||||
if ($function->getAttribute('logging')) {
|
||||
/** @var Document $execution */
|
||||
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
|
||||
}
|
||||
|
||||
$execution->setAttribute('logs', '');
|
||||
$execution->setAttribute('errors', '');
|
||||
|
||||
$headers = [];
|
||||
foreach (($executionResponse['headers'] ?? []) as $key => $value) {
|
||||
$headers[] = ['name' => $key, 'value' => $value];
|
||||
}
|
||||
|
||||
$execution->setAttribute('responseBody', $executionResponse['body'] ?? '');
|
||||
$execution->setAttribute('responseHeaders', $headers);
|
||||
|
||||
$body = $execution['responseBody'] ?? '';
|
||||
|
||||
$encodingKey = \array_search('x-open-runtimes-encoding', \array_column($execution['responseHeaders'], 'name'));
|
||||
|
@ -179,6 +341,15 @@ function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleReques
|
|||
}
|
||||
}
|
||||
|
||||
$contentType = 'text/plain';
|
||||
foreach ($execution['responseHeaders'] as $header) {
|
||||
if (\strtolower($header['name']) === 'content-type') {
|
||||
$contentType = $header['value'];
|
||||
}
|
||||
|
||||
$response->setHeader($header['name'], $header['value']);
|
||||
}
|
||||
|
||||
$response
|
||||
->setContentType($contentType)
|
||||
->setStatusCode($execution['responseStatusCode'] ?? 200)
|
||||
|
@ -205,13 +376,17 @@ App::init()
|
|||
->inject('console')
|
||||
->inject('project')
|
||||
->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->inject('user')
|
||||
->inject('locale')
|
||||
->inject('localeCodes')
|
||||
->inject('clients')
|
||||
->inject('servers')
|
||||
->inject('queueForCertificates')
|
||||
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, Document $user, Locale $locale, array $localeCodes, array $clients, array $servers, Certificate $queueForCertificates) {
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForUsage')
|
||||
->inject('geodb')
|
||||
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, callable $getProjectDB, Document $user, Locale $locale, array $localeCodes, array $clients, array $servers, Certificate $queueForCertificates, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) {
|
||||
/*
|
||||
* Appwrite Router
|
||||
*/
|
||||
|
@ -220,7 +395,7 @@ App::init()
|
|||
$mainDomain = App::getEnv('_APP_DOMAIN', '');
|
||||
// Only run Router when external domain
|
||||
if ($host !== $mainDomain) {
|
||||
if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) {
|
||||
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
@ -571,7 +746,11 @@ App::options()
|
|||
->inject('request')
|
||||
->inject('response')
|
||||
->inject('dbForConsole')
|
||||
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole) {
|
||||
->inject('getProjectDB')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForUsage')
|
||||
->inject('geodb')
|
||||
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Reader $geodb) {
|
||||
/*
|
||||
* Appwrite Router
|
||||
*/
|
||||
|
@ -579,7 +758,7 @@ App::options()
|
|||
$mainDomain = App::getEnv('_APP_DOMAIN', '');
|
||||
// Only run Router when external domain
|
||||
if ($host !== $mainDomain) {
|
||||
if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) {
|
||||
if (router($utopia, $dbForConsole, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $geodb)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
@ -757,10 +936,10 @@ App::error()
|
|||
->setParam('development', App::isDevelopment())
|
||||
->setParam('projectName', $project->getAttribute('name'))
|
||||
->setParam('projectURL', $project->getAttribute('url'))
|
||||
->setParam('message', $error->getMessage())
|
||||
->setParam('type', $type)
|
||||
->setParam('code', $code)
|
||||
->setParam('trace', $trace);
|
||||
->setParam('message', $output['message'] ?? '')
|
||||
->setParam('type', $output['type'] ?? '')
|
||||
->setParam('code', $output['code'] ?? '')
|
||||
->setParam('trace', $output['trace'] ?? []);
|
||||
|
||||
$response->html($layout->render());
|
||||
}
|
||||
|
|
|
@ -252,22 +252,24 @@ App::init()
|
|||
;
|
||||
|
||||
$useCache = $route->getLabel('cache', false);
|
||||
|
||||
if ($useCache) {
|
||||
$key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER);
|
||||
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
|
||||
$cache = new Cache(
|
||||
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
|
||||
);
|
||||
$timestamp = 60 * 60 * 24 * 30;
|
||||
$data = $cache->load($key, $timestamp);
|
||||
|
||||
if (!empty($data) && !$cacheLog->isEmpty()) {
|
||||
$parts = explode('/', $cacheLog->getAttribute('resourceType'));
|
||||
if (!empty($data)) {
|
||||
$data = json_decode($data, true);
|
||||
$parts = explode('/', $data['resourceType']);
|
||||
$type = $parts[0] ?? null;
|
||||
|
||||
if ($type === 'bucket') {
|
||||
$bucketId = $parts[1] ?? null;
|
||||
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
|
||||
|
||||
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
|
||||
|
||||
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
|
||||
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
|
||||
|
@ -279,12 +281,11 @@ App::init()
|
|||
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
|
||||
$validator = new Authorization(Database::PERMISSION_READ);
|
||||
$valid = $validator->isValid($bucket->getRead());
|
||||
|
||||
if (!$fileSecurity && !$valid) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED);
|
||||
}
|
||||
|
||||
$parts = explode('/', $cacheLog->getAttribute('resource'));
|
||||
$parts = explode('/', $data['resource']);
|
||||
$fileId = $parts[1] ?? null;
|
||||
|
||||
if ($fileSecurity && !$valid) {
|
||||
|
@ -301,8 +302,8 @@ App::init()
|
|||
$response
|
||||
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT')
|
||||
->addHeader('X-Appwrite-Cache', 'hit')
|
||||
->setContentType($cacheLog->getAttribute('mimeType'))
|
||||
->send($data)
|
||||
->setContentType($data['contentType'])
|
||||
->send(base64_decode($data['payload']))
|
||||
;
|
||||
} else {
|
||||
$response->addHeader('X-Appwrite-Cache', 'miss');
|
||||
|
@ -533,6 +534,7 @@ App::shutdown()
|
|||
if ($useCache) {
|
||||
$resource = $resourceType = null;
|
||||
$data = $response->getPayload();
|
||||
|
||||
if (!empty($data['payload'])) {
|
||||
$pattern = $route->getLabel('cache.resource', null);
|
||||
if (!empty($pattern)) {
|
||||
|
@ -544,17 +546,22 @@ App::shutdown()
|
|||
$resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user);
|
||||
}
|
||||
|
||||
$key = md5($request->getURI() . '*' . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
|
||||
$signature = md5($data['payload']);
|
||||
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
|
||||
$key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER);
|
||||
$data = json_encode([
|
||||
'resourceType' => $resourceType,
|
||||
'resource' => $resource,
|
||||
'contentType' => $response->getContentType(),
|
||||
'payload' => base64_encode($data['payload']),
|
||||
]) ;
|
||||
|
||||
$signature = md5($data);
|
||||
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
|
||||
$accessedAt = $cacheLog->getAttribute('accessedAt', '');
|
||||
$now = DateTime::now();
|
||||
if ($cacheLog->isEmpty()) {
|
||||
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
|
||||
'$id' => $key,
|
||||
'resource' => $resource,
|
||||
'resourceType' => $resourceType,
|
||||
'mimeType' => $response->getContentType(),
|
||||
'accessedAt' => $now,
|
||||
'signature' => $signature,
|
||||
])));
|
||||
|
@ -567,7 +574,7 @@ App::shutdown()
|
|||
$cache = new Cache(
|
||||
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
|
||||
);
|
||||
$cache->save($key, $data['payload']);
|
||||
$cache->save($key, $data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -112,7 +112,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
|
|||
const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
|
||||
const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
|
||||
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
|
||||
const APP_CACHE_BUSTER = 330;
|
||||
const APP_CACHE_BUSTER = 331;
|
||||
const APP_VERSION_STABLE = '1.4.13';
|
||||
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
|
||||
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
|
||||
|
|
|
@ -253,7 +253,6 @@ try {
|
|||
Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine());
|
||||
}
|
||||
|
||||
|
||||
$worker = $platform->getWorker();
|
||||
|
||||
$worker
|
||||
|
@ -268,7 +267,8 @@ $worker
|
|||
->inject('error')
|
||||
->inject('logger')
|
||||
->inject('log')
|
||||
->action(function (Throwable $error, ?Logger $logger, Log $log) {
|
||||
->inject('project')
|
||||
->action(function (Throwable $error, ?Logger $logger, Log $log, Document $project) {
|
||||
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
|
||||
|
||||
if ($error instanceof PDOException) {
|
||||
|
@ -284,6 +284,7 @@ $worker
|
|||
$log->setAction('appwrite-queue-' . App::getEnv('QUEUE'));
|
||||
$log->addTag('verboseType', get_class($error));
|
||||
$log->addTag('code', $error->getCode());
|
||||
$log->addTag('projectId', $project->getId() ?? 'n/a');
|
||||
$log->addExtra('file', $error->getFile());
|
||||
$log->addExtra('line', $error->getLine());
|
||||
$log->addExtra('trace', $error->getTraceAsString());
|
||||
|
|
|
@ -57,7 +57,7 @@
|
|||
"utopia-php/image": "0.6.*",
|
||||
"utopia-php/locale": "0.4.*",
|
||||
"utopia-php/logger": "0.3.*",
|
||||
"utopia-php/messaging": "0.2.*",
|
||||
"utopia-php/messaging": "0.3.*",
|
||||
"utopia-php/migration": "0.3.*",
|
||||
"utopia-php/orchestration": "0.9.*",
|
||||
"utopia-php/platform": "0.5.*",
|
||||
|
|
20
composer.lock
generated
20
composer.lock
generated
|
@ -4,7 +4,7 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "6d1a2fabffa05c7d7a2094b04dc9e8b9",
|
||||
"content-hash": "f7a4173bb5adeea3518f949fa883a282",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
|
@ -156,11 +156,11 @@
|
|||
},
|
||||
{
|
||||
"name": "appwrite/php-runtimes",
|
||||
"version": "0.13.2",
|
||||
"version": "0.13.3",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/appwrite/runtimes.git",
|
||||
"reference": "214a37c2c66e0f2bc9c30fdfde66955d9fd084a1"
|
||||
"reference": "5d93fc578a9a543bcdc9b2c0562d80a51d56c73d"
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.0",
|
||||
|
@ -195,7 +195,7 @@
|
|||
"php",
|
||||
"runtimes"
|
||||
],
|
||||
"time": "2023-11-22T15:36:00+00:00"
|
||||
"time": "2024-03-01T14:47:47+00:00"
|
||||
},
|
||||
{
|
||||
"name": "chillerlan/php-qrcode",
|
||||
|
@ -1547,16 +1547,16 @@
|
|||
},
|
||||
{
|
||||
"name": "utopia-php/messaging",
|
||||
"version": "0.2.0",
|
||||
"version": "0.3.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/messaging.git",
|
||||
"reference": "2d0f474a106bb1da285f85e105c29b46085d3a43"
|
||||
"reference": "d488223876f88f97bb76fd6681fed0df80558f62"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/messaging/zipball/2d0f474a106bb1da285f85e105c29b46085d3a43",
|
||||
"reference": "2d0f474a106bb1da285f85e105c29b46085d3a43",
|
||||
"url": "https://api.github.com/repos/utopia-php/messaging/zipball/d488223876f88f97bb76fd6681fed0df80558f62",
|
||||
"reference": "d488223876f88f97bb76fd6681fed0df80558f62",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
|
@ -1589,9 +1589,9 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/messaging/issues",
|
||||
"source": "https://github.com/utopia-php/messaging/tree/0.2.0"
|
||||
"source": "https://github.com/utopia-php/messaging/tree/0.3.0"
|
||||
},
|
||||
"time": "2023-09-14T20:48:42+00:00"
|
||||
"time": "2023-11-14T21:02:37+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/migration",
|
||||
|
|
|
@ -927,7 +927,6 @@ services:
|
|||
# - appwrite
|
||||
# volumes:
|
||||
# - appwrite-uploads:/storage/uploads
|
||||
|
||||
# Dev Tools Start ------------------------------------------------------------------------------------------
|
||||
#
|
||||
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
Get the number of projects containing metrics that are waiting to be processed in the Appwrite internal queue server.
|
|
@ -1 +0,0 @@
|
|||
Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.
|
1
docs/references/health/get-storage.md
Normal file
1
docs/references/health/get-storage.md
Normal file
|
@ -0,0 +1 @@
|
|||
Check the Appwrite storage device is up and connection is successful.
|
|
@ -10,7 +10,6 @@ class Delete extends Event
|
|||
{
|
||||
protected string $type = '';
|
||||
protected ?Document $document = null;
|
||||
protected ?string $resourceType = null;
|
||||
protected ?string $resource = null;
|
||||
protected ?string $datetime = null;
|
||||
protected ?string $hourlyUsageRetentionDatetime = null;
|
||||
|
@ -108,19 +107,6 @@ class Delete extends Event
|
|||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the resource type for the delete event.
|
||||
*
|
||||
* @param string $resourceType
|
||||
* @return self
|
||||
*/
|
||||
public function setResourceType(string $resourceType): self
|
||||
{
|
||||
$this->resourceType = $resourceType;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the set document for the delete event.
|
||||
*
|
||||
|
@ -147,7 +133,6 @@ class Delete extends Event
|
|||
'type' => $this->type,
|
||||
'document' => $this->document,
|
||||
'resource' => $this->resource,
|
||||
'resourceType' => $this->resourceType,
|
||||
'datetime' => $this->datetime,
|
||||
'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime
|
||||
]);
|
||||
|
|
|
@ -338,6 +338,14 @@ class Mail extends Event
|
|||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set attachment
|
||||
* @param string $content
|
||||
* @param string $filename
|
||||
* @param string $encoding
|
||||
* @param string $type
|
||||
* @return self
|
||||
*/
|
||||
public function setAttachment(string $content, string $filename, string $encoding = 'base64', string $type = 'plain/text')
|
||||
{
|
||||
$this->attachment = [
|
||||
|
@ -349,11 +357,45 @@ class Mail extends Event
|
|||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get attachment
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getAttachment(): array
|
||||
{
|
||||
return $this->attachment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset attachment
|
||||
*
|
||||
* @return self
|
||||
*/
|
||||
public function resetAttachment(): self
|
||||
{
|
||||
$this->attachment = [];
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset
|
||||
*
|
||||
* @return self
|
||||
*/
|
||||
public function reset(): self
|
||||
{
|
||||
$this->project = null;
|
||||
$this->recipient = '';
|
||||
$this->name = '';
|
||||
$this->subject = '';
|
||||
$this->body = '';
|
||||
$this->variables = [];
|
||||
$this->bodyTemplate = '';
|
||||
$this->attachment = [];
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the event and sends it to the mails worker.
|
||||
*
|
||||
|
@ -365,6 +407,7 @@ class Mail extends Event
|
|||
$client = new Client($this->queue, $this->connection);
|
||||
|
||||
return $client->enqueue([
|
||||
'project' => $this->project,
|
||||
'recipient' => $this->recipient,
|
||||
'name' => $this->name,
|
||||
'subject' => $this->subject,
|
||||
|
|
|
@ -270,7 +270,7 @@ class CalcTierStats extends Action
|
|||
$limit = $periods[$range]['limit'];
|
||||
$period = $periods[$range]['period'];
|
||||
|
||||
$requestDocs = $dbForProject->find('stats', [
|
||||
$requestDocs = $dbForProject->find('stats_v2', [
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::equal('period', [$period]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -167,8 +167,8 @@ class CreateInfMetric extends Action
|
|||
|
||||
try {
|
||||
$id = \md5("_inf_{$metric}");
|
||||
$dbForProject->deleteDocument('stats', $id);
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
$dbForProject->deleteDocument('stats_v2', $id);
|
||||
$dbForProject->createDocument('stats_v2', new Document([
|
||||
'$id' => $id,
|
||||
'metric' => $metric,
|
||||
'period' => 'inf',
|
||||
|
@ -190,7 +190,7 @@ class CreateInfMetric extends Action
|
|||
protected function getFromMetric(database $dbForProject, string $metric): int|float
|
||||
{
|
||||
|
||||
return $dbForProject->sum('stats', 'value', [
|
||||
return $dbForProject->sum('stats_v2', 'value', [
|
||||
Query::equal('metric', [
|
||||
$metric,
|
||||
]),
|
||||
|
|
|
@ -7,7 +7,9 @@ use Utopia\CLI\Console;
|
|||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Client;
|
||||
use Utopia\Queue\Connection;
|
||||
use Utopia\Validator\Integer;
|
||||
use Utopia\Validator\WhiteList;
|
||||
use Utopia\Validator\Wildcard;
|
||||
|
||||
class QueueRetry extends Action
|
||||
{
|
||||
|
@ -35,21 +37,25 @@ class QueueRetry extends Action
|
|||
Event::MIGRATIONS_QUEUE_NAME,
|
||||
Event::HAMSTER_CLASS_NAME
|
||||
]), 'Queue name')
|
||||
->param('limit', 0, new Wildcard(), 'jobs limit', true)
|
||||
->inject('queue')
|
||||
->callback(fn ($name, $queue) => $this->action($name, $queue));
|
||||
->callback(fn ($name, $limit, $queue) => $this->action($name, $limit, $queue));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $name The name of the queue to retry jobs from
|
||||
* @param mixed $limit
|
||||
* @param Connection $queue
|
||||
*/
|
||||
public function action(string $name, Connection $queue): void
|
||||
public function action(string $name, mixed $limit, Connection $queue): void
|
||||
{
|
||||
|
||||
if (!$name) {
|
||||
Console::error('Missing required parameter $name');
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = (int)$limit;
|
||||
$queueClient = new Client($name, $queue);
|
||||
|
||||
if ($queueClient->countFailedJobs() === 0) {
|
||||
|
@ -59,6 +65,6 @@ class QueueRetry extends Action
|
|||
|
||||
Console::log('Retrying failed jobs...');
|
||||
|
||||
$queueClient->retry();
|
||||
$queueClient->retry($limit);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -68,13 +68,18 @@ class Deletes extends Action
|
|||
$datetime = $payload['datetime'] ?? null;
|
||||
$hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null;
|
||||
$resource = $payload['resource'] ?? null;
|
||||
$resourceType = $payload['resourceType'] ?? null;
|
||||
$document = new Document($payload['document'] ?? []);
|
||||
$project = new Document($payload['project'] ?? []);
|
||||
|
||||
switch (strval($type)) {
|
||||
case DELETE_TYPE_DOCUMENT:
|
||||
switch ($document->getCollection()) {
|
||||
case DELETE_TYPE_DATABASES:
|
||||
$this->deleteDatabase($getProjectDB, $document, $project);
|
||||
break;
|
||||
case DELETE_TYPE_COLLECTIONS:
|
||||
$this->deleteCollection($getProjectDB, $document, $project);
|
||||
break;
|
||||
case DELETE_TYPE_PROJECTS:
|
||||
$this->deleteProject($dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $document);
|
||||
break;
|
||||
|
@ -103,6 +108,10 @@ class Deletes extends Action
|
|||
$this->deleteRule($dbForConsole, $document);
|
||||
break;
|
||||
default:
|
||||
if (\str_starts_with($document->getCollection(), 'database_')) {
|
||||
$this->deleteCollection($getProjectDB, $document, $project);
|
||||
break;
|
||||
}
|
||||
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
|
||||
break;
|
||||
}
|
||||
|
@ -136,7 +145,7 @@ class Deletes extends Action
|
|||
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
|
||||
break;
|
||||
case DELETE_TYPE_CACHE_BY_RESOURCE:
|
||||
$this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType);
|
||||
$this->deleteCacheByResource($project, $getProjectDB, $resource);
|
||||
break;
|
||||
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
|
||||
$this->deleteCacheByDate($project, $getProjectDB, $datetime);
|
||||
|
@ -194,37 +203,32 @@ class Deletes extends Action
|
|||
* @param string $resource
|
||||
* @return void
|
||||
* @throws Authorization
|
||||
* @param string|null $resourceType
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void
|
||||
private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void
|
||||
{
|
||||
$projectId = $project->getId();
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]);
|
||||
|
||||
$cache = new Cache(
|
||||
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
|
||||
);
|
||||
if ($document) {
|
||||
$cache = new Cache(
|
||||
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
|
||||
);
|
||||
|
||||
$query[] = Query::equal('resource', [$resource]);
|
||||
if (!empty($resourceType)) {
|
||||
$query[] = Query::equal('resourceType', [$resourceType]);
|
||||
}
|
||||
$this->deleteById(
|
||||
$document,
|
||||
$dbForProject,
|
||||
function ($document) use ($cache, $projectId) {
|
||||
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
|
||||
|
||||
$this->deleteByGroup(
|
||||
'cache',
|
||||
$query,
|
||||
$dbForProject,
|
||||
function (Document $document) use ($cache, $projectId) {
|
||||
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
|
||||
|
||||
if ($cache->purge($document->getId())) {
|
||||
Console::success('Deleting cache file: ' . $path);
|
||||
} else {
|
||||
Console::error('Failed to delete cache file: ' . $path);
|
||||
if ($cache->purge($document->getId())) {
|
||||
Console::success('Deleting cache file: ' . $path);
|
||||
} else {
|
||||
Console::error('Failed to delete cache file: ' . $path);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -264,6 +268,72 @@ class Deletes extends Action
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param callable $getProjectDB
|
||||
* @param Document $document
|
||||
* @param Document $project
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void
|
||||
{
|
||||
$databaseId = $document->getId();
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
$this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) {
|
||||
$this->deleteCollection($getProjectDB, $document, $project);
|
||||
});
|
||||
|
||||
$dbForProject->deleteCollection('database_' . $document->getInternalId());
|
||||
$this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param callable $getProjectDB
|
||||
* @param Document $document teams document
|
||||
* @param Document $project
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void
|
||||
{
|
||||
$collectionId = $document->getId();
|
||||
$collectionInternalId = $document->getInternalId();
|
||||
$databaseId = $document->getAttribute('databaseId');
|
||||
$databaseInternalId = $document->getAttribute('databaseInternalId');
|
||||
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
$relationships = \array_filter(
|
||||
$document->getAttribute('attributes'),
|
||||
fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP
|
||||
);
|
||||
|
||||
foreach ($relationships as $relationship) {
|
||||
if (!$relationship['twoWay']) {
|
||||
continue;
|
||||
}
|
||||
$relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']);
|
||||
$dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']);
|
||||
$dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId());
|
||||
$dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId());
|
||||
}
|
||||
|
||||
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
|
||||
|
||||
$this->deleteByGroup('attributes', [
|
||||
Query::equal('databaseInternalId', [$databaseInternalId]),
|
||||
Query::equal('collectionInternalId', [$collectionInternalId])
|
||||
], $dbForProject);
|
||||
|
||||
$this->deleteByGroup('indexes', [
|
||||
Query::equal('databaseInternalId', [$databaseInternalId]),
|
||||
Query::equal('collectionInternalId', [$collectionInternalId])
|
||||
], $dbForProject);
|
||||
|
||||
$this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Database $dbForConsole
|
||||
* @param callable $getProjectDB
|
||||
|
@ -275,7 +345,7 @@ class Deletes extends Action
|
|||
{
|
||||
$dbForProject = $getProjectDB($project);
|
||||
// Delete Usage stats
|
||||
$this->deleteByGroup('stats', [
|
||||
$this->deleteByGroup('stats_v2', [
|
||||
Query::lessThan('time', $hourlyUsageRetentionDatetime),
|
||||
Query::equal('period', ['1h']),
|
||||
], $dbForProject);
|
||||
|
|
|
@ -286,7 +286,7 @@ class Hamster extends Action
|
|||
$limit = $periodValue['limit'];
|
||||
$period = $periodValue['period'];
|
||||
|
||||
$requestDocs = $dbForProject->find('stats', [
|
||||
$requestDocs = $dbForProject->find('stats_v2', [
|
||||
Query::equal('period', [$period]),
|
||||
Query::equal('metric', [$metric]),
|
||||
Query::limit($limit),
|
||||
|
|
|
@ -88,14 +88,29 @@ class Messaging extends Action
|
|||
return;
|
||||
}
|
||||
|
||||
$sms = match ($this->dsn->getHost()) {
|
||||
'mock' => new Mock($this->user, $this->secret), // used for tests
|
||||
'twilio' => new Twilio($this->user, $this->secret),
|
||||
'text-magic' => new TextMagic($this->user, $this->secret),
|
||||
'telesign' => new Telesign($this->user, $this->secret),
|
||||
'msg91' => new Msg91($this->user, $this->secret),
|
||||
'vonage' => new Vonage($this->user, $this->secret),
|
||||
default => null
|
||||
|
||||
switch ($this->dsn->getHost()) {
|
||||
case 'mock':
|
||||
$sms = new Mock($this->user, $this->secret); // used for tests
|
||||
break;
|
||||
case 'twilio':
|
||||
$sms = new Twilio($this->user, $this->secret);
|
||||
break;
|
||||
case 'text-magic':
|
||||
$sms = new TextMagic($this->user, $this->secret);
|
||||
break;
|
||||
case 'telesign':
|
||||
$sms = new Telesign($this->user, $this->secret);
|
||||
break;
|
||||
case 'msg91':
|
||||
$sms = new Msg91($this->user, $this->secret);
|
||||
$sms->setTemplate($this->dsn->getParam('template'));
|
||||
break;
|
||||
case 'vonage':
|
||||
$sms = new Vonage($this->user, $this->secret);
|
||||
break;
|
||||
default:
|
||||
$sms = null;
|
||||
};
|
||||
|
||||
if (empty(App::getEnv('_APP_SMS_PROVIDER'))) {
|
||||
|
|
|
@ -76,6 +76,7 @@ class Usage extends Action
|
|||
}
|
||||
|
||||
$this->stats[$projectId]['project'] = $project;
|
||||
$this->stats[$projectId]['receivedAt'] = DateTime::now();
|
||||
foreach ($payload['metrics'] ?? [] as $metric) {
|
||||
$this->keys++;
|
||||
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
|
||||
|
@ -129,8 +130,8 @@ class Usage extends Action
|
|||
}
|
||||
break;
|
||||
case $document->getCollection() === 'databases': // databases
|
||||
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
|
||||
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
|
||||
$collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
|
||||
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
|
||||
if (!empty($collections['value'])) {
|
||||
$metrics[] = [
|
||||
'key' => METRIC_COLLECTIONS,
|
||||
|
@ -148,7 +149,7 @@ class Usage extends Action
|
|||
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
|
||||
$parts = explode('_', $document->getCollection());
|
||||
$databaseInternalId = $parts[1] ?? 0;
|
||||
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
|
||||
$documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
|
||||
|
||||
if (!empty($documents['value'])) {
|
||||
$metrics[] = [
|
||||
|
@ -163,8 +164,8 @@ class Usage extends Action
|
|||
break;
|
||||
|
||||
case $document->getCollection() === 'buckets':
|
||||
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
|
||||
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
|
||||
$files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
|
||||
$storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
|
||||
|
||||
if (!empty($files['value'])) {
|
||||
$metrics[] = [
|
||||
|
@ -182,13 +183,13 @@ class Usage extends Action
|
|||
break;
|
||||
|
||||
case $document->getCollection() === 'functions':
|
||||
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
|
||||
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
|
||||
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
|
||||
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
|
||||
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
|
||||
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
|
||||
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
|
||||
$deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
|
||||
$deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
|
||||
$builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
|
||||
$buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
|
||||
$buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
|
||||
$executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
|
||||
$executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
|
||||
|
||||
if (!empty($deployments['value'])) {
|
||||
$metrics[] = [
|
||||
|
|
|
@ -59,12 +59,12 @@ class UsageDump extends Action
|
|||
foreach ($payload['stats'] ?? [] as $stats) {
|
||||
$project = new Document($stats['project'] ?? []);
|
||||
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
|
||||
|
||||
$receivedAt = $stats['receivedAt'] ?? 'NONE';
|
||||
if ($numberOfKeys === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys');
|
||||
console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] ReceivedAt [' . $receivedAt . '] ' . $numberOfKeys . ' keys');
|
||||
|
||||
try {
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
@ -78,7 +78,7 @@ class UsageDump extends Action
|
|||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
|
||||
try {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
$dbForProject->createDocument('stats_v2', new Document([
|
||||
'$id' => $id,
|
||||
'period' => $period,
|
||||
'time' => $time,
|
||||
|
@ -89,14 +89,14 @@ class UsageDump extends Action
|
|||
} catch (Duplicate $th) {
|
||||
if ($value < 0) {
|
||||
$dbForProject->decreaseDocumentAttribute(
|
||||
'stats',
|
||||
'stats_v2',
|
||||
$id,
|
||||
'value',
|
||||
abs($value)
|
||||
);
|
||||
} else {
|
||||
$dbForProject->increaseDocumentAttribute(
|
||||
'stats',
|
||||
'stats_v2',
|
||||
$id,
|
||||
'value',
|
||||
$value
|
||||
|
|
|
@ -295,7 +295,7 @@ class OpenAPI3 extends Format
|
|||
switch ((!empty($validator)) ? \get_class($validator) : '') {
|
||||
case 'Utopia\Validator\Text':
|
||||
$node['schema']['type'] = $validator->getType();
|
||||
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
|
||||
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
|
||||
break;
|
||||
case 'Utopia\Validator\Boolean':
|
||||
$node['schema']['type'] = $validator->getType();
|
||||
|
@ -303,14 +303,14 @@ class OpenAPI3 extends Format
|
|||
break;
|
||||
case 'Utopia\Database\Validator\UID':
|
||||
$node['schema']['type'] = $validator->getType();
|
||||
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
|
||||
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
|
||||
break;
|
||||
case 'Appwrite\Utopia\Database\Validator\CustomId':
|
||||
if ($route->getLabel('sdk.methodType', '') === 'upload') {
|
||||
$node['schema']['x-upload-id'] = true;
|
||||
}
|
||||
$node['schema']['type'] = $validator->getType();
|
||||
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
|
||||
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
|
||||
break;
|
||||
case 'Utopia\Database\Validator\DatetimeValidator':
|
||||
$node['schema']['type'] = $validator->getType();
|
||||
|
|
|
@ -297,7 +297,7 @@ class Swagger2 extends Format
|
|||
switch ((!empty($validator)) ? \get_class($validator) : '') {
|
||||
case 'Utopia\Validator\Text':
|
||||
$node['type'] = $validator->getType();
|
||||
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
|
||||
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
|
||||
break;
|
||||
case 'Utopia\Validator\Boolean':
|
||||
$node['type'] = $validator->getType();
|
||||
|
@ -308,11 +308,11 @@ class Swagger2 extends Format
|
|||
$node['x-upload-id'] = true;
|
||||
}
|
||||
$node['type'] = $validator->getType();
|
||||
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
|
||||
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
|
||||
break;
|
||||
case 'Utopia\Database\Validator\UID':
|
||||
$node['type'] = $validator->getType();
|
||||
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
|
||||
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
|
||||
break;
|
||||
case 'Utopia\Database\Validator\DatetimeValidator':
|
||||
$node['type'] = $validator->getType();
|
||||
|
|
|
@ -407,6 +407,24 @@ class HealthCustomServerTest extends Scope
|
|||
return [];
|
||||
}
|
||||
|
||||
public function testStorageSuccess(): array
|
||||
{
|
||||
/**
|
||||
* Test for SUCCESS
|
||||
*/
|
||||
$response = $this->client->call(Client::METHOD_GET, '/health/storage', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), []);
|
||||
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertEquals('pass', $response['body']['status']);
|
||||
$this->assertIsInt($response['body']['ping']);
|
||||
$this->assertLessThan(100, $response['body']['ping']);
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
public function testStorageAntiVirusSuccess(): array
|
||||
{
|
||||
/**
|
||||
|
@ -494,52 +512,4 @@ class HealthCustomServerTest extends Scope
|
|||
|
||||
return [];
|
||||
}
|
||||
|
||||
public function testUsageSuccess()
|
||||
{
|
||||
/**
|
||||
* Test for SUCCESS
|
||||
*/
|
||||
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), []);
|
||||
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertIsInt($response['body']['size']);
|
||||
$this->assertLessThan(100, $response['body']['size']);
|
||||
|
||||
/**
|
||||
* Test for FAILURE
|
||||
*/
|
||||
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage?threshold=0', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), []);
|
||||
$this->assertEquals(503, $response['headers']['status-code']);
|
||||
}
|
||||
|
||||
public function testUsageDumpSuccess()
|
||||
{
|
||||
/**
|
||||
* Test for SUCCESS
|
||||
*/
|
||||
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), []);
|
||||
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertIsInt($response['body']['size']);
|
||||
$this->assertLessThan(100, $response['body']['size']);
|
||||
|
||||
/**
|
||||
* Test for FAILURE
|
||||
*/
|
||||
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump?threshold=0', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), []);
|
||||
$this->assertEquals(503, $response['headers']['status-code']);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1850,7 +1850,7 @@ class ProjectsConsoleClientTest extends Scope
|
|||
'x-appwrite-project' => $id,
|
||||
], $this->getHeaders()));
|
||||
|
||||
$this->assertEquals(503, $response['headers']['status-code']);
|
||||
$this->assertEquals(403, $response['headers']['status-code']);
|
||||
|
||||
$response = $this->client->call(Client::METHOD_POST, '/teams', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
|
@ -1860,7 +1860,7 @@ class ProjectsConsoleClientTest extends Scope
|
|||
'name' => 'Arsenal'
|
||||
]);
|
||||
|
||||
$this->assertEquals(503, $response['headers']['status-code']);
|
||||
$this->assertEquals(403, $response['headers']['status-code']);
|
||||
|
||||
// Cleanup
|
||||
|
||||
|
|
Loading…
Reference in a new issue