1
0
Fork 0
mirror of synced 2024-05-02 11:52:38 +12:00

sync: main -> 1.5.x

This commit is contained in:
Torsten Dittmann 2024-03-07 17:16:39 +01:00
commit dfef0e732c
17 changed files with 196 additions and 126 deletions

2
.env
View file

@ -79,7 +79,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_USAGE_AGGREGATION_INTERVAL=20
_APP_USAGE_AGGREGATION_INTERVAL=30
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_STATS=enabled

View file

@ -1,4 +1,4 @@
> Great news! Appwrite Cloud is now in public beta! Sign up at [cloud.appwrite.io](https://cloud.appwrite.io) for a hassle-free, hosted experience. Join us in the Cloud today! ☁️🎉
> Our Appwrite Init event has concluded. You can check out all the new and upcoming features [on our Init website](https://appwrite.io/init) 🚀
<br />
<p align="center">

View file

@ -34,6 +34,28 @@ $commonCollections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'resourceType',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('mimeType'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255, // https://tools.ietf.org/html/rfc4288#section-4.2
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_DATETIME,

View file

@ -2237,7 +2237,7 @@ App::get('/v1/account/logs')
}
$response->dynamic(new Document([
'total' => $audit->countLogsByUser($user->getId()),
'total' => $audit->countLogsByUser($user->getInternalId()),
'logs' => $output,
]), Response::MODEL_LOG_LIST);
});

View file

@ -76,7 +76,7 @@ $getUserGitHub = function (string $userId, Document $project, Database $dbForPro
}
if (empty($gitHubSession)) {
throw new Exception(Exception::GENERAL_UNKNOWN, 'GitHub session not found.');
throw new Exception(Exception::USER_SESSION_NOT_FOUND, 'GitHub session not found.');
}
$provider = $gitHubSession->getAttribute('provider', '');

View file

@ -659,6 +659,60 @@ App::get('/v1/health/queue/functions')
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/usage')
->desc('Get usage queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getQueueUsage')
->label('sdk.description', '/docs/references/health/get-queue-usage.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::USAGE_QUEUE_NAME, $queue);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/queue/usage-dump')
->desc('Get usage dump queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getQueueUsage')
->label('sdk.description', '/docs/references/health/get-queue-usage-dump.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::USAGE_DUMP_QUEUE_NAME, $queue);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/storage/local')
->desc('Get local storage')
->groups(['api', 'health'])
@ -796,6 +850,7 @@ App::get('/v1/health/queue/failed/:name')
Event::MAILS_QUEUE_NAME,
Event::FUNCTIONS_QUEUE_NAME,
Event::USAGE_QUEUE_NAME,
Event::USAGE_DUMP_QUEUE_NAME,
Event::WEBHOOK_CLASS_NAME,
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,

View file

@ -1481,6 +1481,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
if ($deviceDeleted) {
$queueForDeletes
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
->setResourceType('bucket/' . $bucket->getId())
->setResource('file/' . $fileId)
;

View file

@ -813,6 +813,9 @@ App::get('/v1/users/:userId/logs')
$output[$i] = new Document([
'event' => $log['event'],
'userId' => ID::custom($log['data']['userId']),
'userEmail' => $log['data']['userEmail'] ?? null,
'userName' => $log['data']['userName'] ?? null,
'ip' => $log['ip'],
'time' => $log['time'],
'osCode' => $os['osCode'],
@ -841,7 +844,7 @@ App::get('/v1/users/:userId/logs')
}
$response->dynamic(new Document([
'total' => $audit->countLogsByUser($user->getId()),
'total' => $audit->countLogsByUser($user->getInternalId()),
'logs' => $output,
]), Response::MODEL_LOG_LIST);
});

View file

@ -404,24 +404,22 @@ App::init()
;
$useCache = $route->getLabel('cache', false);
if ($useCache) {
$key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER);
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
);
$timestamp = 60 * 60 * 24 * 30;
$data = $cache->load($key, $timestamp);
if (!empty($data)) {
$data = json_decode($data, true);
$parts = explode('/', $data['resourceType']);
if (!empty($data) && !$cacheLog->isEmpty()) {
$parts = explode('/', $cacheLog->getAttribute('resourceType'));
$type = $parts[0] ?? null;
if ($type === 'bucket') {
$bucketId = $parts[1] ?? null;
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@ -433,11 +431,12 @@ App::init()
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
$validator = new Authorization(Database::PERMISSION_READ);
$valid = $validator->isValid($bucket->getRead());
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$parts = explode('/', $data['resource']);
$parts = explode('/', $cacheLog->getAttribute('resource'));
$fileId = $parts[1] ?? null;
if ($fileSecurity && !$valid) {
@ -454,8 +453,8 @@ App::init()
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $timestamp) . ' GMT')
->addHeader('X-Appwrite-Cache', 'hit')
->setContentType($data['contentType'])
->send(base64_decode($data['payload']))
->setContentType($cacheLog->getAttribute('mimeType'))
->send($data)
;
} else {
$response->addHeader('X-Appwrite-Cache', 'miss');
@ -651,7 +650,6 @@ App::shutdown()
if ($useCache) {
$resource = $resourceType = null;
$data = $response->getPayload();
if (!empty($data['payload'])) {
$pattern = $route->getLabel('cache.resource', null);
if (!empty($pattern)) {
@ -663,22 +661,17 @@ App::shutdown()
$resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user);
}
$key = md5($request->getURI() . implode('*', $request->getParams()) . '*' . APP_CACHE_BUSTER);
$data = json_encode([
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
$signature = md5($data);
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
$key = md5($request->getURI() . '*' . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$signature = md5($data['payload']);
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
$accessedAt = $cacheLog->getAttribute('accessedAt', '');
$now = DateTime::now();
if ($cacheLog->isEmpty()) {
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
'$id' => $key,
'resource' => $resource,
'resourceType' => $resourceType,
'mimeType' => $response->getContentType(),
'accessedAt' => $now,
'signature' => $signature,
])));
@ -691,7 +684,7 @@ App::shutdown()
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())
);
$cache->save($key, $data);
$cache->save($key, $data['payload']);
}
}
}

View file

@ -971,6 +971,7 @@ services:
# - appwrite
# volumes:
# - appwrite-uploads:/storage/uploads
# Dev Tools Start ------------------------------------------------------------------------------------------
#
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack

View file

@ -0,0 +1 @@
Get the number of projects containing metrics that are waiting to be processed in the Appwrite internal queue server.

View file

@ -0,0 +1 @@
Get the number of metrics that are waiting to be processed in the Appwrite internal queue server.

View file

@ -10,6 +10,7 @@ class Delete extends Event
{
protected string $type = '';
protected ?Document $document = null;
protected ?string $resourceType = null;
protected ?string $resource = null;
protected ?string $datetime = null;
protected ?string $hourlyUsageRetentionDatetime = null;
@ -107,6 +108,19 @@ class Delete extends Event
return $this;
}
/**
* Sets the resource type for the delete event.
*
* @param string $resourceType
* @return self
*/
public function setResourceType(string $resourceType): self
{
$this->resourceType = $resourceType;
return $this;
}
/**
* Returns the set document for the delete event.
*
@ -133,6 +147,7 @@ class Delete extends Event
'type' => $this->type,
'document' => $this->document,
'resource' => $this->resource,
'resourceType' => $this->resourceType,
'datetime' => $this->datetime,
'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime
]);

View file

@ -71,6 +71,7 @@ class Deletes extends Action
$datetime = $payload['datetime'] ?? null;
$hourlyUsageRetentionDatetime = $payload['hourlyUsageRetentionDatetime'] ?? null;
$resource = $payload['resource'] ?? null;
$resourceType = $payload['resourceType'] ?? null;
$document = new Document($payload['document'] ?? []);
$project = new Document($payload['project'] ?? []);
@ -80,12 +81,6 @@ class Deletes extends Action
switch (\strval($type)) {
case DELETE_TYPE_DOCUMENT:
switch ($document->getCollection()) {
case DELETE_TYPE_DATABASES:
$this->deleteDatabase($getProjectDB, $document, $project);
break;
case DELETE_TYPE_COLLECTIONS:
$this->deleteCollection($getProjectDB, $document, $project);
break;
case DELETE_TYPE_PROJECTS:
$this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document);
break;
@ -114,10 +109,6 @@ class Deletes extends Action
$this->deleteRule($dbForConsole, $document);
break;
default:
if (\str_starts_with($document->getCollection(), 'database_')) {
$this->deleteCollection($getProjectDB, $document, $project);
break;
}
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
break;
}
@ -147,7 +138,7 @@ class Deletes extends Action
$this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project, $getProjectDB, $resource);
$this->deleteCacheByResource($project, $getProjectDB, $resource, $resourceType);
break;
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
$this->deleteCacheByDate($project, $getProjectDB, $datetime);
@ -332,32 +323,37 @@ class Deletes extends Action
* @param string $resource
* @return void
* @throws Authorization
* @param string|null $resourceType
* @throws Exception
*/
private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource): void
private function deleteCacheByResource(Document $project, callable $getProjectDB, string $resource, string $resourceType = null): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
$document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]);
if ($document) {
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$this->deleteById(
$document,
$dbForProject,
function ($document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
if ($cache->purge($document->getId())) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('Failed to delete cache file: ' . $path);
}
}
);
$query[] = Query::equal('resource', [$resource]);
if (!empty($resourceType)) {
$query[] = Query::equal('resourceType', [$resourceType]);
}
$this->deleteByGroup(
'cache',
$query,
$dbForProject,
function (Document $document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
if ($cache->purge($document->getId())) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('Failed to delete cache file: ' . $path);
}
}
);
}
/**
@ -397,72 +393,6 @@ class Deletes extends Action
);
}
/**
* @param callable $getProjectDB
* @param Document $document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteDatabase(callable $getProjectDB, Document $document, Document $project): void
{
$databaseId = $document->getId();
$dbForProject = $getProjectDB($project);
$this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($getProjectDB, $project) {
$this->deleteCollection($getProjectDB, $document, $project);
});
$dbForProject->deleteCollection('database_' . $document->getInternalId());
$this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId, $project);
}
/**
* @param callable $getProjectDB
* @param Document $document teams document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteCollection(callable $getProjectDB, Document $document, Document $project): void
{
$collectionId = $document->getId();
$collectionInternalId = $document->getInternalId();
$databaseId = $document->getAttribute('databaseId');
$databaseInternalId = $document->getAttribute('databaseInternalId');
$dbForProject = $getProjectDB($project);
$relationships = \array_filter(
$document->getAttribute('attributes'),
fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP
);
foreach ($relationships as $relationship) {
if (!$relationship['twoWay']) {
continue;
}
$relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']);
$dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']);
$dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId());
$dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId());
}
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
$this->deleteByGroup('attributes', [
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteByGroup('indexes', [
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteAuditLogsByResource($getProjectDB, 'database/' . $databaseId . '/collection/' . $collectionId, $project);
}
/**
* @param Database $dbForConsole
* @param callable $getProjectDB

View file

@ -291,7 +291,7 @@ class OpenAPI3 extends Format
case 'Utopia\Database\Validator\UID':
case 'Utopia\Validator\Text':
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Validator\Boolean':
$node['schema']['type'] = $validator->getType();
@ -302,7 +302,7 @@ class OpenAPI3 extends Format
$node['schema']['x-upload-id'] = true;
}
$node['schema']['type'] = $validator->getType();
$node['schema']['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['schema']['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\DatetimeValidator':
$node['schema']['type'] = $validator->getType();

View file

@ -290,7 +290,7 @@ class Swagger2 extends Format
case 'Utopia\Validator\Text':
case 'Utopia\Database\Validator\UID':
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Validator\Boolean':
$node['type'] = $validator->getType();
@ -301,7 +301,7 @@ class Swagger2 extends Format
$node['x-upload-id'] = true;
}
$node['type'] = $validator->getType();
$node['x-example'] = '[' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . ']';
$node['x-example'] = '<' . \strtoupper(Template::fromCamelCaseToSnake($node['name'])) . '>';
break;
case 'Utopia\Database\Validator\DatetimeValidator':
$node['type'] = $validator->getType();

View file

@ -511,4 +511,52 @@ class HealthCustomServerTest extends Scope
return [];
}
public function testUsageSuccess()
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(503, $response['headers']['status-code']);
}
public function testUsageDumpSuccess()
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(503, $response['headers']['status-code']);
}
}