CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-20 20:52:36 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feat-database-indexing' of https://github.com/appwrite/appwrite into feat-db-refactor-ui-fixes

Browse source
This commit is contained in:
Torsten Dittmann 2021-11-24 14:56:45 +01:00
parent 9e8cb3a3f2 a5d7af9bd2
commit db0a75a18a
4 changed files with 117 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
app/controllers/api/database.php
View file

@ -40,13 +40,14 @@ use DeviceDetector\DeviceDetector;
* @param Utopia\Database\Document $attribute
* @param Appwrite\Utopia\Response $response
* @param Utopia\Database\Database $dbForInternal
* @param Utopia\Database\Database $dbForExternal
* @param Appwrite\Event\Event $database
* @param Appwrite\Event\Event $audits
* @param Appwrite\Stats\Stats $usage
*
* @return Document Newly created attribute document
*/
function createAttribute($collectionId, $attribute, $response, $dbForInternal, $database, $audits, $usage): Document
function createAttribute($collectionId, $attribute, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage): Document
{
$attributeId = $attribute->getId();
$type = $attribute->getAttribute('type', '');
@ -108,6 +109,7 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $
}
$dbForInternal->deleteCachedDocument('collections', $collectionId);
$dbForExternal->deleteCachedCollection($collectionId);
// Pass clone of $attribute object to workers
// so we can later modify Document to fit response model
@ -728,12 +730,14 @@ App::post('/v1/database/collections/:collectionId/attributes/string')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
@ -751,7 +755,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string')
'required' => $required,
'default' => $default,
'array' => $array,
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE_STRING);
});
@ -775,12 +779,14 @@ App::post('/v1/database/collections/:collectionId/attributes/email')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
@ -793,7 +799,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email')
'default' => $default,
'array' => $array,
'format' => APP_DATABASE_ATTRIBUTE_EMAIL,
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE_EMAIL);
});
@ -818,12 +824,14 @@ App::post('/v1/database/collections/:collectionId/attributes/enum')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $elements, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $elements, $required, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
@ -848,7 +856,7 @@ App::post('/v1/database/collections/:collectionId/attributes/enum')
'array' => $array,
'format' => APP_DATABASE_ATTRIBUTE_ENUM,
'formatOptions' => ['elements' => $elements],
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE_ENUM);
});
@ -872,12 +880,14 @@ App::post('/v1/database/collections/:collectionId/attributes/ip')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
@ -890,7 +900,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip')
'default' => $default,
'array' => $array,
'format' => APP_DATABASE_ATTRIBUTE_IP,
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE_IP);
});
@ -914,10 +924,11 @@ App::post('/v1/database/collections/:collectionId/attributes/url')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
@ -932,7 +943,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url')
'default' => $default,
'array' => $array,
'format' => APP_DATABASE_ATTRIBUTE_URL,
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE_URL);
});
@ -958,12 +969,14 @@ App::post('/v1/database/collections/:collectionId/attributes/integer')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
@ -994,7 +1007,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer')
'min' => $min,
'max' => $max,
],
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$formatOptions = $attribute->getAttribute('formatOptions', []);
@ -1027,12 +1040,14 @@ App::post('/v1/database/collections/:collectionId/attributes/float')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForInternal, $dbForExternal,$database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Utopia\Database\Database $dbForExternal*/
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $audits */
/** @var Appwrite\Stats\Stats $usage */
@ -1063,7 +1078,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float')
'min' => $min,
'max' => $max,
],
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$formatOptions = $attribute->getAttribute('formatOptions', []);
@ -1094,10 +1109,11 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean')
->param('array', false, new Boolean(), 'Is attribute an array?', true)
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $database, $audits, $usage) {
->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForInternal, $dbForExternal, $database, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal*/
/** @var Appwrite\Event\Event $database */
@ -1111,7 +1127,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean')
'required' => $required,
'default' => $default,
'array' => $array,
]), $response, $dbForInternal, $database, $audits, $usage);
]), $response, $dbForInternal, $dbForExternal, $database, $audits, $usage);
$response->dynamic($attribute, Response::MODEL_ATTRIBUTE_BOOLEAN);
});
@ -1229,13 +1245,15 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId')
->param('attributeId', '', new Key(), 'Attribute ID.')
->inject('response')
->inject('dbForInternal')
->inject('dbForExternal')
->inject('database')
->inject('events')
->inject('audits')
->inject('usage')
->action(function ($collectionId, $attributeId, $response, $dbForInternal, $database, $events, $audits, $usage) {
->action(function ($collectionId, $attributeId, $response, $dbForInternal, $dbForExternal, $database, $events, $audits, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Utopia\Database\Database $dbForInternal */
/** @var Utopia\Database\Database $dbForExternal */
/** @var Appwrite\Event\Event $database */
/** @var Appwrite\Event\Event $events */
/** @var Appwrite\Event\Event $audits */
@ -1259,6 +1277,7 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId')
}
$dbForInternal->deleteCachedDocument('collections', $collectionId);
$dbForExternal->deleteCachedCollection($collection->getId());
$database
->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE)

12
app/tasks/usage.php
View file

@ -246,17 +246,19 @@ $cli
$max = 10;
$sleep = 1;
$database = $client->selectDB('telegraf');
do { // check if telegraf database is ready
$attempts++;
if (!in_array('telegraf', $client->listDatabases())) {
try {
$attempts++;
$database = $client->selectDB('telegraf');
if(in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
} else {
break; // leave the do-while if successful
}
} while ($attempts < $max);

87
app/workers/deletes.php
View file

@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\CLI\Console;
use Utopia\Audit\Audit;
require_once __DIR__.'/../init.php';
require_once __DIR__ . '/../init.php';
Console::title('Deletes V1 Worker');
Console::success(APP_NAME . ' deletes worker v1 has started' . "\n");
@ -31,12 +31,11 @@ class DeletesV1 extends Worker
{
$projectId = $this->args['projectId'] ?? '';
$type = $this->args['type'] ?? '';
switch (strval($type)) {
case DELETE_TYPE_DOCUMENT:
$document = $this->args['document'] ?? [];
$document = new Document($document);
$document = new Document($this->args['document'] ?? []);
switch ($document->getCollection()) {
case DELETE_TYPE_COLLECTIONS:
$this->deleteCollection($document, $projectId);
@ -72,8 +71,7 @@ class DeletesV1 extends Worker
break;
case DELETE_TYPE_REALTIME:
//$this->deleteRealtimeUsage($this->args['timestamp']);
//TODO: implement this
$this->deleteRealtimeUsage($this->args['timestamp']);
break;
case DELETE_TYPE_CERTIFICATES:
@ -120,12 +118,10 @@ class DeletesV1 extends Worker
* @param int $timestamp1d
* @param int $timestamp30m
*/
protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) {
$this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) {
if (!($dbForInternal = $this->getInternalDB($projectId))) {
throw new Exception('Failed to get projectDB for project '.$projectId);
}
protected function deleteUsageStats(int $timestamp1d, int $timestamp30m)
{
$this->deleteForProjectIds(function (string $projectId) use ($timestamp1d, $timestamp30m) {
$dbForInternal = $this->getInternalDB($projectId);
// Delete Usage stats
$this->deleteByGroup('stats', [
new Query('time', Query::TYPE_LESSER, [$timestamp1d]),
@ -138,7 +134,7 @@ class DeletesV1 extends Worker
], $dbForInternal);
});
}
/**
* @param Document $document teams document
* @param string $projectId
@ -159,13 +155,14 @@ class DeletesV1 extends Worker
protected function deleteProject(Document $document): void
{
$projectId = $document->getId();
// Delete all DBs
$this->getExternalDB($projectId)->delete();
$this->getInternalDB($projectId)->delete();
// Delete all storage directories
$uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId());
$cache = new Local(APP_STORAGE_CACHE.'/app-'.$document->getId());
$uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId());
$cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId());
$uploads->delete($uploads->getRoot(), true);
$cache->delete($cache->getRoot(), true);
@ -183,12 +180,12 @@ class DeletesV1 extends Worker
// Delete Memberships and decrement team membership counts
$this->deleteByGroup('memberships', [
new Query('userId', Query::TYPE_EQUAL, [$userId])
], $this->getInternalDB($projectId), function(Document $document) use ($projectId, $userId) {
], $this->getInternalDB($projectId), function (Document $document) use ($projectId) {
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
$team = $this->getInternalDB($projectId)->getDocument('teams', $teamId);
if(!$team->isEmpty()) {
if (!$team->isEmpty()) {
$team = $this->getInternalDB($projectId)->updateDocument('teams', $teamId, new Document(\array_merge($team->getArrayCopy(), [
'sum' => \max($team->getAttribute('sum', 0) - 1, 0), // Ensure that sum >= 0
])));
@ -202,11 +199,8 @@ class DeletesV1 extends Worker
*/
protected function deleteExecutionLogs(int $timestamp): void
{
$this->deleteForProjectIds(function(string $projectId) use ($timestamp) {
if (!($dbForInternal = $this->getInternalDB($projectId))) {
throw new Exception('Failed to get projectDB for project '.$projectId);
}
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$dbForInternal = $this->getInternalDB($projectId);
// Delete Executions
$this->deleteByGroup('executions', [
new Query('dateCreated', Query::TYPE_LESSER, [$timestamp])
@ -214,18 +208,33 @@ class DeletesV1 extends Worker
});
}
/**
* @param int $timestamp
*/
protected function deleteRealtimeUsage(int $timestamp): void
{
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$dbForInternal = $this->getInternalDB($projectId);
// Delete Dead Realtime Logs
$this->deleteByGroup('realtime', [
new Query('timestamp', Query::TYPE_LESSER, [$timestamp])
], $dbForInternal);
});
}
/**
* @param int $timestamp
*/
protected function deleteAbuseLogs(int $timestamp): void
{
if($timestamp == 0) {
if ($timestamp == 0) {
throw new Exception('Failed to delete audit logs. No timestamp provided');
}
$this->deleteForProjectIds(function($projectId) use ($timestamp){
$timeLimit = new TimeLimit("", 0, 1, $this->getInternalDB($projectId));
$abuse = new Abuse($timeLimit);
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$dbForInternal = $this->getInternalDB($projectId);
$timeLimit = new TimeLimit("", 0, 1, $dbForInternal);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($timestamp);
if (!$status) {
@ -239,11 +248,12 @@ class DeletesV1 extends Worker
*/
protected function deleteAuditLogs(int $timestamp): void
{
if($timestamp == 0) {
if ($timestamp == 0) {
throw new Exception('Failed to delete audit logs. No timestamp provided');
}
$this->deleteForProjectIds(function($projectId) use ($timestamp){
$audit = new Audit($this->getInternalDB($projectId));
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$dbForInternal = $this->getInternalDB($projectId);
$audit = new Audit($dbForInternal);
$status = $audit->cleanup($timestamp);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
@ -258,12 +268,12 @@ class DeletesV1 extends Worker
protected function deleteFunction(Document $document, string $projectId): void
{
$dbForInternal = $this->getInternalDB($projectId);
$device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId);
$device = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
// Delete Tags
$this->deleteByGroup('tags', [
new Query('functionId', Query::TYPE_EQUAL, [$document->getId()])
], $dbForInternal, function(Document $document) use ($device) {
], $dbForInternal, function (Document $document) use ($device) {
if ($device->delete($document->getAttribute('path', ''))) {
Console::success('Delete code tag: ' . $document->getAttribute('path', ''));
@ -290,8 +300,8 @@ class DeletesV1 extends Worker
{
Authorization::disable();
if($database->deleteDocument($document->getCollection(), $document->getId())) {
Console::success('Deleted document "'.$document->getId().'" successfully');
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
Console::success('Deleted document "' . $document->getId() . '" successfully');
if (is_callable($callback)) {
$callback($document);
@ -319,16 +329,15 @@ class DeletesV1 extends Worker
$executionStart = \microtime(true);
while($sum === $limit) {
while ($sum === $limit) {
Authorization::disable();
$projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit));
Authorization::reset();
$chunk++;
$projectIds = array_map (function ($project) {
return $project->getId();
}, $projects);
/** @var string[] $projectIds */
$projectIds = array_map(fn(Document $project) => $project->getId(), $projects);
$sum = count($projects);
@ -359,7 +368,7 @@ class DeletesV1 extends Worker
$executionStart = \microtime(true);
while($sum === $limit) {
while ($sum === $limit) {
$chunk++;
Authorization::disable();

25
tests/e2e/Services/Database/DatabaseCustomServerTest.php
View file

@ -193,6 +193,22 @@ class DatabaseCustomServerTest extends Scope
// Wait for database worker to finish creating attributes
sleep(2);
// Creating document to ensure cache is purged on schema change
$document = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'documentId' => 'unique()',
'data' => [
'firstName' => 'lorem',
'lastName' => 'ipsum',
'unneeded' => 'dolor'
],
'read' => ['role:all'],
'write' => ['role:all'],
]);
$index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/indexes', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@ -236,6 +252,15 @@ class DatabaseCustomServerTest extends Scope
sleep(2);
// Check document to ensure cache is purged on schema change
$document = $this->client->call(Client::METHOD_GET, '/database/collections/' . $actors['body']['$id'] . '/documents/' . $document['body']['$id'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]));
$this->assertNotContains($unneededId, $document['body']);
$collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $actors['body']['$id'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],