Merge pull request #6934 from appwrite/feat-delegate-deletes
Delegate custom deletes
This commit is contained in:
commit
be5ed9b282
4 changed files with 223 additions and 105 deletions
|
@ -680,9 +680,9 @@ App::delete('/v1/databases/:databaseId')
|
|||
->param('databaseId', '', new UID(), 'Database ID.')
|
||||
->inject('response')
|
||||
->inject('dbForProject')
|
||||
->inject('queueForDatabase')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForDeletes')
|
||||
->action(function (string $databaseId, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
|
||||
->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
|
||||
|
||||
$database = $dbForProject->getDocument('databases', $databaseId);
|
||||
|
||||
|
@ -697,9 +697,9 @@ App::delete('/v1/databases/:databaseId')
|
|||
$dbForProject->deleteCachedDocument('databases', $database->getId());
|
||||
$dbForProject->deleteCachedCollection('databases_' . $database->getInternalId());
|
||||
|
||||
$queueForDeletes
|
||||
->setType(DELETE_TYPE_DOCUMENT)
|
||||
->setDocument($database);
|
||||
$queueForDatabase
|
||||
->setType(DATABASE_TYPE_DELETE_DATABASE)
|
||||
->setDatabase($database);
|
||||
|
||||
$queueForEvents
|
||||
->setParam('databaseId', $database->getId())
|
||||
|
@ -1058,10 +1058,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
|
|||
->param('collectionId', '', new UID(), 'Collection ID.')
|
||||
->inject('response')
|
||||
->inject('dbForProject')
|
||||
->inject('mode')
|
||||
->inject('queueForDatabase')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForDeletes')
|
||||
->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, string $mode, Event $queueForEvents, Delete $queueForDeletes) {
|
||||
->inject('mode')
|
||||
->action(function (string $databaseId, string $collectionId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, string $mode) {
|
||||
|
||||
$database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId));
|
||||
|
||||
|
@ -1081,9 +1081,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
|
|||
|
||||
$dbForProject->deleteCachedCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
|
||||
|
||||
$queueForDeletes
|
||||
->setType(DELETE_TYPE_DOCUMENT)
|
||||
->setDocument($collection);
|
||||
$queueForDatabase
|
||||
->setType(DATABASE_TYPE_DELETE_COLLECTION)
|
||||
->setDatabase($database)
|
||||
->setCollection($collection);
|
||||
|
||||
$queueForEvents
|
||||
->setContext('database', $database)
|
||||
|
@ -2337,8 +2338,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
|
|||
->setType(DATABASE_TYPE_DELETE_ATTRIBUTE)
|
||||
->setCollection($collection)
|
||||
->setDatabase($db)
|
||||
->setDocument($attribute)
|
||||
;
|
||||
->setDocument($attribute);
|
||||
|
||||
// Select response model based on type and format
|
||||
$type = $attribute->getAttribute('type');
|
||||
|
@ -3524,10 +3524,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
|
|||
->inject('requestTimestamp')
|
||||
->inject('response')
|
||||
->inject('dbForProject')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForDeletes')
|
||||
->inject('queueForEvents')
|
||||
->inject('mode')
|
||||
->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes, string $mode) {
|
||||
->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, string $mode) {
|
||||
$database = Authorization::skip(fn() => $dbForProject->getDocument('databases', $databaseId));
|
||||
|
||||
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
|
||||
|
|
|
@ -146,6 +146,8 @@ const DATABASE_TYPE_CREATE_ATTRIBUTE = 'createAttribute';
|
|||
const DATABASE_TYPE_CREATE_INDEX = 'createIndex';
|
||||
const DATABASE_TYPE_DELETE_ATTRIBUTE = 'deleteAttribute';
|
||||
const DATABASE_TYPE_DELETE_INDEX = 'deleteIndex';
|
||||
const DATABASE_TYPE_DELETE_COLLECTION = 'deleteCollection';
|
||||
const DATABASE_TYPE_DELETE_DATABASE = 'deleteDatabase';
|
||||
// Build Worker Types
|
||||
const BUILD_TYPE_DEPLOYMENT = 'deployment';
|
||||
const BUILD_TYPE_RETRY = 'retry';
|
||||
|
|
15
composer.lock
generated
15
composer.lock
generated
|
@ -2463,22 +2463,23 @@
|
|||
},
|
||||
{
|
||||
"name": "utopia-php/logger",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/logger.git",
|
||||
"reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc"
|
||||
"reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/logger/zipball/de623f1ec1c672c795d113dd25c5bf212f7ef4fc",
|
||||
"reference": "de623f1ec1c672c795d113dd25c5bf212f7ef4fc",
|
||||
"url": "https://api.github.com/repos/utopia-php/logger/zipball/9151b7d16eab18d4c37c34643041cc0f33ca4a6c",
|
||||
"reference": "9151b7d16eab18d4c37c34643041cc0f33ca4a6c",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"laravel/pint": "1.2.*",
|
||||
"phpstan/phpstan": "1.9.x-dev",
|
||||
"phpunit/phpunit": "^9.3",
|
||||
"vimeo/psalm": "4.0.1"
|
||||
|
@ -2510,9 +2511,9 @@
|
|||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/logger/issues",
|
||||
"source": "https://github.com/utopia-php/logger/tree/0.3.1"
|
||||
"source": "https://github.com/utopia-php/logger/tree/0.3.2"
|
||||
},
|
||||
"time": "2023-02-10T15:52:50+00:00"
|
||||
"time": "2023-10-16T08:16:19+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/messaging",
|
||||
|
@ -6019,5 +6020,5 @@
|
|||
"platform-overrides": {
|
||||
"php": "8.0"
|
||||
},
|
||||
"plugin-api-version": "2.2.0"
|
||||
"plugin-api-version": "2.6.0"
|
||||
}
|
||||
|
|
|
@ -4,13 +4,18 @@ namespace Appwrite\Platform\Workers;
|
|||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Messaging\Adapter\Realtime;
|
||||
use Appwrite\Utopia\Response\Model\Platform;
|
||||
use Exception;
|
||||
use Utopia\Audit\Audit;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Authorization;
|
||||
use Utopia\Database\Exception\Conflict;
|
||||
use Utopia\Database\Exception\Restricted;
|
||||
use Utopia\Database\Exception\Structure;
|
||||
use Utopia\Database\Exception as DatabaseException;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
|
||||
|
@ -55,15 +60,13 @@ class Databases extends Action
|
|||
$document = new Document($payload['document'] ?? []);
|
||||
$database = new Document($payload['database'] ?? []);
|
||||
|
||||
if ($collection->isEmpty()) {
|
||||
throw new \Exception('Missing collection');
|
||||
}
|
||||
|
||||
if ($document->isEmpty()) {
|
||||
throw new \Exception('Missing document');
|
||||
if ($database->isEmpty()) {
|
||||
throw new Exception('Missing database');
|
||||
}
|
||||
|
||||
match (strval($type)) {
|
||||
DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $project, $dbForProject),
|
||||
DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $project, $dbForProject),
|
||||
DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
|
||||
DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForConsole, $dbForProject),
|
||||
DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForConsole, $dbForProject),
|
||||
|
@ -86,6 +89,12 @@ class Databases extends Action
|
|||
*/
|
||||
private function createAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
|
||||
{
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception('Missing collection');
|
||||
}
|
||||
if ($attribute->isEmpty()) {
|
||||
throw new Exception('Missing attribute');
|
||||
}
|
||||
|
||||
$projectId = $project->getId();
|
||||
|
||||
|
@ -172,25 +181,7 @@ class Databases extends Action
|
|||
);
|
||||
}
|
||||
} finally {
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $events[0],
|
||||
payload: $attribute,
|
||||
project: $project,
|
||||
);
|
||||
|
||||
Realtime::send(
|
||||
projectId: 'console',
|
||||
payload: $attribute->getArrayCopy(),
|
||||
events: $events,
|
||||
channels: $target['channels'],
|
||||
roles: $target['roles'],
|
||||
options: [
|
||||
'projectId' => $projectId,
|
||||
'databaseId' => $database->getId(),
|
||||
'collectionId' => $collection->getId()
|
||||
]
|
||||
);
|
||||
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
|
||||
}
|
||||
|
||||
if ($type === Database::VAR_RELATIONSHIP && $options['twoWay']) {
|
||||
|
@ -214,6 +205,13 @@ class Databases extends Action
|
|||
**/
|
||||
private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForConsole, Database $dbForProject): void
|
||||
{
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception('Missing collection');
|
||||
}
|
||||
if ($attribute->isEmpty()) {
|
||||
throw new Exception('Missing attribute');
|
||||
}
|
||||
|
||||
$projectId = $project->getId();
|
||||
|
||||
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [
|
||||
|
@ -283,25 +281,7 @@ class Databases extends Action
|
|||
);
|
||||
}
|
||||
} finally {
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $events[0],
|
||||
payload: $attribute,
|
||||
project: $project
|
||||
);
|
||||
|
||||
Realtime::send(
|
||||
projectId: 'console',
|
||||
payload: $attribute->getArrayCopy(),
|
||||
events: $events,
|
||||
channels: $target['channels'],
|
||||
roles: $target['roles'],
|
||||
options: [
|
||||
'projectId' => $projectId,
|
||||
'databaseId' => $database->getId(),
|
||||
'collectionId' => $collection->getId()
|
||||
]
|
||||
);
|
||||
$this->trigger($database, $collection, $attribute, $project, $projectId, $events);
|
||||
}
|
||||
|
||||
// The underlying database removes/rebuilds indexes when attribute is removed
|
||||
|
@ -379,6 +359,13 @@ class Databases extends Action
|
|||
*/
|
||||
private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
|
||||
{
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception('Missing collection');
|
||||
}
|
||||
if ($index->isEmpty()) {
|
||||
throw new Exception('Missing index');
|
||||
}
|
||||
|
||||
$projectId = $project->getId();
|
||||
|
||||
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [
|
||||
|
@ -411,25 +398,7 @@ class Databases extends Action
|
|||
$index->setAttribute('status', 'failed')
|
||||
);
|
||||
} finally {
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $events[0],
|
||||
payload: $index,
|
||||
project: $project
|
||||
);
|
||||
|
||||
Realtime::send(
|
||||
projectId: 'console',
|
||||
payload: $index->getArrayCopy(),
|
||||
events: $events,
|
||||
channels: $target['channels'],
|
||||
roles: $target['roles'],
|
||||
options: [
|
||||
'projectId' => $projectId,
|
||||
'databaseId' => $database->getId(),
|
||||
'collectionId' => $collection->getId()
|
||||
]
|
||||
);
|
||||
$this->trigger($database, $collection, $index, $project, $projectId, $events);
|
||||
}
|
||||
|
||||
$dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collectionId);
|
||||
|
@ -450,6 +419,13 @@ class Databases extends Action
|
|||
*/
|
||||
private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForConsole, Database $dbForProject): void
|
||||
{
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception('Missing collection');
|
||||
}
|
||||
if ($index->isEmpty()) {
|
||||
throw new Exception('Missing index');
|
||||
}
|
||||
|
||||
$projectId = $project->getId();
|
||||
|
||||
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [
|
||||
|
@ -470,7 +446,6 @@ class Databases extends Action
|
|||
} catch (\Exception $e) {
|
||||
Console::error($e->getMessage());
|
||||
|
||||
|
||||
if ($e instanceof DatabaseException) {
|
||||
$index->setAttribute('error', $e->getMessage());
|
||||
}
|
||||
|
@ -480,16 +455,159 @@ class Databases extends Action
|
|||
$index->setAttribute('status', 'stuck')
|
||||
);
|
||||
} finally {
|
||||
$this->trigger($database, $collection, $index, $project, $projectId, $events);
|
||||
}
|
||||
|
||||
$dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Document $database
|
||||
* @param Document $project
|
||||
* @param $dbForProject
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function deleteDatabase(Document $database, Document $project, $dbForProject): void
|
||||
{
|
||||
$this->deleteByGroup('database_' . $database->getInternalId(), [], $dbForProject, function ($collection) use ($database, $project, $dbForProject) {
|
||||
$this->deleteCollection($database, $collection, $project, $dbForProject);
|
||||
});
|
||||
|
||||
$dbForProject->deleteCollection('database_' . $database->getInternalId());
|
||||
|
||||
$this->deleteAuditLogsByResource('database/' . $database->getId(), $project, $dbForProject);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Document $database
|
||||
* @param Document $collection
|
||||
* @param Document $project
|
||||
* @param Database $dbForProject
|
||||
* @return void
|
||||
* @throws Authorization
|
||||
* @throws Conflict
|
||||
* @throws DatabaseException
|
||||
* @throws Restricted
|
||||
* @throws Structure
|
||||
*/
|
||||
protected function deleteCollection(Document $database, Document $collection, Document $project, Database $dbForProject): void
|
||||
{
|
||||
if ($collection->isEmpty()) {
|
||||
throw new Exception('Missing collection');
|
||||
}
|
||||
|
||||
$collectionId = $collection->getId();
|
||||
$collectionInternalId = $collection->getInternalId();
|
||||
$databaseId = $database->getId();
|
||||
$databaseInternalId = $database->getInternalId();
|
||||
|
||||
$relationships = \array_filter(
|
||||
$collection->getAttribute('attributes'),
|
||||
fn ($attribute) => $attribute['type'] === Database::VAR_RELATIONSHIP
|
||||
);
|
||||
|
||||
foreach ($relationships as $relationship) {
|
||||
if (!$relationship['twoWay']) {
|
||||
continue;
|
||||
}
|
||||
$relatedCollection = $dbForProject->getDocument('database_' . $databaseInternalId, $relationship['relatedCollection']);
|
||||
$dbForProject->deleteDocument('attributes', $databaseInternalId . '_' . $relatedCollection->getInternalId() . '_' . $relationship['twoWayKey']);
|
||||
$dbForProject->deleteCachedDocument('database_' . $databaseInternalId, $relatedCollection->getId());
|
||||
$dbForProject->deleteCachedCollection('database_' . $databaseInternalId . '_collection_' . $relatedCollection->getInternalId());
|
||||
}
|
||||
|
||||
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
|
||||
|
||||
$this->deleteByGroup('attributes', [
|
||||
Query::equal('databaseInternalId', [$databaseInternalId]),
|
||||
Query::equal('collectionInternalId', [$collectionInternalId])
|
||||
], $dbForProject);
|
||||
|
||||
$this->deleteByGroup('indexes', [
|
||||
Query::equal('databaseInternalId', [$databaseInternalId]),
|
||||
Query::equal('collectionInternalId', [$collectionInternalId])
|
||||
], $dbForProject);
|
||||
|
||||
$this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project, $dbForProject);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $resource
|
||||
* @param Document $project
|
||||
* @param Database $dbForProject
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function deleteAuditLogsByResource(string $resource, Document $project, Database $dbForProject): void
|
||||
{
|
||||
$this->deleteByGroup(Audit::COLLECTION, [
|
||||
Query::equal('resource', [$resource])
|
||||
], $dbForProject);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $collection collectionID
|
||||
* @param array $queries
|
||||
* @param Database $database
|
||||
* @param callable|null $callback
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
||||
{
|
||||
$count = 0;
|
||||
$chunk = 0;
|
||||
$limit = 50;
|
||||
$sum = $limit;
|
||||
|
||||
$executionStart = \microtime(true);
|
||||
|
||||
while ($sum === $limit) {
|
||||
$chunk++;
|
||||
|
||||
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
|
||||
|
||||
$sum = count($results);
|
||||
|
||||
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
|
||||
|
||||
foreach ($results as $document) {
|
||||
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
|
||||
Console::success('Deleted document "' . $document->getId() . '" successfully');
|
||||
|
||||
if (\is_callable($callback)) {
|
||||
$callback($document);
|
||||
}
|
||||
} else {
|
||||
Console::error('Failed to delete document: ' . $document->getId());
|
||||
}
|
||||
$count++;
|
||||
}
|
||||
}
|
||||
|
||||
$executionEnd = \microtime(true);
|
||||
|
||||
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||
}
|
||||
|
||||
protected function trigger(
|
||||
Document $database,
|
||||
Document $collection,
|
||||
Document $attribute,
|
||||
Document $project,
|
||||
string $projectId,
|
||||
array $events
|
||||
): void {
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $events[0],
|
||||
payload: $index,
|
||||
project: $project
|
||||
payload: $attribute,
|
||||
project: $project,
|
||||
);
|
||||
|
||||
Realtime::send(
|
||||
projectId: 'console',
|
||||
payload: $index->getArrayCopy(),
|
||||
payload: $attribute->getArrayCopy(),
|
||||
events: $events,
|
||||
channels: $target['channels'],
|
||||
roles: $target['roles'],
|
||||
|
@ -500,7 +618,4 @@ class Databases extends Action
|
|||
]
|
||||
);
|
||||
}
|
||||
|
||||
$dbForProject->deleteCachedDocument('database_' . $database->getInternalId(), $collection->getId());
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue