Merge pull request #751 from christyjacob4/delete-collection-documents
feat: delete documents using delete worker
This commit is contained in:
commit
8f65ff1086
4 changed files with 272 additions and 33 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -4,4 +4,5 @@
|
||||||
/tests/resources/storage/
|
/tests/resources/storage/
|
||||||
/.idea/
|
/.idea/
|
||||||
.DS_Store
|
.DS_Store
|
||||||
.php_cs.cache
|
.php_cs.cache
|
||||||
|
debug/
|
|
@ -235,7 +235,7 @@ App::delete('/v1/database/collections/:collectionId')
|
||||||
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
|
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
|
||||||
->label('sdk.response.model', Response::MODEL_NONE)
|
->label('sdk.response.model', Response::MODEL_NONE)
|
||||||
->param('collectionId', '', new UID(), 'Collection unique ID.')
|
->param('collectionId', '', new UID(), 'Collection unique ID.')
|
||||||
->action(function ($collectionId, $response, $projectDB, $webhooks, $audits) {
|
->action(function ($collectionId, $response, $projectDB, $webhooks, $audits, $deletes) {
|
||||||
/** @var Appwrite\Utopia\Response $response */
|
/** @var Appwrite\Utopia\Response $response */
|
||||||
/** @var Appwrite\Database\Database $projectDB */
|
/** @var Appwrite\Database\Database $projectDB */
|
||||||
/** @var Appwrite\Event\Event $webhooks */
|
/** @var Appwrite\Event\Event $webhooks */
|
||||||
|
@ -250,7 +250,11 @@ App::delete('/v1/database/collections/:collectionId')
|
||||||
if (!$projectDB->deleteDocument($collectionId)) {
|
if (!$projectDB->deleteDocument($collectionId)) {
|
||||||
throw new Exception('Failed to remove collection from DB', 500);
|
throw new Exception('Failed to remove collection from DB', 500);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$deletes
|
||||||
|
->setParam('document', $collection)
|
||||||
|
;
|
||||||
|
|
||||||
$webhooks
|
$webhooks
|
||||||
->setParam('payload', $response->output($collection, Response::MODEL_COLLECTION))
|
->setParam('payload', $response->output($collection, Response::MODEL_COLLECTION))
|
||||||
;
|
;
|
||||||
|
@ -262,7 +266,7 @@ App::delete('/v1/database/collections/:collectionId')
|
||||||
;
|
;
|
||||||
|
|
||||||
$response->noContent();
|
$response->noContent();
|
||||||
}, ['response', 'projectDB', 'webhooks', 'audits']);
|
}, ['response', 'projectDB', 'webhooks', 'audits', 'deletes']);
|
||||||
|
|
||||||
App::post('/v1/database/collections/:collectionId/documents')
|
App::post('/v1/database/collections/:collectionId/documents')
|
||||||
->desc('Create Document')
|
->desc('Create Document')
|
||||||
|
|
|
@ -10,31 +10,43 @@ use Appwrite\Database\Database;
|
||||||
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
|
use Appwrite\Database\Adapter\MySQL as MySQLAdapter;
|
||||||
use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
use Appwrite\Database\Adapter\Redis as RedisAdapter;
|
||||||
use Appwrite\Database\Document;
|
use Appwrite\Database\Document;
|
||||||
|
use Appwrite\Database\Validator\Authorization;
|
||||||
use Appwrite\Storage\Device\Local;
|
use Appwrite\Storage\Device\Local;
|
||||||
|
use Utopia\CLI\Console;
|
||||||
use Utopia\Config\Config;
|
use Utopia\Config\Config;
|
||||||
|
|
||||||
class DeletesV1
|
class DeletesV1
|
||||||
{
|
{
|
||||||
public $args = [];
|
public $args = [];
|
||||||
|
|
||||||
|
protected $consoleDB = null;
|
||||||
|
|
||||||
public function setUp(): void
|
public function setUp(): void
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public function perform()
|
public function perform()
|
||||||
{
|
{
|
||||||
|
$projectId = $this->args['projectId'];
|
||||||
$document = $this->args['document'];
|
$document = $this->args['document'];
|
||||||
|
|
||||||
$document = new Document($document);
|
$document = new Document($document);
|
||||||
|
|
||||||
switch ($document->getCollection()) {
|
switch (strval($document->getCollection())) {
|
||||||
case Database::SYSTEM_COLLECTION_PROJECTS:
|
case Database::SYSTEM_COLLECTION_PROJECTS:
|
||||||
$this->deleteProject($document);
|
$this->deleteProject($document);
|
||||||
break;
|
break;
|
||||||
case Database::SYSTEM_COLLECTION_USERS:
|
case Database::SYSTEM_COLLECTION_FUNCTIONS:
|
||||||
$this->deleteUser($document);
|
$this->deleteFunction($document, $projectId);
|
||||||
|
break;
|
||||||
|
case Database::SYSTEM_COLLECTION_USERS:
|
||||||
|
$this->deleteUser($document, $projectId);
|
||||||
|
break;
|
||||||
|
case Database::SYSTEM_COLLECTION_COLLECTIONS:
|
||||||
|
$this->deleteDocuments($document, $projectId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
Console::error('No lazy delete operation available for document of type: '.$document->getCollection());
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -43,50 +55,163 @@ class DeletesV1
|
||||||
{
|
{
|
||||||
// ... Remove environment for this job
|
// ... Remove environment for this job
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected function deleteDocuments(Document $document, $projectId)
|
||||||
|
{
|
||||||
|
$collectionId = $document->getId();
|
||||||
|
|
||||||
|
// Delete Documents in the deleted collection
|
||||||
|
$this->deleteByGroup([
|
||||||
|
'$collection='.$collectionId
|
||||||
|
], $this->getProjectDB($projectId));
|
||||||
|
}
|
||||||
|
|
||||||
protected function deleteProject(Document $document)
|
protected function deleteProject(Document $document)
|
||||||
{
|
{
|
||||||
global $register;
|
|
||||||
|
|
||||||
$consoleDB = new Database();
|
|
||||||
$consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
|
|
||||||
$consoleDB->setNamespace('app_console'); // Main DB
|
|
||||||
$consoleDB->setMocks(Config::getParam('collections', []));
|
|
||||||
|
|
||||||
// Delete all DBs
|
// Delete all DBs
|
||||||
$consoleDB->deleteNamespace($document->getId());
|
$this->getConsoleDB()->deleteNamespace($document->getId());
|
||||||
$uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId());
|
$uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId());
|
||||||
$cache = new Local(APP_STORAGE_CACHE.'/app-'.$document->getId());
|
$cache = new Local(APP_STORAGE_CACHE.'/app-'.$document->getId());
|
||||||
|
|
||||||
|
// Delete all storage directories
|
||||||
$uploads->delete($uploads->getRoot(), true);
|
$uploads->delete($uploads->getRoot(), true);
|
||||||
$cache->delete($cache->getRoot(), true);
|
$cache->delete($cache->getRoot(), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function deleteUser(Document $user)
|
protected function deleteUser(Document $document, $projectId)
|
||||||
{
|
{
|
||||||
global $projectDB;
|
$tokens = $document->getAttribute('tokens', []);
|
||||||
|
|
||||||
$tokens = $user->getAttribute('tokens', []);
|
|
||||||
|
|
||||||
foreach ($tokens as $token) {
|
foreach ($tokens as $token) {
|
||||||
if (!$projectDB->deleteDocument($token->getId())) {
|
if (!$this->getProjectDB($projectId)->deleteDocument($token->getId())) {
|
||||||
throw new Exception('Failed to remove token from DB', 500);
|
throw new Exception('Failed to remove token from DB', 500);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$memberships = $projectDB->getCollection([
|
// Delete Memberships
|
||||||
'limit' => 2000, // TODO add members limit
|
$this->deleteByGroup([
|
||||||
'offset' => 0,
|
'$collection='.Database::SYSTEM_COLLECTION_MEMBERSHIPS,
|
||||||
'filters' => [
|
'userId='.$document->getId(),
|
||||||
'$collection='.Database::SYSTEM_COLLECTION_MEMBERSHIPS,
|
], $this->getProjectDB($projectId));
|
||||||
'userId='.$user->getId(),
|
}
|
||||||
],
|
|
||||||
]);
|
|
||||||
|
|
||||||
foreach ($memberships as $membership) {
|
protected function deleteFunction(Document $document, $projectId)
|
||||||
if (!$projectDB->deleteDocument($membership->getId())) {
|
{
|
||||||
throw new Exception('Failed to remove team membership from DB', 500);
|
$projectDB = $this->getProjectDB($projectId);
|
||||||
|
$device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId);
|
||||||
|
|
||||||
|
// Delete Tags
|
||||||
|
$this->deleteByGroup([
|
||||||
|
'$collection='.Database::SYSTEM_COLLECTION_TAGS,
|
||||||
|
'functionId='.$document->getId(),
|
||||||
|
], $projectDB, function(Document $document) use ($device) {
|
||||||
|
|
||||||
|
if ($device->delete($document->getAttribute('path', ''))) {
|
||||||
|
Console::success('Delete code tag: '.$document->getAttribute('path', ''));
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Console::error('Dailed to delete code tag: '.$document->getAttribute('path', ''));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Delete Executions
|
||||||
|
$this->deleteByGroup([
|
||||||
|
'$collection='.Database::SYSTEM_COLLECTION_EXECUTIONS,
|
||||||
|
'functionId='.$document->getId(),
|
||||||
|
], $projectDB);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function deleteById(Document $document, Database $database, callable $callback = null): bool
|
||||||
|
{
|
||||||
|
Authorization::disable();
|
||||||
|
|
||||||
|
if($database->deleteDocument($document->getId())) {
|
||||||
|
Console::success('Deleted document "'.$document->getId().'" successfully');
|
||||||
|
|
||||||
|
if(is_callable($callback)) {
|
||||||
|
$callback($document);
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Console::error('Failed to delete document: '.$document->getId());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
Authorization::reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function deleteByGroup(array $filters, Database $database, callable $callback = null)
|
||||||
|
{
|
||||||
|
$count = 0;
|
||||||
|
$chunk = 0;
|
||||||
|
$limit = 50;
|
||||||
|
$results = [];
|
||||||
|
$sum = $limit;
|
||||||
|
|
||||||
|
$executionStart = \microtime(true);
|
||||||
|
|
||||||
|
while($sum === $limit) {
|
||||||
|
$chunk++;
|
||||||
|
|
||||||
|
Authorization::disable();
|
||||||
|
|
||||||
|
$results = $database->getCollection([
|
||||||
|
'limit' => $limit,
|
||||||
|
'offset' => 0,
|
||||||
|
'orderField' => '$id',
|
||||||
|
'orderType' => 'ASC',
|
||||||
|
'orderCast' => 'string',
|
||||||
|
'filters' => $filters,
|
||||||
|
]);
|
||||||
|
|
||||||
|
Authorization::reset();
|
||||||
|
|
||||||
|
$sum = count($results);
|
||||||
|
|
||||||
|
Console::info('Deleting chunk #'.$chunk.'. Found '.$sum.' documents');
|
||||||
|
|
||||||
|
foreach ($results as $document) {
|
||||||
|
$this->deleteById($document, $database, $callback);
|
||||||
|
$count++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$executionEnd = \microtime(true);
|
||||||
|
|
||||||
|
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
/**
|
||||||
|
* @return Database;
|
||||||
|
*/
|
||||||
|
protected function getConsoleDB(): Database
|
||||||
|
{
|
||||||
|
global $register;
|
||||||
|
|
||||||
|
if($this->consoleDB === null) {
|
||||||
|
$this->consoleDB = new Database();
|
||||||
|
$this->consoleDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
|
||||||
|
$this->consoleDB->setNamespace('app_console'); // Main DB
|
||||||
|
$this->consoleDB->setMocks(Config::getParam('collections', []));
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->consoleDB;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return Database;
|
||||||
|
*/
|
||||||
|
protected function getProjectDB($projectId): Database
|
||||||
|
{
|
||||||
|
global $register;
|
||||||
|
|
||||||
|
$projectDB = new Database();
|
||||||
|
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($register), $register));
|
||||||
|
$projectDB->setNamespace('app_'.$projectId); // Main DB
|
||||||
|
$projectDB->setMocks(Config::getParam('collections', []));
|
||||||
|
|
||||||
|
return $projectDB;
|
||||||
|
}
|
||||||
|
}
|
|
@ -5,10 +5,119 @@ namespace Tests\E2E\Services\Database;
|
||||||
use Tests\E2E\Scopes\ProjectCustom;
|
use Tests\E2E\Scopes\ProjectCustom;
|
||||||
use Tests\E2E\Scopes\Scope;
|
use Tests\E2E\Scopes\Scope;
|
||||||
use Tests\E2E\Scopes\SideServer;
|
use Tests\E2E\Scopes\SideServer;
|
||||||
|
use Tests\E2E\Client;
|
||||||
|
|
||||||
class DatabaseCustomServerTest extends Scope
|
class DatabaseCustomServerTest extends Scope
|
||||||
{
|
{
|
||||||
use DatabaseBase;
|
use DatabaseBase;
|
||||||
use ProjectCustom;
|
use ProjectCustom;
|
||||||
use SideServer;
|
use SideServer;
|
||||||
|
|
||||||
|
public function testDeleteCollection()
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Test for SUCCESS
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Create collection
|
||||||
|
$actors = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([
|
||||||
|
'content-type' => 'application/json',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id'],
|
||||||
|
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||||
|
]), [
|
||||||
|
'name' => 'Actors',
|
||||||
|
'read' => ['*'],
|
||||||
|
'write' => ['role:1', 'role:2'],
|
||||||
|
'rules' => [
|
||||||
|
[
|
||||||
|
'label' => 'First Name',
|
||||||
|
'key' => 'firstName',
|
||||||
|
'type' => 'text',
|
||||||
|
'default' => '',
|
||||||
|
'required' => true,
|
||||||
|
'array' => false
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'label' => 'Last Name',
|
||||||
|
'key' => 'lastName',
|
||||||
|
'type' => 'text',
|
||||||
|
'default' => '',
|
||||||
|
'required' => true,
|
||||||
|
'array' => false
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
|
||||||
|
$this->assertEquals($actors['headers']['status-code'], 201);
|
||||||
|
$this->assertEquals($actors['body']['$collection'], 0);
|
||||||
|
$this->assertEquals($actors['body']['name'], 'Actors');
|
||||||
|
$this->assertIsArray($actors['body']['$permissions']);
|
||||||
|
$this->assertIsArray($actors['body']['$permissions']['read']);
|
||||||
|
$this->assertIsArray($actors['body']['$permissions']['write']);
|
||||||
|
$this->assertCount(1, $actors['body']['$permissions']['read']);
|
||||||
|
$this->assertCount(2, $actors['body']['$permissions']['write']);
|
||||||
|
|
||||||
|
// Add Documents to the collection
|
||||||
|
$document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([
|
||||||
|
'content-type' => 'application/json',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id'],
|
||||||
|
], $this->getHeaders()), [
|
||||||
|
'data' => [
|
||||||
|
'firstName' => 'Tom',
|
||||||
|
'lastName' => 'Holland',
|
||||||
|
],
|
||||||
|
'read' => ['user:'.$this->getUser()['$id']],
|
||||||
|
'write' => ['user:'.$this->getUser()['$id']],
|
||||||
|
]);
|
||||||
|
|
||||||
|
$document2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $actors['body']['$id'] . '/documents', array_merge([
|
||||||
|
'content-type' => 'application/json',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id'],
|
||||||
|
], $this->getHeaders()), [
|
||||||
|
'data' => [
|
||||||
|
'firstName' => 'Samuel',
|
||||||
|
'lastName' => 'Jackson',
|
||||||
|
],
|
||||||
|
'read' => ['user:'.$this->getUser()['$id']],
|
||||||
|
'write' => ['user:'.$this->getUser()['$id']],
|
||||||
|
]);
|
||||||
|
|
||||||
|
$this->assertEquals($document1['headers']['status-code'], 201);
|
||||||
|
$this->assertEquals($document1['body']['$collection'], $actors['body']['$id']);
|
||||||
|
$this->assertIsArray($document1['body']['$permissions']);
|
||||||
|
$this->assertIsArray($document1['body']['$permissions']['read']);
|
||||||
|
$this->assertIsArray($document1['body']['$permissions']['write']);
|
||||||
|
$this->assertCount(1, $document1['body']['$permissions']['read']);
|
||||||
|
$this->assertCount(1, $document1['body']['$permissions']['write']);
|
||||||
|
$this->assertEquals($document1['body']['firstName'], 'Tom');
|
||||||
|
$this->assertEquals($document1['body']['lastName'], 'Holland');
|
||||||
|
|
||||||
|
$this->assertEquals($document2['headers']['status-code'], 201);
|
||||||
|
$this->assertEquals($document2['body']['$collection'], $actors['body']['$id']);
|
||||||
|
$this->assertIsArray($document2['body']['$permissions']);
|
||||||
|
$this->assertIsArray($document2['body']['$permissions']['read']);
|
||||||
|
$this->assertIsArray($document2['body']['$permissions']['write']);
|
||||||
|
$this->assertCount(1, $document2['body']['$permissions']['read']);
|
||||||
|
$this->assertCount(1, $document2['body']['$permissions']['write']);
|
||||||
|
$this->assertEquals($document2['body']['firstName'], 'Samuel');
|
||||||
|
$this->assertEquals($document2['body']['lastName'], 'Jackson');
|
||||||
|
|
||||||
|
// Delete the actors collection
|
||||||
|
$response = $this->client->call(Client::METHOD_DELETE, '/database/collections/'.$actors['body']['$id'], array_merge([
|
||||||
|
'content-type' => 'application/json',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id'],
|
||||||
|
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||||
|
], $this->getHeaders()));
|
||||||
|
|
||||||
|
$this->assertEquals($response['headers']['status-code'], 204);
|
||||||
|
$this->assertEquals($response['body'],"");
|
||||||
|
|
||||||
|
// Try to get the collection and check if it has been deleted
|
||||||
|
$response = $this->client->call(Client::METHOD_GET, '/database/collections/'.$actors['body']['$id'], array_merge([
|
||||||
|
'content-type' => 'application/json',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id']
|
||||||
|
], $this->getHeaders()));
|
||||||
|
|
||||||
|
$this->assertEquals($response['headers']['status-code'], 404);
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in a new issue