From 84e9881a6a4638f9253ca409d685442178ce3a39 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 12:45:03 -0400 Subject: [PATCH 01/21] Delete from attribute/index tables on deleteCollection --- app/controllers/api/database.php | 5 +++++ app/workers/deletes.php | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7df3172484..191ed38e5f 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -426,6 +426,11 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Failed to remove collection from DB', 500); } + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) + ; + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1894aaf76b..759409d286 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -44,6 +44,9 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { // TODO@kodumbeats define these as constants somewhere + case 'collections': + $this->deleteCollection($document, $projectId); + break; case 'projects': $this->deleteProject($document); break; @@ -88,6 +91,25 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param Document $document teams document + * @param string $projectId + */ + protected function deleteCollection(Document $document, string $projectId): void + { + $collectionId = $document->getId(); + + $dbForInternal = $this->getInternalDB($projectId); + + $this->deleteByGroup('attributes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $this->deleteByGroup('indexes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + } /** * @param Document $document teams document From 8c3ca6808c8a1d9c48b095b66f669d53fe91144d Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 12:46:05 -0400 Subject: [PATCH 02/21] Test deleteCollection removal of attributes/indexes --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index e5a75fa685..79750318b2 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -403,5 +403,13 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(400, $tooMany['headers']['status-code']); $this->assertEquals('Index limit exceeded', $tooMany['body']['message']); + + $collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $collection['headers']['status-code']); } } \ No newline at end of file From 5802212dd8d56d61ea2cf518fdc64d32ed5b9813 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 10 Sep 2021 16:08:49 +0545 Subject: [PATCH 03/21] fix usage daemon first time startup --- app/tasks/usage.php | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index e66bec113f..4ff5e8cad9 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -238,10 +238,27 @@ $cli /** * Aggregate InfluxDB every 30 seconds + * @var InfluxDB\Client $client */ $client = $register->get('influxdb'); if ($client) { + $attempts = 0; + $max = 10; + $sleep = 1; + $database = $client->selectDB('telegraf'); + do { // check if telegraf database is ready + $attempts++; + if(!in_array('telegraf', $client->listDatabases())) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } else { + break; // leave the do-while if successful + } + } while ($attempts < $max); // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics @@ -318,7 +335,23 @@ $cli $latestProject = null; do { // Loop over all the projects - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // list projects + try { + $attempts++; + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed access console db: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); if (empty($projects)) { continue; From ccae13ac19ca1162044a80774de7d332800eb445 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 15 Sep 2021 10:39:49 +0545 Subject: [PATCH 04/21] fix bucket delete --- app/controllers/api/storage.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 879df731c9..6a9e0e01c0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -442,15 +442,15 @@ App::delete('/v1/storage/buckets/:bucketId') throw new Exception('Bucket not found', 404); } + if(!$dbForInternal->deleteDocument('buckets', $bucketId)) { + throw new Exception('Failed to remove project from DB', 500); + } + $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) ->setParam('document', $bucket) ; - if(!$dbForInternal->deleteDocument('buckets', $bucketId)) { - throw new Exception('Failed to remove project from DB', 500); - } - $events ->setParam('eventData', $response->output($bucket, Response::MODEL_BUCKET)) ; From 49cfddae1e435504869dad34b2d6391a8b70ad22 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 15 Sep 2021 10:40:30 +0545 Subject: [PATCH 05/21] Update app/controllers/api/storage.php Co-authored-by: Torsten Dittmann --- app/controllers/api/storage.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 6a9e0e01c0..16a2905833 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -380,13 +380,13 @@ App::put('/v1/storage/buckets/:bucketId') $read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions $write ??= $bucket->getAttribute('$write',[]); // By default inherit write permissions - $read??=$bucket->getAttribute('$read', []); // By default inherit read permissions - $write??=$bucket->getAttribute('$write', []); // By default inherit write permissions - $maximumFileSize??=$bucket->getAttribute('maximumFileSize', (int)App::getEnv('_APP_STORAGE_LIMIT', 0)); - $allowedFileExtensions??=$bucket->getAttribute('allowedFileExtensions', []); - $enabled??=$bucket->getAttribute('enabled', true); - $encryption??=$bucket->getAttribute('encryption', true); - $antiVirus??=$bucket->getAttribute('antiVirus', true); + $read ??= $bucket->getAttribute('$read', []); // By default inherit read permissions + $write ??= $bucket->getAttribute('$write', []); // By default inherit write permissions + $maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int)App::getEnv('_APP_STORAGE_LIMIT', 0)); + $allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []); + $enabled ??= $bucket->getAttribute('enabled', true); + $encryption ??= $bucket->getAttribute('encryption', true); + $antiVirus ??= $bucket->getAttribute('antiVirus', true); $bucket = $dbForInternal->updateDocument('buckets', $bucket->getId(), $bucket ->setAttribute('name',$name) From 8cde715d04027c4f7b9559c15b65ff19f4d7d7e1 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:28:40 -0400 Subject: [PATCH 06/21] Use correct constant when deleting usage stats --- app/tasks/maintenance.php | 2 +- app/workers/deletes.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index fdfd7c942b..e0ef1adc01 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -42,7 +42,7 @@ $cli function notifyDeleteUsageStats(int $interval30m, int $interval1d) { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_USAGE_STATS, + 'type' => DELETE_TYPE_USAGE, 'timestamp1d' => time() - $interval1d, 'timestamp30m' => time() - $interval30m, ]); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f9571949f2..7016244f92 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -79,7 +79,7 @@ class DeletesV1 extends Worker $this->deleteCertificates($document); break; - case DELETE_TYPE_USAGE_STATS: + case DELETE_TYPE_USAGE: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); break; default: From e90036f67451b3a52bdedc473b240f80c154deb7 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:46:08 -0400 Subject: [PATCH 07/21] Exit tests if a container has status exited --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index a90890423c..325865fcd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,12 @@ install: script: - docker ps -a +# Tests should fail if any container is in exited status +- ALL_UP=`docker ps -aq --filter "status=exited"` +- > + if [[ "$ALL_UP" != "" ]]; then + exit 1 + fi - docker-compose logs appwrite - docker-compose logs mariadb - docker-compose logs appwrite-worker-functions From bfdf6de15f101ec1cfd62373e646cb0862ab3eb9 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 14:02:18 -0400 Subject: [PATCH 08/21] Delete collection document from internal collections table --- app/workers/deletes.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 78ceeb45a6..f557a3a549 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -112,6 +112,8 @@ class DeletesV1 extends Worker $this->deleteByGroup('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) ], $dbForInternal); + + $dbForInternal->deleteDocument('collections', $collectionId); } /** From 43d5d67f4321bd5e33af515228e152ca1e058302 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 10:46:59 +0530 Subject: [PATCH 09/21] feat(tests): check for failing test --- phpunit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index f0a2813f5a..56885de032 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,7 +19,7 @@ ./tests/e2e/Client.php ./tests/e2e/General ./tests/e2e/Scopes - ./tests/e2e/Services/Account + ./tests/e2e/Services/Functions/FunctionsBase.php ./tests/e2e/Services/Functions/FunctionsCustomServerTest.php ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php From f8067b9cfbb72ba2d5368a2daa3c5e9bdee38f79 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 11:08:21 +0530 Subject: [PATCH 10/21] feat(tests): check for failing test --- phpunit.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 56885de032..57e374c80c 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,14 +13,14 @@ - ./tests/unit/ + - ./tests/e2e/Client.php + + ./tests/e2e/Services/Account + - ./tests/e2e/Services/Functions/FunctionsBase.php + \ No newline at end of file From 3f4a0756ad60c104f6e459285ca6f26dfc3604b7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 12:36:24 +0530 Subject: [PATCH 11/21] feat(tests): check for failing test --- phpunit.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 57e374c80c..572dceaeea 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,10 +19,10 @@ - ./tests/e2e/Services/Account - + ./tests/e2e/Services/Database - ./tests/e2e/Services/Health + - ./tests/e2e/Services/Database - + ./tests/e2e/Services/Health ./tests/e2e/Services/Locale - ./tests/e2e/Services/Projects - ./tests/e2e/Services/Storage - ./tests/e2e/Services/Teams - ./tests/e2e/Services/Users - ./tests/e2e/Services/Workers - ./tests/e2e/Services/Webhooks --> + + + + + + From b580810c7ddef61cd0573cb0b259257a39e6c041 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 13:29:20 +0530 Subject: [PATCH 13/21] feat(tests): check for failing test --- .travis.yml | 3 +++ phpunit.xml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a90890423c..f5e7ee4681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: x-large + language: shell notifications: diff --git a/phpunit.xml b/phpunit.xml index 44cc631d0e..bd38ab510a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,11 +19,11 @@ - + ./tests/e2e/Services/Account - ./tests/e2e/Services/Health - ./tests/e2e/Services/Locale + From cad09af1bdb37c8d31a072ae641b83702f306884 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 13:49:18 +0530 Subject: [PATCH 14/21] feat(tests): check for failing test --- .travis.yml | 2 +- phpunit.xml | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5e7ee4681..2fa9ff9357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ arch: os: linux vm: - size: x-large + size: large language: shell diff --git a/phpunit.xml b/phpunit.xml index bd38ab510a..f0a2813f5a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,26 +13,26 @@ - + ./tests/unit/ - + ./tests/e2e/Scopes ./tests/e2e/Services/Account - - - - - - - - - - + ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php \ No newline at end of file From 086f6312116cb1eea975e36286738b1c4f9ae955 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 17:04:23 +0530 Subject: [PATCH 15/21] feat(tests): check for failing test --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2fa9ff9357..fe7bc3098e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,10 @@ arch: os: linux +# Small change to check build failure vm: - size: large - + size: large + language: shell notifications: From 351b252c2cf2a1aa6e64242c0bc9b8a046f6cad0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 17:35:53 +0530 Subject: [PATCH 16/21] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe7bc3098e..077c8cc787 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check build failure +# Small change to check build vm: size: large From 2853b7f52a5958a293b6877491cc9ad122529d33 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 18:10:18 +0530 Subject: [PATCH 17/21] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 077c8cc787..d2f93874b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check build +# Small change to check vm: size: large From c03dc64adb8d1c524119abaac5026c75acd50b33 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 18:48:23 +0530 Subject: [PATCH 18/21] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d2f93874b5..89dad4bbab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check +# Small change to vm: size: large From 13024e97f2ebe8dc244b9f1a0fd20fb4be73ae3f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 19:27:12 +0530 Subject: [PATCH 19/21] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 89dad4bbab..a1a2f6bd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to +# Small change vm: size: large From b390c663a456bd855ef93847dacff78615a21e84 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 16 Sep 2021 20:57:25 -0400 Subject: [PATCH 20/21] Upgrade travis instance from medium to large --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a90890423c..2fa9ff9357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: large + language: shell notifications: From cb7c810207c79eddf8daac344eadc9b8ed4092e4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 19 Sep 2021 11:49:24 -0400 Subject: [PATCH 21/21] Drop collection table in deletes worker --- app/controllers/api/database.php | 2 -- app/workers/deletes.php | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index dcef7105ea..26d93ae5be 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -620,8 +620,6 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } - $dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker - if (!$dbForInternal->deleteDocument('collections', $collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f557a3a549..8382e14b14 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -104,6 +104,7 @@ class DeletesV1 extends Worker $collectionId = $document->getId(); $dbForInternal = $this->getInternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $this->deleteByGroup('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) @@ -113,7 +114,7 @@ class DeletesV1 extends Worker new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) ], $dbForInternal); - $dbForInternal->deleteDocument('collections', $collectionId); + $dbForExternal->deleteCollection($collectionId); } /**