Merge branch 'feat-database-indexing' into feat-storage-buckets

.travis.yml

@@ -6,6 +6,10 @@ arch:
 
 os: linux
 
+# Small change
+vm:
+  size: large 
+  
 language: shell
 
 notifications:
@@ -35,6 +39,12 @@ install:
 
 script:
 - docker ps -a
+# Tests should fail if any container is in exited status
+- ALL_UP=`docker ps -aq --filter "status=exited"`
+- >
+  if [[ "$ALL_UP" != "" ]]; then
+    exit 1
+  fi
 - docker-compose logs appwrite
 - docker-compose logs mariadb
 - docker-compose logs appwrite-worker-functions

app/controllers/api/database.php

@@ -620,12 +620,15 @@ App::delete('/v1/database/collections/:collectionId')
             throw new Exception('Collection not found', 404);
         }
 
-        $dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker
-
         if (!$dbForInternal->deleteDocument('collections', $collectionId)) {
             throw new Exception('Failed to remove collection from DB', 500);
         }
 
+        $deletes
+            ->setParam('type', DELETE_TYPE_DOCUMENT)
+            ->setParam('document', $collection)
+        ;
+
         $usage->setParam('database.collections.delete', 1);
 
         $events

app/tasks/maintenance.php

@@ -42,7 +42,7 @@ $cli
         function notifyDeleteUsageStats(int $interval30m, int $interval1d) 
         {
             Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
-                'type' => DELETE_TYPE_USAGE_STATS,
+                'type' => DELETE_TYPE_USAGE,
                 'timestamp1d' => time() - $interval1d,
                 'timestamp30m' => time() - $interval30m,
             ]);

app/tasks/usage.php

@@ -255,10 +255,27 @@ $cli
 
             /**
              * Aggregate InfluxDB every 30 seconds
+             * @var InfluxDB\Client $client
              */
             $client = $register->get('influxdb');
             if ($client) {
+                $attempts = 0;
+                $max = 10;
+                $sleep = 1;
+
                 $database = $client->selectDB('telegraf');
+                do { // check if telegraf database is ready
+                    $attempts++;
+                    if(!in_array('telegraf', $client->listDatabases())) {
+                        Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
+                        if($attempts >= $max) {
+                            throw new \Exception('InfluxDB database not ready yet');
+                        }
+                        sleep($sleep);
+                    } else {
+                        break; // leave the do-while if successful
+                    }
+                } while ($attempts < $max);
 
                 // sync data
                 foreach ($globalMetrics as $metric => $options) { //for each metrics
@@ -335,7 +352,23 @@ $cli
                 $latestProject = null;
 
                 do { // Loop over all the projects
-                    $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
+                    $attempts = 0;
+                    $max = 10;
+                    $sleep = 1;
+
+                    do { // list projects
+                        try {
+                            $attempts++;
+                            $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
+                            break; // leave the do-while if successful
+                        } catch (\Exception $e) {
+                            Console::warning("Console DB not ready yet. Retrying ({$attempts})...");
+                            if ($attempts >= $max) {
+                                throw new \Exception('Failed access console db: ' . $e->getMessage());
+                            }
+                            sleep($sleep);
+                        }
+                    } while ($attempts < $max);
 
                     if (empty($projects)) {
                         continue;

app/workers/deletes.php

@@ -44,6 +44,9 @@ class DeletesV1 extends Worker
                 
                 switch ($document->getCollection()) {
                     // TODO@kodumbeats define these as constants somewhere
+                    case 'collections':
+                        $this->deleteCollection($document, $projectId);
+                        break;
                     case 'projects':
                         $this->deleteProject($document);
                         break;
@@ -79,7 +82,7 @@ class DeletesV1 extends Worker
                 $this->deleteCertificates($document);
                 break;
 
-            case DELETE_TYPE_USAGE_STATS:
+            case DELETE_TYPE_USAGE:
                 $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']);
                 break;
             default:
@@ -92,6 +95,28 @@ class DeletesV1 extends Worker
     {
     }
 
+    /**
+     * @param Document $document teams document
+     * @param string $projectId
+     */
+    protected function deleteCollection(Document $document, string $projectId): void
+    {
+        $collectionId = $document->getId();
+
+        $dbForInternal = $this->getInternalDB($projectId);
+        $dbForExternal = $this->getExternalDB($projectId);
+
+        $this->deleteByGroup('attributes', [
+            new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
+        ], $dbForInternal);
+
+        $this->deleteByGroup('indexes', [
+            new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
+        ], $dbForInternal);
+
+        $dbForExternal->deleteCollection($collectionId);
+    }
+
     /**
      * @param int $timestamp1d
      * @param int $timestamp30m

tests/e2e/Services/Database/DatabaseCustomServerTest.php

@@ -403,5 +403,13 @@ class DatabaseCustomServerTest extends Scope
 
         $this->assertEquals(400, $tooMany['headers']['status-code']);
         $this->assertEquals('Index limit exceeded', $tooMany['body']['message']);
+
+        $collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ]));
+
+        $this->assertEquals(204, $collection['headers']['status-code']);
     }
 }