From 5645a003ab8d481e0611a34ebe8a70f5a3876379 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 17 Feb 2022 13:48:25 +0000 Subject: [PATCH 1/4] use internal id for bucket files tables --- app/controllers/api/storage.php | 92 ++++++++++++++++----------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 691c870c9b..ec05c81a60 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -99,7 +99,7 @@ App::post('/v1/storage/buckets') ]); } - $dbForProject->createCollection('bucket_' . $bucketId, $attributes, $indexes); + $dbForProject->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); $bucket = $dbForProject->createDocument('buckets', new Document([ '$id' => $bucketId, @@ -483,11 +483,11 @@ App::post('/v1/storage/buckets/:bucketId/files') $path = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root if ($permissionBucket) { - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $fileId) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } $metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)]; @@ -579,11 +579,11 @@ App::post('/v1/storage/buckets/:bucketId/files') 'metadata' => $metadata, ]); if ($permissionBucket) { - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $doc) { - return $dbForProject->createDocument('bucket_' . $bucketId, $doc); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $doc) { + return $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); }); } else { - $file = $dbForProject->createDocument('bucket_' . $bucketId, $doc); + $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); } } else { $file = $file @@ -601,11 +601,11 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setAttribute('chunksUploaded', $chunksUploaded); if ($permissionBucket) { - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId, $file) { - return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $fileId, $file) { + return $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); }); } else { - $file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); } } @@ -649,11 +649,11 @@ App::post('/v1/storage/buckets/:bucketId/files') 'metadata' => $metadata, ]); if ($permissionBucket) { - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $doc) { - return $dbForProject->createDocument('bucket_' . $bucketId, $doc); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $doc) { + return $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); }); } else { - $file = $dbForProject->createDocument('bucket_' . $bucketId, $doc); + $file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc); } } else { $file = $file @@ -661,11 +661,11 @@ App::post('/v1/storage/buckets/:bucketId/files') ->setAttribute('metadata', $metadata); if ($permissionBucket) { - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId, $file) { - return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $fileId, $file) { + return $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); }); } else { - $file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file); + $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file); } } } catch (StructureException $exception) { @@ -737,10 +737,10 @@ App::get('/v1/storage/buckets/:bucketId/files') if (!empty($cursor)) { if ($bucket->getAttribute('permission') === 'bucket') { $cursorFile = Authorization::skip(function () use ($dbForProject, $bucket, $cursor) { - return $dbForProject->getDocument('bucket_' . $bucket->getId(), $cursor); + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $cursor); }); } else { - $cursorFile = $dbForProject->getDocument('bucket_' . $bucket->getId(), $cursor); + $cursorFile = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $cursor); } if ($cursorFile->isEmpty()) { @@ -755,11 +755,11 @@ App::get('/v1/storage/buckets/:bucketId/files') } if ($bucket->getAttribute('permission') === 'bucket') { - $files = Authorization::skip(function () use ($dbForProject, $bucketId, $queries, $limit, $offset, $cursor, $cursorDirection, $orderType) { - return $dbForProject->find('bucket_' . $bucketId, $queries, $limit, $offset, [], [$orderType], $cursorFile ?? null, $cursorDirection); + $files = Authorization::skip(function () use ($dbForProject, $bucket, $queries, $limit, $offset, $cursor, $cursorDirection, $orderType) { + return $dbForProject->find('bucket_' . $bucket->getInternalId(), $queries, $limit, $offset, [], [$orderType], $cursorFile ?? null, $cursorDirection); }); } else { - $files = $dbForProject->find('bucket_' . $bucketId, $queries, $limit, $offset, [], [$orderType], $cursorFile ?? null, $cursorDirection); + $files = $dbForProject->find('bucket_' . $bucket->getInternalId(), $queries, $limit, $offset, [], [$orderType], $cursorFile ?? null, $cursorDirection); } $usage @@ -769,7 +769,7 @@ App::get('/v1/storage/buckets/:bucketId/files') $response->dynamic(new Document([ 'files' => $files, - 'sum' => $dbForProject->count('bucket_' . $bucketId, $queries, APP_LIMIT_COUNT), + 'sum' => $dbForProject->count('bucket_' . $bucket->getInternalId(), $queries, APP_LIMIT_COUNT), ]), Response::MODEL_FILE_LIST); }); @@ -813,11 +813,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId') } if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $fileId) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { @@ -902,11 +902,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') if ($bucket->getAttribute('permission') === 'bucket') { // skip authorization - $file = Authorization::skip(function () use ($dbForProject, $bucketId, $fileId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $bucket, $fileId) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { @@ -1061,11 +1061,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') } if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucketId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucket) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { @@ -1206,11 +1206,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') } if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucketId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucket) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } $mimes = Config::getParam('storage-mimes'); @@ -1387,11 +1387,11 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') } if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucketId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucket) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { @@ -1399,14 +1399,14 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') } if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucketId, $file, $read, $write) { - return $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file + $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucket, $file, $read, $write) { + return $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file ->setAttribute('$read', $read) ->setAttribute('$write', $write) ); }); } else { - $file = $dbForProject->updateDocument('bucket_' . $bucketId, $fileId, $file + $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file ->setAttribute('$read', $read) ->setAttribute('$write', $write) ); @@ -1475,11 +1475,11 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') } if ($bucket->getAttribute('permission') === 'bucket') { - $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucketId) { - return $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = Authorization::skip(function () use ($dbForProject, $fileId, $bucket) { + return $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $file = $dbForProject->getDocument('bucket_' . $bucketId, $fileId); + $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId); } if ($file->isEmpty() || $file->getAttribute('bucketId') !== $bucketId) { @@ -1503,11 +1503,11 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') $deviceLocal->delete($cacheDir, true); if ($bucket->getAttribute('permission') === 'bucket') { - $deleted = Authorization::skip(function () use ($dbForProject, $fileId, $bucketId) { - return $dbForProject->deleteDocument('bucket_' . $bucketId, $fileId); + $deleted = Authorization::skip(function () use ($dbForProject, $fileId, $bucket) { + return $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId); }); } else { - $deleted = $dbForProject->deleteDocument('bucket_' . $bucketId, $fileId); + $deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId); } if (!$deleted) { throw new Exception('Failed to remove file from DB', 500, Exception::GENERAL_SERVER_ERROR); From 5eb6eeb6b8b776ff69f3ae5e36c43feedd423516 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Feb 2022 01:36:25 +0000 Subject: [PATCH 2/4] fix deletes worker --- app/workers/deletes.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 7eb2ed7094..fd77980f0e 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -458,9 +458,8 @@ class DeletesV1 extends Worker protected function deleteBucket(Document $document, string $projectId) { - $bucketId = $document->getId(); $dbForProject = $this->getProjectDB($projectId); - $dbForProject->deleteCollection('bucket_' . $bucketId); + $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); @@ -483,6 +482,6 @@ class DeletesV1 extends Worker break; } - $device->deletePath($bucketId); + $device->deletePath($document->getId()); } } From 0b6bed0515000916268b53ed39d885542e77d984 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 18 Feb 2022 02:14:57 +0000 Subject: [PATCH 3/4] fix create bucket --- app/controllers/api/storage.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index ec05c81a60..38734cf79b 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -99,8 +99,6 @@ App::post('/v1/storage/buckets') ]); } - $dbForProject->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); - $bucket = $dbForProject->createDocument('buckets', new Document([ '$id' => $bucketId, '$collection' => 'buckets', @@ -117,6 +115,10 @@ App::post('/v1/storage/buckets') '$write' => $write ?? [], 'search' => implode(' ', [$bucketId, $name]), ])); + + $bucket = $dbForProject->getDocument('buckets', $bucketId); + + $dbForProject->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } catch (Duplicate $th) { throw new Exception('Bucket already exists', 409, Exception::STORAGE_BUCKET_ALREADY_EXISTS); } From 658d2e08550f41fcdd93155c553a2c50f2d47c83 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 22 Feb 2022 18:01:03 +0100 Subject: [PATCH 4/4] tests: fix realtime tests --- tests/e2e/Services/Realtime/RealtimeBase.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/Services/Realtime/RealtimeBase.php b/tests/e2e/Services/Realtime/RealtimeBase.php index 4357bec383..62bb7f5d3b 100644 --- a/tests/e2e/Services/Realtime/RealtimeBase.php +++ b/tests/e2e/Services/Realtime/RealtimeBase.php @@ -48,6 +48,7 @@ trait RealtimeBase $this->assertEquals('error', $payload['type']); $this->assertEquals(1008, $payload['data']['code']); $this->assertEquals('Missing channels', $payload['data']['message']); + \usleep(250000); // 250ms $this->expectException(ConnectionException::class); // Check if server disconnnected client $client->close(); }