From a3fb635168faf2e397082e370e712c9cf6adc783 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 18 Jul 2021 12:48:48 +0545 Subject: [PATCH 1/8] wip --- app/init.php | 1 + app/workers/deletes.php | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 74b354c4d7..7cae96006e 100644 --- a/app/init.php +++ b/app/init.php @@ -86,6 +86,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; +const DELETE_TYPE_BUCKETS = 'buckets'; // Auth Types const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1242d7a8ab..b6fca63433 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -76,7 +76,12 @@ class DeletesV1 extends Worker $document = new Document($this->args['document']); $this->deleteCertificates($document); break; - + + case DELETE_TYPE_BUCKETS: + $bucket = new Document($this->args['document']); + $this->deleteBucket($bucket, $projectId); + break; + default: Console::error('No delete operation for type: '.$type); break; @@ -351,6 +356,11 @@ class DeletesV1 extends Worker Console::info("No certificate files found for {$domain}"); } } + + protected function deleteBucket(Document $document, string $projectId) + { + + } /** * @param string $projectId From 5d14dcfa9d8cd1c7e33a4898cbcf72cdf70b1d01 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jul 2021 15:51:06 +0545 Subject: [PATCH 2/8] delete storage bucket --- app/workers/deletes.php | 11 +++++++++++ composer.json | 2 +- composer.lock | 17 +++++++++-------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 319281837c..f75ccd13bc 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -358,7 +358,18 @@ class DeletesV1 extends Worker protected function deleteBucket(Document $document, string $projectId) { + /** @var Utopia\Database\Database $db */ + $db = $this->getInternalDB($projectId); + $bucketId = $document->getId(); + + $this->deleteByGroup('files',[ + new Query('bucketId', Query::TYPE_EQUAL, [$bucketId]) + ], $db, function () use ($projectId, $bucketId) { + $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); + $device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId); + }); + $db->deleteDocument('buckets', $bucketId); } /** diff --git a/composer.json b/composer.json index bb8003ffb6..fe2eac9604 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", "utopia-php/swoole": "0.2.*", - "utopia-php/storage": "0.5.*", + "utopia-php/storage": "dev-feat-large-file-support", "utopia-php/image": "0.5.*", "resque/php-resque": "1.3.6", "matomo/device-detector": "4.2.3", diff --git a/composer.lock b/composer.lock index f8acf9d0d0..28097b3cae 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "17c1c46d283e4745df9915d082c22078", + "content-hash": "ab60fcd2b30e9c0ddcb490cd67a758c3", "packages": [ { "name": "adhocore/jwt", @@ -2337,16 +2337,16 @@ }, { "name": "utopia-php/storage", - "version": "0.5.0", + "version": "dev-feat-large-file-support", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "92ae20c7a2ac329f573a58a82dc245134cc63408" + "reference": "0d031ffb159efa78b52db547b3471f228472a1a1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/92ae20c7a2ac329f573a58a82dc245134cc63408", - "reference": "92ae20c7a2ac329f573a58a82dc245134cc63408", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/0d031ffb159efa78b52db547b3471f228472a1a1", + "reference": "0d031ffb159efa78b52db547b3471f228472a1a1", "shasum": "" }, "require": { @@ -2383,9 +2383,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.5.0" + "source": "https://github.com/utopia-php/storage/tree/feat-large-file-support" }, - "time": "2021-04-15T16:43:12+00:00" + "time": "2021-07-19T05:38:11+00:00" }, { "name": "utopia-php/swoole", @@ -6237,7 +6237,8 @@ "minimum-stability": "stable", "stability-flags": { "utopia-php/abuse": 20, - "utopia-php/audit": 20 + "utopia-php/audit": 20, + "utopia-php/storage": 20 }, "prefer-stable": false, "prefer-lowest": false, From 1cac46e1606b6e6bb551375150daa2b71232e9fd Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jul 2021 15:52:29 +0545 Subject: [PATCH 3/8] update to reuse --- app/workers/deletes.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f75ccd13bc..6dddc59895 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -361,7 +361,6 @@ class DeletesV1 extends Worker /** @var Utopia\Database\Database $db */ $db = $this->getInternalDB($projectId); $bucketId = $document->getId(); - $this->deleteByGroup('files',[ new Query('bucketId', Query::TYPE_EQUAL, [$bucketId]) @@ -369,7 +368,7 @@ class DeletesV1 extends Worker $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); $device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId); }); - $db->deleteDocument('buckets', $bucketId); + $this->deleteById($document, $db); } /** From 474cbfb198c756002ad5e343012f430dc5eca9d8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jul 2021 15:54:29 +0545 Subject: [PATCH 4/8] reuse existing delete const --- app/init.php | 1 - app/workers/deletes.php | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/init.php b/app/init.php index d689bba9f7..15d0bec83c 100644 --- a/app/init.php +++ b/app/init.php @@ -85,7 +85,6 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_BUCKETS = 'buckets'; // Auth Types const APP_AUTH_TYPE_SESSION = 'Session'; const APP_AUTH_TYPE_JWT = 'JWT'; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 6dddc59895..7e615063b9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -53,6 +53,9 @@ class DeletesV1 extends Worker case 'teams': $this->deleteMemberships($document, $projectId); break; + case 'buckets': + $this->deleteBucket($document, $projectId); + break; default: Console::error('No lazy delete operation available for document of type: '.$document->getCollection()); break; @@ -76,11 +79,6 @@ class DeletesV1 extends Worker $this->deleteCertificates($document); break; - case DELETE_TYPE_BUCKETS: - $bucket = new Document($this->args['document']); - $this->deleteBucket($bucket, $projectId); - break; - default: Console::error('No delete operation for type: '.$type); break; From a6d79059c6457c7798cb339db6a8d51ed8134e77 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jul 2021 15:57:15 +0545 Subject: [PATCH 5/8] fix delete --- app/workers/deletes.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 7e615063b9..8a7f0c45b8 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -356,17 +356,14 @@ class DeletesV1 extends Worker protected function deleteBucket(Document $document, string $projectId) { - /** @var Utopia\Database\Database $db */ - $db = $this->getInternalDB($projectId); $bucketId = $document->getId(); $this->deleteByGroup('files',[ new Query('bucketId', Query::TYPE_EQUAL, [$bucketId]) - ], $db, function () use ($projectId, $bucketId) { + ], $this->getInternalDB($projectId), function () use ($projectId, $bucketId) { $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); $device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId); }); - $this->deleteById($document, $db); } /** From 893b0626a2aec55e424cf9df83577205c37a67da Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 28 Jul 2021 15:59:47 +0545 Subject: [PATCH 6/8] fix deleting path --- app/workers/deletes.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 8a7f0c45b8..e436060854 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -360,10 +360,10 @@ class DeletesV1 extends Worker $this->deleteByGroup('files',[ new Query('bucketId', Query::TYPE_EQUAL, [$bucketId]) - ], $this->getInternalDB($projectId), function () use ($projectId, $bucketId) { - $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); - $device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId); - }); + ], $this->getInternalDB($projectId)); + + $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); + $device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId); } /** From 5aa6e6fda7e231377a98ae62dc47e99680ee0207 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 30 Jul 2021 12:07:02 +0545 Subject: [PATCH 7/8] remove unnecessary functions --- app/workers/deletes.php | 44 ----------------------------------------- 1 file changed, 44 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e436060854..d98042f51a 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -365,48 +365,4 @@ class DeletesV1 extends Worker $device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId); $device->deletePath($device->getRoot() . DIRECTORY_SEPARATOR . $bucketId); } - - /** - * @param string $projectId - * @return Database - */ - protected function getInternalDB($projectId): Database - { - global $register; - - $cache = new Cache(new RedisCache($register->get('cache'))); - $dbForInternal = new Database(new MariaDB($register->get('db')), $cache); - $dbForInternal->setNamespace('project_'.$projectId.'_internal'); // Main DB - - return $dbForInternal; - } - - /** - * @param string $projectId - * @return Database - */ - protected function getExternalDB($projectId): Database - { - global $register; - - $cache = new Cache(new RedisCache($register->get('cache'))); - $dbForExternal = new Database(new MariaDB($register->get('db')), $cache); - $dbForExternal->setNamespace('project_'.$projectId.'_external'); // Main DB - - return $dbForExternal; - } - - /** - * @return Database - */ - protected function getConsoleDB(): Database - { - global $register; - - $cache = new Cache(new RedisCache($register->get('cache'))); - $dbForConsole = new Database(new MariaDB($register->get('db')), $cache); - $dbForConsole->setNamespace('project_console_internal'); // Main DB - - return $dbForConsole; - } } From 4b13c4d60828056e950c69e3092f48dd824fbe6b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 1 Aug 2021 11:09:10 +0545 Subject: [PATCH 8/8] Update app/workers/deletes.php Co-authored-by: kodumbeats --- app/workers/deletes.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index d98042f51a..8476dfa719 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -10,9 +10,6 @@ use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; use Utopia\Audit\Audit; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MariaDB; require_once __DIR__.'/../workers.php';