From 6b447232b9f5ba5212289e126f50415ec99fe873 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 21 Oct 2021 15:25:58 +0545 Subject: [PATCH 01/12] fix for buckets --- src/Appwrite/Messaging/Adapter/Realtime.php | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 4269fa04dc..ee42b9effa 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -273,12 +273,6 @@ class Realtime extends Adapter $channels[] = 'teams.' . $payload->getId(); $roles = ['team:' . $payload->getId()]; - break; - case strpos($event, 'database.collections.') === 0: - $channels[] = 'collections'; - $channels[] = 'collections.' . $payload->getId(); - $roles = $payload->getRead(); - break; case strpos($event, 'database.documents.') === 0: $channels[] = 'documents'; @@ -287,8 +281,9 @@ class Realtime extends Adapter $roles = $payload->getRead(); break; - case strpos($event, 'storage.') === 0: + case strpos($event, 'storage.files') === 0: $channels[] = 'files'; + $channels[] = 'buckets.' . $payload->getAttribute('bucketId') . '.files'; $channels[] = 'files.' . $payload->getId(); $roles = $payload->getRead(); From 2481208b384da5449281184b3ddbd393e98a4f7d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 21 Oct 2021 15:26:05 +0545 Subject: [PATCH 02/12] add bucketId --- src/Appwrite/Utopia/Response/Model/File.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Appwrite/Utopia/Response/Model/File.php b/src/Appwrite/Utopia/Response/Model/File.php index 302bb8a61e..d2a5800574 100644 --- a/src/Appwrite/Utopia/Response/Model/File.php +++ b/src/Appwrite/Utopia/Response/Model/File.php @@ -16,6 +16,12 @@ class File extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) + ->addRule('bucketId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Bucket ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) ->addRule('$read', [ 'type' => self::TYPE_STRING, 'description' => 'File read permissions.', From e252906f564b78af2361088564f5183ec471f5e6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 21 Oct 2021 15:26:17 +0545 Subject: [PATCH 03/12] fix test for buckets and database --- tests/e2e/Services/Realtime/RealtimeBase.php | 26 ++++++-------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/tests/e2e/Services/Realtime/RealtimeBase.php b/tests/e2e/Services/Realtime/RealtimeBase.php index 35e20d0a94..a47f36aba2 100644 --- a/tests/e2e/Services/Realtime/RealtimeBase.php +++ b/tests/e2e/Services/Realtime/RealtimeBase.php @@ -623,19 +623,6 @@ trait RealtimeBase 'permission' => 'collection' ]); - $response = json_decode($client->receive(), true); - - $this->assertArrayHasKey('type', $response); - $this->assertArrayHasKey('data', $response); - $this->assertEquals('event', $response['type']); - $this->assertNotEmpty($response['data']); - $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(2, $response['data']['channels']); - $this->assertContains('collections', $response['data']['channels']); - $this->assertContains('collections.' . $actors['body']['$id'], $response['data']['channels']); - $this->assertEquals('database.collections.create', $response['data']['event']); - $this->assertNotEmpty($response['data']['payload']); - $data = ['actorsId' => $actors['body']['$id']]; $name = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['actorsId'] . '/attributes/string', array_merge([ @@ -819,9 +806,10 @@ trait RealtimeBase $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(2, $response['data']['channels']); + $this->assertCount(3, $response['data']['channels']); $this->assertContains('files', $response['data']['channels']); $this->assertContains('files.' . $file['body']['$id'], $response['data']['channels']); + $this->assertContains('buckets.' . $bucketId . '.files', $response['data']['channels']); $this->assertEquals('storage.files.create', $response['data']['event']); $this->assertNotEmpty($response['data']['payload']); @@ -830,7 +818,7 @@ trait RealtimeBase /** * Test File Update */ - $this->client->call(Client::METHOD_PUT, '/storage/buckets/'.$bucketId.'files/' . $data['fileId'], array_merge([ + $this->client->call(Client::METHOD_PUT, '/storage/buckets/'.$bucketId.'/files/' . $data['fileId'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -845,8 +833,9 @@ trait RealtimeBase $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(2, $response['data']['channels']); + $this->assertCount(3, $response['data']['channels']); $this->assertContains('files', $response['data']['channels']); + $this->assertContains('buckets.'.$bucketId.'.files', $response['data']['channels']); $this->assertContains('files.' . $file['body']['$id'], $response['data']['channels']); $this->assertEquals('storage.files.update', $response['data']['event']); $this->assertNotEmpty($response['data']['payload']); @@ -854,7 +843,7 @@ trait RealtimeBase /** * Test File Delete */ - $this->client->call(Client::METHOD_DELETE, '/storage/buckets/'. $bucketId . 'files/' . $data['fileId'], array_merge([ + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/'. $bucketId . '/files/' . $data['fileId'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -866,8 +855,9 @@ trait RealtimeBase $this->assertEquals('event', $response['type']); $this->assertNotEmpty($response['data']); $this->assertArrayHasKey('timestamp', $response['data']); - $this->assertCount(2, $response['data']['channels']); + $this->assertCount(3, $response['data']['channels']); $this->assertContains('files', $response['data']['channels']); + $this->assertContains('buckets.'.$bucketId.'.files', $response['data']['channels']); $this->assertContains('files.' . $file['body']['$id'], $response['data']['channels']); $this->assertEquals('storage.files.delete', $response['data']['event']); $this->assertNotEmpty($response['data']['payload']); From 6fd5df331687c9266be545a76dcc7975c0ba6d57 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 17:26:17 +0200 Subject: [PATCH 04/12] add github action with selfhosted runner --- .github/workflows/tests.yml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..fe11ff62f7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,39 @@ +name: "Tests" + +on: [pull_request] +jobs: + tests: + name: Unit & E2E + runs-on: self-hosted + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + - name: Build Appwrite + # Upstream bug causes buildkit pulls to fail so prefetch base images + # https://github.com/moby/moby/issues/41864 + run: | + echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env + docker pull composer:2.0 + docker pull php:8.0-cli-alpine + docker-compose build + docker-compose up -d + sleep 10 + - name: Doctor + run: docker-compose exec -T appwrite doctor + + - name: Environment Variables + run: docker-compose exec -T appwrite vars + + - name: Run Tests + run: docker-compose exec -T appwrite test --debug \ No newline at end of file From a89e892b3dff21d3ff1a5b0d2660a0093b6d6e99 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 17:32:25 +0200 Subject: [PATCH 05/12] use docker-compose v2 --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe11ff62f7..298fea04de 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,14 +26,14 @@ jobs: echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env docker pull composer:2.0 docker pull php:8.0-cli-alpine - docker-compose build - docker-compose up -d + docker compose build + docker compose up -d sleep 10 - name: Doctor - run: docker-compose exec -T appwrite doctor + run: docker compose exec -T appwrite doctor - name: Environment Variables - run: docker-compose exec -T appwrite vars + run: docker compose exec -T appwrite vars - name: Run Tests - run: docker-compose exec -T appwrite test --debug \ No newline at end of file + run: docker compose exec -T appwrite test --debug \ No newline at end of file From c8036ffc32b1d8e154b82e55bcf892990e049f00 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 18:19:02 +0200 Subject: [PATCH 06/12] reduce timeouts --- .github/workflows/tests.yml | 2 +- tests/e2e/Client.php | 1 - tests/e2e/Scopes/Scope.php | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 298fea04de..574181e4b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: echo "_APP_FUNCTIONS_RUNTIMES=php-8.0" >> .env docker pull composer:2.0 docker pull php:8.0-cli-alpine - docker compose build + docker compose build --progress=plain docker compose up -d sleep 10 - name: Doctor diff --git a/tests/e2e/Client.php b/tests/e2e/Client.php index 82fd642a08..1dc7b71c10 100644 --- a/tests/e2e/Client.php +++ b/tests/e2e/Client.php @@ -156,7 +156,6 @@ class Client */ public function call(string $method, string $path = '', array $headers = [], array $params = []) { - usleep(50000); $headers = array_merge($this->headers, $headers); $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); $responseHeaders = []; diff --git a/tests/e2e/Scopes/Scope.php b/tests/e2e/Scopes/Scope.php index 01673ab7e7..267d155c65 100644 --- a/tests/e2e/Scopes/Scope.php +++ b/tests/e2e/Scopes/Scope.php @@ -33,8 +33,8 @@ abstract class Scope extends TestCase protected function getLastEmail():array { - sleep(10); - + sleep(5); + $emails = json_decode(file_get_contents('http://maildev:1080/email'), true); if ($emails && is_array($emails)) { @@ -46,7 +46,7 @@ abstract class Scope extends TestCase protected function getLastRequest():array { - sleep(5); + sleep(1); $resquest = json_decode(file_get_contents('http://request-catcher:5000/__last_request__'), true); $resquest['data'] = json_decode($resquest['data'], true); From 60b44dcae98b490f878a2c5fcd2c20c1d05d98ee Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 18:30:42 +0200 Subject: [PATCH 07/12] teardown & db race condition --- .github/workflows/tests.yml | 5 ++++- tests/e2e/Services/Database/DatabaseBase.php | 10 +--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 574181e4b5..516a244229 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,4 +36,7 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Tests - run: docker compose exec -T appwrite test --debug \ No newline at end of file + run: docker compose exec -T appwrite test --debug + + - name: Teardown + run: docker compose down -v \ No newline at end of file diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 216b8c0831..c9c6cdd346 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -212,7 +212,6 @@ trait DatabaseBase $this->assertEquals(201, $string['headers']['status-code']); $this->assertEquals('string', $string['body']['key']); $this->assertEquals('string', $string['body']['type']); - $this->assertEquals('processing', $string['body']['status']); $this->assertEquals(false, $string['body']['required']); $this->assertEquals(false, $string['body']['array']); $this->assertEquals(16, $string['body']['size']); @@ -221,7 +220,6 @@ trait DatabaseBase $this->assertEquals(201, $email['headers']['status-code']); $this->assertEquals('email', $email['body']['key']); $this->assertEquals('string', $email['body']['type']); - $this->assertEquals('processing', $email['body']['status']); $this->assertEquals(false, $email['body']['required']); $this->assertEquals(false, $email['body']['array']); $this->assertEquals('email', $email['body']['format']); @@ -230,7 +228,6 @@ trait DatabaseBase $this->assertEquals(201, $enum['headers']['status-code']); $this->assertEquals('enum', $enum['body']['key']); $this->assertEquals('string', $enum['body']['type']); - $this->assertEquals('processing', $enum['body']['status']); $this->assertEquals(false, $enum['body']['required']); $this->assertEquals(false, $enum['body']['array']); $this->assertEquals('enum', $enum['body']['format']); @@ -241,7 +238,6 @@ trait DatabaseBase $this->assertEquals(201, $ip['headers']['status-code']); $this->assertEquals('ip', $ip['body']['key']); $this->assertEquals('string', $ip['body']['type']); - $this->assertEquals('processing', $ip['body']['status']); $this->assertEquals(false, $ip['body']['required']); $this->assertEquals(false, $ip['body']['array']); $this->assertEquals('ip', $ip['body']['format']); @@ -250,7 +246,6 @@ trait DatabaseBase $this->assertEquals(201, $url['headers']['status-code']); $this->assertEquals('url', $url['body']['key']); $this->assertEquals('string', $url['body']['type']); - $this->assertEquals('processing', $url['body']['status']); $this->assertEquals(false, $url['body']['required']); $this->assertEquals(false, $url['body']['array']); $this->assertEquals('url', $url['body']['format']); @@ -259,7 +254,6 @@ trait DatabaseBase $this->assertEquals(201, $integer['headers']['status-code']); $this->assertEquals('integer', $integer['body']['key']); $this->assertEquals('integer', $integer['body']['type']); - $this->assertEquals('processing', $integer['body']['status']); $this->assertEquals(false, $integer['body']['required']); $this->assertEquals(false, $integer['body']['array']); $this->assertEquals(1, $integer['body']['min']); @@ -269,7 +263,6 @@ trait DatabaseBase $this->assertEquals(201, $float['headers']['status-code']); $this->assertEquals('float', $float['body']['key']); $this->assertEquals('double', $float['body']['type']); - $this->assertEquals('processing', $float['body']['status']); $this->assertEquals(false, $float['body']['required']); $this->assertEquals(false, $float['body']['array']); $this->assertEquals(1.5, $float['body']['min']); @@ -279,13 +272,12 @@ trait DatabaseBase $this->assertEquals(201, $boolean['headers']['status-code']); $this->assertEquals('boolean', $boolean['body']['key']); $this->assertEquals('boolean', $boolean['body']['type']); - $this->assertEquals('processing', $boolean['body']['status']); $this->assertEquals(false, $boolean['body']['required']); $this->assertEquals(false, $boolean['body']['array']); $this->assertEquals(true, $boolean['body']['default']); // wait for database worker to create attributes - sleep(5); + sleep(2); $stringResponse = $this->client->call(Client::METHOD_GET, "/database/collections/{$collectionId}/attributes/{$collectionId}_{$string['body']['key']}",array_merge([ 'content-type' => 'application/json', From 744c093739c334da51049836e5df2c9c71c836a2 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 18:34:05 +0200 Subject: [PATCH 08/12] teardown functions --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 516a244229..65b47c3cf3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,4 +39,6 @@ jobs: run: docker compose exec -T appwrite test --debug - name: Teardown - run: docker compose down -v \ No newline at end of file + run: | + docker compose down -v + docker ps -q | xargs docker stop From 2c6f4d0cdb1dc1932f0805b38149b1a52d0198b3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 18:52:13 +0200 Subject: [PATCH 09/12] fix db tests --- tests/e2e/Services/Database/DatabaseBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index c9c6cdd346..77cb5548cc 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -277,7 +277,7 @@ trait DatabaseBase $this->assertEquals(true, $boolean['body']['default']); // wait for database worker to create attributes - sleep(2); + sleep(10); $stringResponse = $this->client->call(Client::METHOD_GET, "/database/collections/{$collectionId}/attributes/{$collectionId}_{$string['body']['key']}",array_merge([ 'content-type' => 'application/json', From 744560fbb9d1e1b13b091103c42bd25398ee83c9 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 18:52:22 +0200 Subject: [PATCH 10/12] always run teardown --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65b47c3cf3..1cf5850dfb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,6 +39,7 @@ jobs: run: docker compose exec -T appwrite test --debug - name: Teardown + if: always() run: | docker compose down -v docker ps -q | xargs docker stop From dc9912753304c6811de9eeaab6dc83301cc141d3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 19:21:52 +0200 Subject: [PATCH 11/12] increase sleeps --- .github/workflows/tests.yml | 2 +- tests/e2e/Services/Database/DatabaseBase.php | 2 +- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1cf5850dfb..2cb350d195 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,4 +42,4 @@ jobs: if: always() run: | docker compose down -v - docker ps -q | xargs docker stop + docker ps -aq | xargs docker rm --force diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 77cb5548cc..1f5c633f1e 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -277,7 +277,7 @@ trait DatabaseBase $this->assertEquals(true, $boolean['body']['default']); // wait for database worker to create attributes - sleep(10); + sleep(30); $stringResponse = $this->client->call(Client::METHOD_GET, "/database/collections/{$collectionId}/attributes/{$collectionId}_{$string['body']['key']}",array_merge([ 'content-type' => 'application/json', diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index f4e1d48041..d268cb492d 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -588,7 +588,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(201, $attribute['headers']['status-code']); } - sleep(5); + sleep(30); $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ 'content-type' => 'application/json', From bc32535905ad63aa61bd245448a4539e9539124c Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 22 Oct 2021 19:36:54 +0200 Subject: [PATCH 12/12] disable travis temporarily --- .travis.yml => .travis.yml.tmp | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .travis.yml => .travis.yml.tmp (100%) diff --git a/.travis.yml b/.travis.yml.tmp similarity index 100% rename from .travis.yml rename to .travis.yml.tmp