Merge pull request #3720 from appwrite/feat-allow-automatic-test-retries

Allow flaky tests to be automatically retried with an attribute

tests/e2e/Scopes/Scope.php

@@ -2,12 +2,15 @@
 
 namespace Tests\E2E\Scopes;
 
+use Appwrite\Tests\Retryable;
 use Tests\E2E\Client;
 use PHPUnit\Framework\TestCase;
 use Utopia\Database\ID;
 
 abstract class Scope extends TestCase
 {
+    use Retryable;
+
     /**
      * @var Client
      */

tests/e2e/Services/Account/AccountBase.php

@@ -2,6 +2,7 @@
 
 namespace Tests\E2E\Services\Account;
 
+use Appwrite\Tests\Retry;
 use Tests\E2E\Client;
 use Utopia\Database\ID;
 use Utopia\Database\DateTime;
@@ -519,6 +520,7 @@ trait AccountBase
     /**
      * @depends testUpdateAccountName
      */
+    #[Retry(count: 1)]
     public function testUpdateAccountPassword($data): array
     {
         $email = $data['email'] ?? '';

tests/e2e/Services/Functions/FunctionsCustomServerTest.php

@@ -2,6 +2,7 @@
 
 namespace Tests\E2E\Services\Functions;
 
+use Appwrite\Tests\Retry;
 use CURLFile;
 use Tests\E2E\Client;
 use Tests\E2E\Scopes\ProjectCustom;
@@ -1209,6 +1210,7 @@ class FunctionsCustomServerTest extends Scope
         $this->assertEquals(204, $response['headers']['status-code']);
     }
 
+    #[Retry(count: 1)]
     public function testCreateCustomRubyExecution()
     {
         $name = 'ruby-3.1';

tests/e2e/Services/Webhooks/WebhooksBase.php

@@ -2,6 +2,7 @@
 
 namespace Tests\E2E\Services\Webhooks;
 
+use Appwrite\Tests\Retry;
 use CURLFile;
 use Tests\E2E\Client;
 use Utopia\Database\DateTime;
@@ -304,6 +305,7 @@ trait WebhooksBase
     /**
      * @depends testCreateCollection
      */
+    #[Retry(count: 1)]
     public function testDeleteDocument(array $data): array
     {
         $actorsId = $data['actorsId'];

tests/extensions/Retry.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace Appwrite\Tests;
+
+/**
+ * Allows test methods to be retried if they fail.
+ *
+ * Requires that the test class extends {@see TestCase} and has trait {@see Retryable}.
+ */
+#[\Attribute(\Attribute::TARGET_METHOD)]
+class Retry
+{
+    public function __construct(protected int $count = 1)
+    {
+    }
+}

tests/extensions/Retryable.php

@@ -0,0 +1,70 @@
+<?php
+
+namespace Appwrite\Tests;
+
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Allows test methods annotated with {@see Retry} to be retried.
+ */
+trait Retryable
+{
+    /**
+     * Custom runBare, hides and defers to PHPUnit {@see TestCase} runBare function,
+     * accounting for any retries configured by the {@see Retry} annotation.
+     *
+     * @return void
+     * @throws \ReflectionException
+     * @throws \Throwable
+     */
+    public function runBare(): void
+    {
+        $retries = $this->getNumberOfRetries();
+        $ex = null;
+        for ($i = 0; $i <= $retries; ++$i) {
+            try {
+                parent::runBare();
+                return;
+            } catch (\Throwable | \Exception $ex) {
+                // Swallow the exception until we have exhausted our retries.
+                if ($i !== $retries) {
+                    echo 'Flaky test failed, retrying...' . PHP_EOL;
+                }
+            }
+        }
+        if ($ex) {
+            throw $ex;
+        }
+    }
+
+    /**
+     * @return int
+     * @throws \ReflectionException
+     */
+    private function getNumberOfRetries(): int
+    {
+        $root = new \ReflectionClass($this);
+        $case = $this->getTestCaseRoot($root);
+        $name = $case->getProperty('name');
+        $name->setAccessible(true);
+        $name = $name->getValue($this);
+        $method = $root->getMethod($name);
+        $attributes = $method->getAttributes(Retry::class);
+        $attribute = $attributes[0] ?? null;
+        $args = $attribute?->getArguments();
+        $retries = $args['count'] ?? 0;
+        return \max(0, $retries);
+    }
+
+    /**
+     * @param \ReflectionClass $reflection
+     * @return \ReflectionClass
+     */
+    private function getTestCaseRoot(\ReflectionClass $reflection): \ReflectionClass
+    {
+        if ($reflection->getName() === TestCase::class) {
+            return $reflection;
+        }
+        return $this->getTestCaseRoot($reflection->getParentClass());
+    }
+}