Merge branch 'feat-db-pools' into refactor-schedulers-workers

CHANGES.md

@@ -4,6 +4,7 @@
 - Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)
 - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776)
 - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780)
+- Store build output file size [#4844](https://github.com/appwrite/appwrite/pull/4844)
 - Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
 
 # Version 1.1.2

app/config/collections.php

@@ -2561,17 +2561,6 @@ $collections = [
                 'array' => false,
                 'filters' => ['datetime'],
             ],
-            [
-                '$id' => ID::custom('endTime'),
-                'type' => Database::VAR_DATETIME,
-                'format' => '',
-                'size' => 0,
-                'signed' => false,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => ['datetime'],
-            ],
             [
                 '$id' => ID::custom('duration'),
                 'type' => Database::VAR_INTEGER,
@@ -2617,7 +2606,7 @@ $collections = [
                 'filters' => [],
             ],
             [
-                '$id' => ID::custom('outputPath'),
+                '$id' => ID::custom('path'),
                 'type' => Database::VAR_STRING,
                 'format' => '',
                 'size' => 2048,
@@ -2627,6 +2616,17 @@ $collections = [
                 'array' => false,
                 'filters' => [],
             ],
+            [
+                '$id' => ID::custom('size'),
+                'type' => Database::VAR_INTEGER,
+                'format' => '',
+                'size' => 0,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
             [
                 '$id' => ID::custom('stderr'),
                 'type' => Database::VAR_STRING,

app/console

@@ -1 +1 @@
-Subproject commit c1ac63889c0a97b280599355e526669bb207880a
+Subproject commit 43891a526e061454617cbb13def3c4901d99a7f1

app/controllers/api/functions.php

@@ -1197,7 +1197,7 @@ App::post('/v1/functions/:functionId/executions')
                 variables: $vars,
                 timeout: $function->getAttribute('timeout', 0),
                 image: $runtime['image'],
-                source: $build->getAttribute('outputPath', ''),
+                source: $build->getAttribute('path', ''),
                 entrypoint: $deployment->getAttribute('entrypoint', ''),
             );
 

app/workers/builds.php

@@ -65,13 +65,13 @@ function buildDeployment(Document $project, Document $function, Document $deploy
             'startTime' => $startTime,
             'deploymentId' => $deployment->getId(),
             'status' => 'processing',
-            'outputPath' => '',
+            'path' => '',
+            'size' => 0,
             'runtime' => $function->getAttribute('runtime'),
             'source' => $deployment->getAttribute('path'),
             'sourceType' => $device,
             'stdout' => '',
             'stderr' => '',
-            'endTime' => null,
             'duration' => 0
         ]));
         $deployment->setAttribute('buildId', $buildId);
@@ -155,14 +155,12 @@ function buildDeployment(Document $project, Document $function, Document $deploy
             ]
         );
 
-        $endTime = new \DateTime();
-        $endTime->setTimestamp($response['endTimeUnix']);
-
         /** Update the build document */
-        $build->setAttribute('endTime', DateTime::format($endTime));
+        $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
         $build->setAttribute('duration', \intval($response['duration']));
         $build->setAttribute('status', $response['status']);
-        $build->setAttribute('outputPath', $response['outputPath']);
+        $build->setAttribute('path', $response['path']);
+        $build->setAttribute('size', $response['size']);
         $build->setAttribute('stderr', $response['stderr']);
         $build->setAttribute('stdout', $response['stdout']);
 
@@ -193,7 +191,6 @@ function buildDeployment(Document $project, Document $function, Document $deploy
     } catch (\Throwable $th) {
         $endTime = DateTime::now();
         $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
-        $build->setAttribute('endTime', $endTime);
         $build->setAttribute('duration', $interval->format('%s') + 0);
         $build->setAttribute('status', 'failed');
         $build->setAttribute('stderr', $th->getMessage());
@@ -229,6 +226,7 @@ function buildDeployment(Document $project, Document $function, Document $deploy
                 ->setParam('builds.{scope}.compute', 1)
                 ->setParam('buildStatus', $build->getAttribute('status', ''))
                 ->setParam('buildTime', $build->getAttribute('duration'))
+                ->setParam('buildSize', $build->getAttribute('size'))
                 ->setParam('networkRequestSize', 0)
                 ->setParam('networkResponseSize', 0)
                 ->submit();

app/workers/deletes.php

@@ -480,10 +480,10 @@ function deleteFunction(Document $document, Document $project): void
         deleteByGroup('builds', [
             Query::equal('deploymentId', [$deploymentId])
         ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) {
-            if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
-                Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
+            if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
+                Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
             } else {
-                Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
+                Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
             }
         });
     }
@@ -529,10 +529,10 @@ function deleteDeployment(Document $document, Document $project): void
     deleteByGroup('builds', [
         Query::equal('deploymentId', [$deploymentId])
     ], $dbForProject, function (Document $document) use ($storageBuilds) {
-        if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
-            Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
+        if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
+            Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
         } else {
-            Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
+            Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
         }
     });
 

app/workers/functions.php

@@ -135,7 +135,7 @@ Server::setResource('execute', function () {
                 variables: $vars,
                 timeout: $function->getAttribute('timeout', 0),
                 image: $runtime['image'],
-                source: $build->getAttribute('outputPath', ''),
+                source: $build->getAttribute('path', ''),
                 entrypoint: $deployment->getAttribute('entrypoint', ''),
             );
 

docker-compose.yml

@@ -53,6 +53,7 @@ services:
         DEBUG: false
         TESTING: true
         VERSION: dev
+        VITE_CONSOLE_MODE: self-hosted
     ports: 
       - 9501:80
     networks:
@@ -658,7 +659,7 @@ services:
     hostname: exc1
     <<: *x-logging
     stop_signal: SIGINT
-    image: openruntimes/executor:0.1.6
+    image: openruntimes/executor:0.2.0
     networks:
       - appwrite
       - runtimes

src/Appwrite/Migration/Version/V17.php

@@ -56,9 +56,48 @@ class V17 extends Migration
                     } catch (\Throwable $th) {
                         Console::warning("'mimeType' from {$id}: {$th->getMessage()}");
                     }
-
                     break;
+                case 'builds':
+                    try {
+                        /**
+                         * Create 'size' attribute
+                         */
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'size');
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (\Throwable $th) {
+                        Console::warning("'size' from {$id}: {$th->getMessage()}");
+                    }
 
+                    try {
+                        /**
+                         * Delete 'endTime' attribute (use startTime+duration if needed)
+                         */
+                        $this->projectDB->deleteAttribute($id, 'endTime');
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (\Throwable $th) {
+                        Console::warning("'endTime' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        /**
+                         * Rename 'outputPath' to 'path'
+                         */
+                        $this->projectDB->renameAttribute($id, 'outputPath', 'path');
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (\Throwable $th) {
+                        Console::warning("'path' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        /**
+                         * Create 'size'
+                         */
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'size');
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (\Throwable $th) {
+                        Console::warning("'size' from {$id}: {$th->getMessage()}");
+                    }
+                    break;
                 default:
                     break;
             }

src/Appwrite/Usage/Stats.php

@@ -184,6 +184,7 @@ class Stats
 
         $functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
         $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
+        $functionBuildSize = ($this->params['buildSize'] ?? 0); // bytes
         $functionBuildStatus = $this->params['buildStatus'] ?? '';
         $functionCompute = $functionExecutionTime + $functionBuildTime;
         $functionTags = $tags . ',functionId=' . $functionId;
@@ -207,6 +208,7 @@ class Stats
         if ($functionBuild >= 1) {
             $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
             $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
+            $this->statsd->count('builds.{scope}.storage.size' . $functionTags, $functionBuildSize);
         }
         if ($functionBuild + $functionExecution >= 1) {
             $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);

src/Appwrite/Utopia/Response/Model/Build.php

@@ -51,18 +51,18 @@ class Build extends Model
                 'default' => '',
                 'example' => self::TYPE_DATETIME_EXAMPLE,
             ])
-            ->addRule('endTime', [
-                'type' => self::TYPE_DATETIME,
-                'description' => 'The time the build was finished in ISO 8601 format.',
-                'default' => '',
-                'example' => self::TYPE_DATETIME_EXAMPLE,
-            ])
             ->addRule('duration', [
                 'type' => self::TYPE_INTEGER,
                 'description' => 'The build duration in seconds.',
                 'default' => 0,
                 'example' => 0,
             ])
+            ->addRule('size', [
+                'type' => self::TYPE_INTEGER,
+                'description' => 'The code size in bytes.',
+                'default' => 0,
+                'example' => 128,
+            ])
         ;
     }