diff --git a/.env b/.env index f59620ec5..441606192 100644 --- a/.env +++ b/.env @@ -39,6 +39,7 @@ _APP_FUNCTIONS_CONTAINERS=10 _APP_FUNCTIONS_CPUS=4 _APP_FUNCTIONS_MEMORY=2000 _APP_FUNCTIONS_MEMORY_SWAP=2000 +_APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes _APP_EXECUTOR_SECRET=a-random-secret _APP_MAINTENANCE_INTERVAL=86400 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 diff --git a/Dockerfile b/Dockerfile index 3ae4ef217..a8ad50552 100755 --- a/Dockerfile +++ b/Dockerfile @@ -174,6 +174,7 @@ ENV _APP_SERVER=swoole \ _APP_FUNCTIONS_MEMORY=128 \ _APP_FUNCTIONS_MEMORY_SWAP=128 \ _APP_EXECUTOR_SECRET=a-random-secret \ + _APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ _APP_USAGE_STATS=enabled \ diff --git a/app/config/collections.php b/app/config/collections.php index 418fdfa3c..7614cde4a 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1915,7 +1915,18 @@ $collections = [ 'filters' => [], ], [ - '$id' => 'functionId', + '$id' => 'resourceId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'resourceType', 'type' => Database::VAR_STRING, 'format' => '', 'size' => Database::LENGTH_KEY, @@ -1981,39 +1992,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => 'status', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 128, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'buildStdout', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 4096, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'buildStderr', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 4096, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], [ '$id' => 'deploy', 'type' => Database::VAR_BOOLEAN, @@ -2028,9 +2006,16 @@ $collections = [ ], 'indexes' => [ [ - '$id' => '_key_function', + '$id' => '_key_resource', 'type' => Database::INDEX_KEY, - 'attributes' => ['functionId'], + 'attributes' => ['resourceId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_resource_type', + 'type' => Database::INDEX_KEY, + 'attributes' => ['resourceType'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], @@ -2050,10 +2035,10 @@ $collections = [ 'name' => 'Builds', 'attributes' => [ [ - '$id' => 'deploymentId', - 'type' => Database::VAR_STRING, + '$id' => 'startTime', + 'type' => Database::VAR_INTEGER, 'format' => '', - 'size' => Database::LENGTH_KEY, + 'size' => 0, 'signed' => true, 'required' => false, 'default' => null, @@ -2061,7 +2046,7 @@ $collections = [ 'filters' => [], ], [ - '$id' => 'dateCreated', + '$id' => 'endTime', 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, @@ -2071,6 +2056,28 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'duration', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'deploymentId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'runtime', 'type' => Database::VAR_STRING, @@ -2126,28 +2133,6 @@ $collections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => 'time', - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'vars', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => new stdClass(), - 'array' => false, - 'filters' => ['json'], - ], [ '$id' => 'sourceType', 'type' => Database::VAR_STRING, @@ -2169,18 +2154,7 @@ $collections = [ 'default' => '', 'array' => false, 'filters' => [], - ], - [ - '$id' => 'search', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], + ] ], 'indexes' => [ [ @@ -2189,14 +2163,7 @@ $collections = [ 'attributes' => ['deploymentId'], 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_search', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [2048], - 'orders' => [Database::ORDER_ASC], - ], + ] ], ], diff --git a/app/config/variables.php b/app/config/variables.php index 76dbf1cd5..93f124397 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -498,6 +498,15 @@ return [ 'question' => '', 'filter' => '' ], + [ + 'name' => '_APP_EXECUTOR_RUNTIME_NETWORK', + 'description' => 'The docker network used for communication between the executor and runtimes. Change this if you have altered the default network names.', + 'introduction' => '0.13.0', + 'default' => 'appwrite_runtimes', + 'required' => false, + 'question' => '', + 'filter' => '' + ], [ 'name' => '_APP_FUNCTIONS_ENVS', 'description' => 'Deprecated with 0.8.0, use \'_APP_FUNCTIONS_RUNTIMES\' instead!', diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 31a6c593a..e34c4cf01 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -3,6 +3,7 @@ use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; use Appwrite\Database\Validator\CustomId; +use Appwrite\Event\Event; use Utopia\Database\Validator\UID; use Utopia\Storage\Storage; use Utopia\Storage\Validator\File; @@ -522,7 +523,8 @@ App::post('/v1/functions/:functionId/deployments') // Remove deploy for all other deployments. $deployments = $dbForProject->find('deployments', [ new Query('deploy', Query::TYPE_EQUAL, [true]), - new Query('functionId', Query::TYPE_EQUAL, [$functionId]) + new Query('resourceId', Query::TYPE_EQUAL, [$functionId]), + new Query('resourceType', Query::TYPE_EQUAL, ['functions']) ]); foreach ($deployments as $deployment) { @@ -536,20 +538,18 @@ App::post('/v1/functions/:functionId/deployments') '$id' => $deploymentId, '$read' => ['role:all'], '$write' => ['role:all'], - 'functionId' => $function->getId(), + 'resourceId' => $function->getId(), + 'resourceType' => 'functions', 'dateCreated' => time(), 'entrypoint' => $entrypoint, 'path' => $path, 'size' => $size, 'search' => implode(' ', [$deploymentId, $entrypoint]), - 'status' => 'processing', - 'buildStdout' => '', - 'buildStderr' => '', 'deploy' => ($deploy === 'true'), ])); // Enqueue a message to start the build - Resque::enqueue('v1-builds', 'BuildsV1', [ + Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ 'projectId' => $project->getId(), 'functionId' => $function->getId(), 'deploymentId' => $deploymentId, @@ -557,7 +557,7 @@ App::post('/v1/functions/:functionId/deployments') ]); $usage - ->setParam('storage', $deployment->getAttribute('size', 0)) + ->setParam('storage', $deployment->getAttribute('size', 0)) ; @@ -609,18 +609,17 @@ App::get('/v1/functions/:functionId/deployments') $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); } - $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); + $queries[] = new Query('resourceId', Query::TYPE_EQUAL, [$function->getId()]); + $queries[] = new Query('resourceType', Query::TYPE_EQUAL, ['functions']); $results = $dbForProject->find('deployments', $queries, $limit, $offset, [], [$orderType], $cursorDeployment ?? null, $cursorDirection); $sum = $dbForProject->count('deployments', $queries, APP_LIMIT_COUNT); - // Get Current Build Data - foreach ($results as &$deployment) { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - - $deployment['status'] = $build->getAttribute('status', 'processing'); - $deployment['buildStdout'] = $build->getAttribute('stdout', ''); - $deployment['buildStderr'] = $build->getAttribute('stderr', ''); + foreach ($results as $result) { + $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId')); + $result->setAttribute('status', $build->getAttribute('status', 'pending')); + $result->setAttribute('buildStderr', $build->getAttribute('stderr', '')); + $result->setAttribute('buildStdout', $build->getAttribute('stdout', '')); } $response->dynamic(new Document([ @@ -656,7 +655,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId') $deployment = $dbForProject->getDocument('deployments', $deploymentId); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } @@ -761,7 +760,7 @@ App::post('/v1/functions/:functionId/executions') $deployment = Authorization::skip(fn() => $dbForProject->getDocument('deployments', $function->getAttribute('deployment'))); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found. Deploy deployment before trying to execute a function', 404); } @@ -959,76 +958,6 @@ App::get('/v1/functions/:functionId/executions/:executionId') $response->dynamic($execution, Response::MODEL_EXECUTION); }); -App::get('/v1/builds') -->groups(['api', 'functions']) -->desc('List Builds') -->label('scope', 'functions.read') -->label('sdk.auth', [APP_AUTH_TYPE_KEY]) -->label('sdk.namespace', 'functions') -->label('sdk.method', 'builds') -->label('sdk.description', '/docs/references/functions/list-builds.md') -->label('sdk.response.code', Response::STATUS_CODE_OK) -->label('sdk.response.type', Response::CONTENT_TYPE_JSON) -->label('sdk.response.model', Response::MODEL_BUILD_LIST) -->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) -->param('limit', 25, new Range(0, 100), 'Maximum number of builds to return in response. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) -->param('offset', 0, new Range(0, APP_LIMIT_COUNT), 'Offset value. The default value is 0. Use this value to manage pagination. [learn more about pagination](https://appwrite.io/docs/pagination)', true) -->param('cursor', '', new UID(), 'ID of the function used as the starting point for the query, excluding the function itself. Should be used for efficient pagination when working with large sets of data. [learn more about pagination](https://appwrite.io/docs/pagination)', true) -->param('cursorDirection', Database::CURSOR_AFTER, new WhiteList([Database::CURSOR_AFTER, Database::CURSOR_BEFORE]), 'Direction of the cursor.', true) -->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) -->inject('response') -->inject('dbForProject') -->action(function ($search, $limit, $offset, $cursor, $cursorDirection, $orderType, $response, $dbForProject) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForProject */ - - if (!empty($cursor)) { - $cursorFunction = $dbForProject->getDocument('builds', $cursor); - - if ($cursorFunction->isEmpty()) { - throw new Exception("Build '{$cursor}' for the 'cursor' value not found.", 400); - } - } - - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - - $response->dynamic(new Document([ - 'builds' => $dbForProject->find('builds', $queries, $limit, $offset, [], [$orderType], $cursorFunction ?? null, $cursorDirection), - 'sum' => $dbForProject->count('builds', $queries, APP_LIMIT_COUNT), - ]), Response::MODEL_BUILD_LIST); -}); - -App::get('/v1/builds/:buildId') - ->groups(['api', 'functions']) - ->desc('Get Build') - ->label('scope', 'execution.read') - ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) - ->label('sdk.namespace', 'functions') - ->label('sdk.method', 'getBuild') - ->label('sdk.description', '/docs/references/functions/get-build.md') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_BUILD) - ->param('buildId', '', new UID(), 'Build unique ID.') - ->inject('response') - ->inject('dbForProject') - ->action(function ($buildId, $response, $dbForProject) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForProject */ - - $build = Authorization::skip(fn() => $dbForProject->getDocument('builds', $buildId)); - - if ($build->isEmpty()) { - throw new Exception('Build not found', 404); - } - - $response->dynamic($build, Response::MODEL_BUILD); - }); - App::post('/v1/builds/:buildId') ->groups(['api', 'functions']) ->desc('Retry Build') @@ -1060,7 +989,7 @@ App::post('/v1/builds/:buildId') } // Enqueue a message to start the build - Resque::enqueue('v1-builds', 'BuildsV1', [ + Resque::enqueue(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME, [ 'projectId' => $project->getId(), 'buildId' => $buildId, 'type' => BUILD_TYPE_RETRY diff --git a/app/executor.php b/app/executor.php index d8283447f..39ba40862 100644 --- a/app/executor.php +++ b/app/executor.php @@ -25,7 +25,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Logger\Log; -use Utopia\Orchestration\Adapter\DockerAPI; use Utopia\Orchestration\Adapter\DockerCLI; use Utopia\Orchestration\Orchestration; use Utopia\Registry\Registry; @@ -184,7 +183,7 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl // Check if runtime is active $runtime = $runtimes[$function->getAttribute('runtime', '')] ?? null; - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('deployment not found', 404); } @@ -194,6 +193,7 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl // Process environment variables $vars = \array_merge($function->getAttribute('vars', []), [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint', ''), 'APPWRITE_FUNCTION_ID' => $function->getId(), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), @@ -203,8 +203,6 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl 'INTERNAL_RUNTIME_KEY' => $secret ]); - $vars = \array_merge($vars, $build->getAttribute('vars', [])); // for gettng endpoint. - $container = 'appwrite-function-' . $deployment->getId(); if ($activeFunctions->exists($container) && !(\substr($activeFunctions->get($container)['status'], 0, 2) === 'Up')) { // Remove container if not online @@ -301,7 +299,7 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl } // Add to network - $orchestration->networkConnect($container, 'appwrite_runtimes'); + $orchestration->networkConnect($container, App::getEnv('_APP_EXECUTOR_RUNTIME_NETWORK', 'appwrite_runtimes')); $executionEnd = \microtime(true); @@ -317,7 +315,7 @@ function createRuntimeServer(string $functionId, string $projectId, string $depl Console::success('Runtime server is ready to run'); } } catch (\Throwable $th) { - var_dump($th->getTraceAsString()); + Console::error($th->getMessage()); $orchestrationPool->put($orchestration ?? null); throw $th; } @@ -336,7 +334,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $deployment = $database->getDocument('deployments', $function->getAttribute('deployment', '')); $build = $database->getDocument('builds', $deployment->getAttribute('buildId', '')); - if ($deployment->getAttribute('functionId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new Exception('Deployment not found', 404); } @@ -365,7 +363,7 @@ function execute(string $trigger, string $projectId, string $executionId, string } - if ($build->getAttribute('status') == 'building') { + if ($build->getAttribute('status') === 'building') { $execution ->setAttribute('status', 'failed') @@ -387,6 +385,7 @@ function execute(string $trigger, string $projectId, string $executionId, string // Process environment variables $vars = \array_merge($function->getAttribute('vars', []), [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint', ''), 'APPWRITE_FUNCTION_ID' => $function->getId(), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), @@ -401,8 +400,6 @@ function execute(string $trigger, string $projectId, string $executionId, string 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, ]); - $vars = \array_merge($vars, $build->getAttribute('vars', [])); - $container = 'appwrite-function-' . $deployment->getId(); try { @@ -413,8 +410,8 @@ function execute(string $trigger, string $projectId, string $executionId, string '$id' => $buildId, '$read' => ($userId !== '') ? ['user:' . $userId] : [], '$write' => [], + 'startTime' => time(), 'deploymentId' => $deployment->getId(), - 'dateCreated' => time(), 'status' => 'processing', 'outputPath' => '', 'runtime' => $function->getAttribute('runtime', ''), @@ -422,15 +419,8 @@ function execute(string $trigger, string $projectId, string $executionId, string 'sourceType' => Storage::DEVICE_LOCAL, 'stdout' => '', 'stderr' => '', - 'time' => 0, - 'vars' => [ - 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint'), - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'], - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'], - 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId, - ] + 'endTime' => 0, + 'duration' => 0 ])); $deployment->setAttribute('buildId', $buildId); @@ -484,6 +474,7 @@ function execute(string $trigger, string $projectId, string $executionId, string // Process environment variables $vars = \array_merge($function->getAttribute('vars', []), [ + 'ENTRYPOINT_NAME' => $deployment->getAttribute('entrypoint', ''), 'APPWRITE_FUNCTION_ID' => $function->getId(), 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), @@ -498,8 +489,6 @@ function execute(string $trigger, string $projectId, string $executionId, string 'APPWRITE_FUNCTION_PROJECT_ID' => $projectId ]); - $vars = \array_merge($vars, $build->getAttribute('vars', [])); - $stdout = ''; $stderr = ''; @@ -520,7 +509,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $body = \json_encode([ 'path' => '/usr/code', - 'file' => $build->getAttribute('vars', [])['ENTRYPOINT_NAME'], + 'file' => $vars['ENTRYPOINT_NAME'], 'env' => $vars, 'payload' => $data, 'timeout' => $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)) @@ -635,7 +624,7 @@ function execute(string $trigger, string $projectId, string $executionId, string roles: $target['roles'] ); - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $statsd = $register->get('statsd'); $usage = new Stats($statsd); @@ -684,6 +673,10 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) $build = $database->getDocument('builds', $buildId); $deployment = $database->getDocument('deployments', $deploymentId); + + // Start tracking time + $buildStart = \time(); + try { // If we already have a built package ready there is no need to rebuild. if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) { @@ -736,11 +729,25 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) throw new Exception('Code is not readable: ' . $build->getAttribute('source', '')); } - $vars = $build->getAttribute('vars', []); + $deployment = $database->getDocument('deployments', $build->getAttribute('deploymentId', '')); + $resourceId = $deployment->getAttribute('resourceId', ''); + $resourceType = $deployment->getAttribute('resourceType', ''); - // Start tracking time - $buildStart = \microtime(true); - $time = \time(); + if (empty($resourceId)) { + throw new Exception('Invalid resource ID on build ' . $build->getId()); + } + + if (empty($resourceType)) { + throw new Exception('Invalid resource type on build' . $build->getId()); + } + + $resource = $database->getDocument($resourceType, $resourceId); + + if ($resource->isEmpty()) { + throw new Exception('Resource not found on build ' . $build->getId()); + } + + $vars = $resource->getAttribute('vars', []); $orchestration ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) @@ -766,7 +773,7 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) workdir: '/usr/code', labels: [ 'appwrite-type' => 'function', - 'appwrite-created' => strval($time), + 'appwrite-created' => strval($buildStart), 'appwrite-runtime' => $build->getAttribute('runtime', ''), 'appwrite-project' => $projectID, 'appwrite-build' => $buildId, @@ -871,7 +878,7 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) } } - if ($buildStdout == '') { + if ($buildStdout === '') { $buildStdout = 'Build Successful!'; } @@ -880,22 +887,24 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) ->setAttribute('status', 'ready') ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) ->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096))) - ->setAttribute('time', $time); + ->setAttribute('startTime', $buildStart) + ->setAttribute('endTime', \time()) + ->setAttribute('duration', \time() - $buildStart); // Update build with built code attribute $build = $database->updateDocument('builds', $buildId, $build); - $deployment->setAttribute('status', 'ready'); - $database->updateDocument('deployments', $deploymentId, $deployment); - - $buildEnd = \microtime(true); + $buildEnd = \time(); Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds'); } catch (Exception $e) { $build ->setAttribute('status', 'failed') ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) - ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))); + ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))) + ->setAttribute('startTime', $buildStart) + ->setAttribute('endTime', \microtime(true)) + ->setAttribute('duration', \microtime(true) - $buildStart); $build = $database->updateDocument('builds', $buildId, $build); @@ -923,16 +932,16 @@ function runBuildStage(string $buildId, string $deploymentId, string $projectID) App::post('/v1/functions/:functionId/executions') ->desc('Execute a function') - ->param('trigger', '', new Text(1024)) - ->param('projectId', '', new Text(1024)) - ->param('executionId', '', new Text(1024), '', true) - ->param('functionId', '', new Text(1024)) - ->param('event', '', new Text(1024), '', true) - ->param('eventData', '', new Text(10240), '', true) - ->param('data', '', new Text(1024), '', true) - ->param('webhooks', [], new ArrayList(new JSON()), '', true) - ->param('userId', '', new Text(1024), '', true) - ->param('jwt', '', new Text(1024), '', true) + ->param('trigger', '', new Text(1024), 'What triggered this execution, can be http / schedule / event') + ->param('projectId', '', new Text(1024), 'The ProjectID this execution belongs to') + ->param('executionId', '', new Text(1024), 'An optional execution ID, If not specified a new execution document is created.', true) + ->param('functionId', '', new Text(1024), 'The FunctionID to execute') + ->param('event', '', new Text(1024), 'The event that triggered this execution', true) + ->param('eventData', '', new Text(0), 'Extra Data for the event', true) + ->param('data', '', new Text(1024), 'Data to be forwarded to the function, this is user specified.', true) + ->param('webhooks', [], new ArrayList(new JSON()), 'Any webhooks that need to be triggered after this execution', true) + ->param('userId', '', new Text(1024), 'The UserID of the user who triggered the execution if it was called from a client SDK', true) + ->param('jwt', '', new Text(1024), 'A JWT of the user who triggered the execution if it was called from a client SDK', true) ->inject('response') ->inject('dbForProject') ->action( diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index a4e8aeb0c..6d2e39f47 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -191,6 +191,7 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_RUNTIME_NETWORK - _APP_USAGE_STATS - _APP_STATSD_HOST - _APP_STATSD_PORT @@ -221,6 +222,31 @@ services: - _APP_DB_USER - _APP_DB_PASS + appwrite-worker-builds: + image: /: + entrypoint: worker-builds + container_name: appwrite-worker-builds + restart: unless-stopped + networks: + - appwrite + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_EXECUTOR_SECRET + appwrite-worker-audits: image: /: entrypoint: worker-audits diff --git a/app/workers/builds.php b/app/workers/builds.php index 0e529249d..25792c8de 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -138,7 +138,6 @@ class BuildsV1 extends Worker ] ])); } catch (\Throwable $th) { - $deployment->setAttribute('status', 'failed'); $deployment->setAttribute('buildId', ''); $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); Console::error($th->getMessage()); @@ -152,8 +151,6 @@ class BuildsV1 extends Worker $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); $this->createBuild($projectId, $functionId, $deploymentId, $buildId); } catch (\Throwable $th) { - $deployment->setAttribute('status', 'failed'); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); Console::error($th->getMessage()); throw $th; } diff --git a/docker-compose.yml b/docker-compose.yml index 68cc05c64..a73b34a7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -385,8 +385,6 @@ services: - /usr/src/code/app/executor.php - -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php stop_signal: SIGINT - ports: - - "8080:8080" build: context: . args: @@ -427,6 +425,7 @@ services: - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_SECRET + - _APP_EXECUTOR_RUNTIME_NETWORK - _APP_USAGE_STATS - _APP_STATSD_HOST - _APP_STATSD_PORT diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index a367e0790..2bd73248c 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -30,6 +30,9 @@ class Event const CERTIFICATES_QUEUE_NAME = 'v1-certificates'; const CERTIFICATES_CLASS_NAME = 'CertificatesV1'; + + const BUILDS_QUEUE_NAME = 'v1-builds'; + const BUILDS_CLASS_NAME = 'BuildsV1'; /** * @var string diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php index fbf55a4a2..218e82d5e 100644 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ b/src/Appwrite/Utopia/Response/Model/Build.php @@ -16,12 +16,18 @@ class Build extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('dateCreated', [ + ->addRule('startTime', [ 'type' => self::TYPE_INTEGER, 'description' => 'The deployment creation date in Unix timestamp.', 'default' => 0, 'example' => 1592981250, ]) + ->addRule('deploymentId', [ + 'type' => self::TYPE_STRING, + 'description' => 'The deployment that created this build', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) // Build Status // Failed - The deployment build has failed. More details can usually be found in buildStderr // Ready - The deployment build was successful and the deployment is ready to be deployed @@ -45,7 +51,13 @@ class Build extends Model 'default' => '', 'example' => '', ]) - ->addRule('time', [ + ->addRule('endTime', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The time the build was finished in Unix timestamp.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('duration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The build time in seconds.', 'default' => 0, diff --git a/src/Appwrite/Utopia/Response/Model/Deployment.php b/src/Appwrite/Utopia/Response/Model/Deployment.php index db1f518c9..28a01ad6c 100644 --- a/src/Appwrite/Utopia/Response/Model/Deployment.php +++ b/src/Appwrite/Utopia/Response/Model/Deployment.php @@ -16,12 +16,18 @@ class Deployment extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('functionId', [ + ->addRule('resourceId', [ 'type' => self::TYPE_STRING, - 'description' => 'Function ID.', + 'description' => 'Resource ID.', 'default' => '', 'example' => '5e5ea6g16897e', ]) + ->addRule('resourceType', [ + 'type' => self::TYPE_STRING, + 'description' => 'Resource type.', + 'default' => '', + 'example' => 'functions', + ]) ->addRule('dateCreated', [ 'type' => self::TYPE_INTEGER, 'description' => 'The deployment creation date in Unix timestamp.', @@ -30,7 +36,7 @@ class Deployment extends Model ]) ->addRule('entrypoint', [ 'type' => self::TYPE_STRING, - 'description' => 'The entrypoint file to use to execute the delpoyment code.', + 'description' => 'The entrypoint file to use to execute the deployment code.', 'default' => '', 'example' => 'enabled', ]) @@ -40,41 +46,36 @@ class Deployment extends Model 'default' => 0, 'example' => 128, ]) - // Build Status - // Failed - The deployment build has failed. More details can usually be found in buildStderr - // Ready - The deployment build was successful and the deployment is ready to be deployed - // Processing - The deployment is currently waiting to have a build triggered - // Building - The deployment is currently being built - ->addRule('status', [ - 'type' => self::TYPE_STRING, - 'description' => 'The deployment\'s current built status', - 'default' => '', - 'example' => 'ready', - ]) ->addRule('buildId', [ 'type' => self::TYPE_STRING, 'description' => 'The current build ID.', 'default' => '', 'example' => '5e5ea5c16897e', ]) - ->addRule('buildStdout', [ - 'type' => self::TYPE_STRING, - 'description' => 'The stdout of the build.', - 'default' => '', - 'example' => '', - ]) - ->addRule('buildStderr', [ - 'type' => self::TYPE_STRING, - 'description' => 'The stderr of the build.', - 'default' => '', - 'example' => '', - ]) ->addRule('deploy', [ 'type' => self::TYPE_BOOLEAN, 'description' => 'Whether the deployment should be automatically deployed.', 'default' => false, 'example' => true, ]) + ->addRule('status', [ + 'type' => self::TYPE_STRING, + 'description' => 'The deployment status.', + 'default' => '', + 'example' => 'enabled', + ]) + ->addRule('buildStdout', [ + 'type' => self::TYPE_STRING, + 'description' => 'The build stdout.', + 'default' => '', + 'example' => 'enabled', + ]) + ->addRule('buildStderr', [ + 'type' => self::TYPE_STRING, + 'description' => 'The build stderr.', + 'default' => '', + 'example' => 'enabled', + ]) ; } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index d740e3942..f7a0275e9 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -331,38 +331,6 @@ class FunctionsCustomServerTest extends Scope return $data; } - /** - * @depends testCreateDeployment - */ - public function testListBuild(array $data):array - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/builds', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(1, $response['body']['sum']); - $this->assertEquals($response['body']['builds'][0]['status'], 'ready'); - - /** - * Check Queries work - */ - $responseQuery = $this->client->call(Client::METHOD_GET, '/builds?status=status.equal(\"ready\")', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals(200, $responseQuery['headers']['status-code']); - $this->assertEquals(1, $responseQuery['body']['sum']); - $this->assertEquals($responseQuery['body']['builds'][0]['status'], 'ready'); - - return $data; - } - /** * @depends testCreateDeployment */ diff --git a/tests/resources/docker/docker-compose.yml b/tests/resources/docker/docker-compose.yml index 76d62317a..395557372 100644 --- a/tests/resources/docker/docker-compose.yml +++ b/tests/resources/docker/docker-compose.yml @@ -266,6 +266,34 @@ services: - _APP_REDIS_PORT - _APP_SMTP_HOST - _APP_SMTP_PORT + + appwrite-worker-builds: + entrypoint: worker-builds + container_name: appwrite-worker-builds + build: + context: . + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - redis + - mariadb + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_EXECUTOR_SECRET appwrite-schedule: entrypoint: schedule