diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7152f0c90a..bf743cda87 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -433,7 +433,7 @@ App::delete('/v1/functions/:functionId') // Request executor to delete tag containers $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/function"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'functionId' => $functionId @@ -590,7 +590,7 @@ App::post('/v1/functions/:functionId/tags') $function = $dbForProject->getDocument('functions', $functionId); $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/tag"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/tag"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'functionId' => $function->getId(), @@ -769,7 +769,7 @@ App::delete('/v1/functions/:functionId/tags/:tagId') // Request executor to delete tag containers $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/tag"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/cleanup/tag"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'tagId' => $tagId @@ -927,7 +927,7 @@ App::post('/v1/functions/:functionId/executions') // Directly execute function. $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'trigger' => 'http', @@ -1159,7 +1159,7 @@ App::post('/v1/builds/:buildId') // Retry build $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/{$buildId}"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/build/{$buildId}"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); \curl_setopt($ch, CURLOPT_TIMEOUT, 900); diff --git a/app/executor.php b/app/executor.php index 83a8b536d1..ef8536a44a 100644 --- a/app/executor.php +++ b/app/executor.php @@ -1,40 +1,40 @@ get(); + try { + $orchestration = $orchestrationPool->get(); - Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...'); + Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...'); - $response = $orchestration->pull($runtime['image']); + $response = $orchestration->pull($runtime['image']); - if ($response) { - Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); - } else { - Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); + if ($response) { + Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!"); + } else { + Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!"); + } + } catch (\Throwable $th) { + } finally { + $orchestrationPool->put($orchestration); } - - $orchestrationPool->put($orchestration); }); } }); @@ -123,11 +127,15 @@ try { $activeFunctions->create(); Co\run(function () use ($orchestrationPool, $activeFunctions) { - $orchestration = $orchestrationPool->get(); - $executionStart = \microtime(true); + try { + $orchestration = $orchestrationPool->get(); + $executionStart = \microtime(true); + $residueList = $orchestration->list(['label' => 'appwrite-type=function']); + } catch (\Throwable $th) { + } finally { + $orchestrationPool->put($orchestration); + } - $residueList = $orchestration->list(['label' => 'appwrite-type=function']); - $orchestrationPool->put($orchestration); foreach ($residueList as $value) { go(fn () => $activeFunctions->set($value->getName(), [ @@ -311,9 +319,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI } catch (\Throwable $th) { $orchestrationPool->put($orchestration); throw $th; - } finally { - $orchestrationPool->put($orchestration); } + $orchestrationPool->put($orchestration); }; function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array @@ -404,7 +411,7 @@ function execute(string $trigger, string $projectId, string $executionId, string $database->createDocument('builds', new Document([ '$id' => $buildId, '$read' => ($userId !== '') ? ['user:' . $userId] : [], - '$write' => ['role:all'], + '$write' => [], 'dateCreated' => time(), 'status' => 'processing', 'outputPath' => '', @@ -443,7 +450,7 @@ function execute(string $trigger, string $projectId, string $executionId, string } try { - if (!$activeFunctions->exists($container)) { // Create contianer if not ready + if (!$activeFunctions->exists($container)) { // Create container if not ready createRuntimeServer($functionId, $projectId, $tag->getId(), $database); } else if ($activeFunctions->get($container)['status'] === 'Down') { sleep(1); @@ -651,6 +658,261 @@ function execute(string $trigger, string $projectId, string $executionId, string ]; }; +function runBuildStage(string $buildId, string $projectID): Document +{ + global $runtimes; + global $orchestrationPool; + global $register; + + /** @var Orchestration $orchestration */ + $orchestration = $orchestrationPool->get(); + + $buildStdout = ''; + $buildStderr = ''; + + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + $cache = new Cache(new RedisCache($redis)); + + $database = new Database(new MariaDB($db), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace('_project_' . $projectID); + + // Check if build has already been run + $build = $database->getDocument('builds', $buildId); + + try { + // If we already have a built package ready there is no need to rebuild. + if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) { + return $build; + } + + // Update Tag Status + $build->setAttribute('status', 'building'); + + $database->updateDocument('builds', $build->getId(), $build); + + // Check if runtime is active + $runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null; + + if (\is_null($runtime)) { + throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported'); + } + + // Grab Tag Files + $tagPath = $build->getAttribute('source', ''); + $sourceType = $build->getAttribute('sourceType', ''); + + $device = Storage::getDevice('builds'); + + $tagPathTarget = '/tmp/project-' . $projectID . '/' . $build->getId() . '/code.tar.gz'; + $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME); + + $container = 'build-stage-' . $build->getId(); + + // Perform various checks + if (!\file_exists($tagPathTargetDir)) { + if (@\mkdir($tagPathTargetDir, 0777, true)) { + \chmod($tagPathTargetDir, 0777); + } else { + throw new Exception('Can\'t create directory ' . $tagPathTargetDir); + } + } + + if (!\file_exists($tagPathTarget)) { + if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { + if (!\copy($tagPath, $tagPathTarget)) { + throw new Exception('Can\'t create temporary code file ' . $tagPathTarget); + } + } else { + $buffer = $device->read($tagPath); + \file_put_contents($tagPathTarget, $buffer); + } + } + + if (!$device->exists($tagPath)) { + throw new Exception('Code is not readable: ' . $build->getAttribute('source', '')); + } + + $vars = $build->getAttribute('vars', []); + + // Start tracking time + $buildStart = \microtime(true); + $time = \time(); + + $orchestration + ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) + ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) + ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); + + $vars = array_map(fn ($v) => strval($v), $vars); + $path = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode'; + + if (!\file_exists($path)) { + if (@\mkdir($path, 0777, true)) { + \chmod($path, 0777); + } else { + throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode'); + } + } + + // Launch build container + $id = $orchestration->run( + image: $runtime['base'], + name: $container, + vars: $vars, + workdir: '/usr/code', + labels: [ + 'appwrite-type' => 'function', + 'appwrite-created' => strval($time), + 'appwrite-runtime' => $build->getAttribute('runtime', ''), + 'appwrite-project' => $projectID, + 'appwrite-build' => $build->getId(), + ], + command: [ + 'tail', + '-f', + '/dev/null' + ], + hostname: $container, + mountFolder: $tagPathTargetDir, + volumes: [ + '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode' . ':/usr/builtCode:rw' + ] + ); + + if (empty($id)) { + throw new Exception('Failed to start build container'); + } + + // Extract user code into build container + $untarStdout = ''; + $untarStderr = ''; + + $untarSuccess = $orchestration->execute( + name: $container, + command: [ + 'sh', + '-c', + 'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz' + ], + stdout: $untarStdout, + stderr: $untarStderr, + timeout: 60 + ); + + if (!$untarSuccess) { + throw new Exception('Failed to extract tar: ' . $untarStderr); + } + + // Build Code / Install Dependencies + $buildSuccess = $orchestration->execute( + name: $container, + command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'], + stdout: $buildStdout, + stderr: $buildStderr, + timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900) + ); + + if (!$buildSuccess) { + throw new Exception('Failed to build dependencies: ' . $buildStderr); + } + + // Repackage Code and Save. + $compressStdout = ''; + $compressStderr = ''; + + $builtCodePath = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode/code.tar.gz'; + + $compressSuccess = $orchestration->execute( + name: $container, + command: [ + 'tar', '-C', '/usr/code', '-czvf', '/usr/builtCode/code.tar.gz', './' + ], + stdout: $compressStdout, + stderr: $compressStderr, + timeout: 60 + ); + + if (!$compressSuccess) { + throw new Exception('Failed to compress built code: ' . $compressStderr); + } + + // Remove Container + $orchestration->remove($id, true); + + // Check if the build was successful by checking if file exists + if (!\file_exists($builtCodePath)) { + throw new Exception('Something went wrong during the build process.'); + } + + // Upload new code + $device = Storage::getDevice('builds'); + + $path = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + + if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists + if (@\mkdir(\dirname($path), 0777, true)) { + \chmod(\dirname($path), 0777); + } else { + throw new Exception('Can\'t create directory: ' . \dirname($path)); + } + } + + if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { + if (!$device->move($builtCodePath, $path)) { + throw new Exception('Failed to upload built code upload to storage', 500); + } + } else { + if (!$device->upload($builtCodePath, $path)) { + throw new Exception('Failed to upload built code upload to storage', 500); + } + } + + if ($buildStdout == '') { + $buildStdout = 'Build Successful!'; + } + + $build + ->setAttribute('outputPath', $path) + ->setAttribute('status', 'ready') + ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) + ->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096))) + ->setAttribute('time', $time); + + // Update build with built code attribute + $build = $database->updateDocument('builds', $buildId, $build); + + $buildEnd = \microtime(true); + + Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds'); + } catch (Exception $e) { + $build + ->setAttribute('status', 'failed') + ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) + ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))); + + $build = $database->updateDocument('builds', $buildId, $build); + + // also remove the container if it exists + if (isset($id)) { + $orchestration->remove($id, true); + } + + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); + + throw new Exception('Build failed: ' . $e->getMessage()); + } + + $orchestrationPool->put($orchestration); + + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); + + return $build; +} + App::post('/v1/execute') // Define Route ->desc('Execute a function') ->param('trigger', '', new Text(1024)) @@ -682,7 +944,6 @@ App::post('/v1/execute') // Define Route } ); - // Cleanup Endpoints used internally by appwrite when a function or tag gets deleted to also clean up their containers App::post('/v1/cleanup/function') ->param('functionId', '', new UID()) @@ -857,45 +1118,48 @@ App::post('/v1/tag') // Build Code go(function () use ($projectID, $tagId, $buildId, $functionId, $function, $register) { - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - $cache = new Cache(new RedisCache($redis)); + try { + $db = $register->get('dbPool')->get(); + $redis = $register->get('redisPool')->get(); + $cache = new Cache(new RedisCache($redis)); - $dbForProject = new Database(new MariaDB($db), $cache); - $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForProject->setNamespace('_project_' . $projectID); - // Build Code - runBuildStage($buildId, $projectID); + $dbForProject = new Database(new MariaDB($db), $cache); + $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $dbForProject->setNamespace('_project_' . $projectID); + // Build Code + runBuildStage($buildId, $projectID); - // Update the schedule - $schedule = $function->getAttribute('schedule', ''); - $cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null; - $next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; + // Update the schedule + $schedule = $function->getAttribute('schedule', ''); + $cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null; + $next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0; - // Grab tag - $tag = $dbForProject->getDocument('tags', $tagId); + // Grab tag + $tag = $dbForProject->getDocument('tags', $tagId); - // Grab build - $build = $dbForProject->getDocument('builds', $buildId); + // Grab build + $build = $dbForProject->getDocument('builds', $buildId); - // If the build failed, it won't be possible to deploy - if ($build->getAttribute('status') !== 'ready') { - return; + // If the build failed, it won't be possible to deploy + if ($build->getAttribute('status') !== 'ready') { + return; + } + + if ($tag->getAttribute('automaticDeploy') === true) { + // Update the function document setting the tag as the active one + $function + ->setAttribute('tag', $tag->getId()) + ->setAttribute('scheduleNext', (int)$next); + $function = $dbForProject->updateDocument('functions', $function->getId(), $function); + } + + // Deploy Runtime Server + createRuntimeServer($functionId, $projectID, $tagId, $dbForProject); + } catch (\Throwable $th) { + } finally { + $register->get('dbPool')->put($db); + $register->get('redisPool')->put($redis); } - - if ($tag->getAttribute('deploy') === true) { - // Update the function document setting the tag as the active one - $function - ->setAttribute('tag', $tag->getId()) - ->setAttribute('scheduleNext', (int)$next); - $function = $dbForProject->updateDocument('functions', $function->getId(), $function); - } - - // Deploy Runtime Server - createRuntimeServer($functionId, $projectID, $tagId, $dbForProject); - - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); }); if (false === $function) { @@ -961,265 +1225,9 @@ App::post('/v1/build/:buildId') // Start a Build } }); -function runBuildStage(string $buildId, string $projectID): Document -{ - global $runtimes; - global $orchestrationPool; - global $register; - - /** @var Orchestration $orchestration */ - $orchestration = $orchestrationPool->get(); - - $buildStdout = ''; - $buildStderr = ''; - - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - $cache = new Cache(new RedisCache($redis)); - - $database = new Database(new MariaDB($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_project_' . $projectID); - - // Check if build has already been run - $build = $database->getDocument('builds', $buildId); - - try { - // If we already have a built package ready there is no need to rebuild. - if ($build->getAttribute('status') === 'ready' && \file_exists($build->getAttribute('outputPath'))) { - return $build; - } - - // Update Tag Status - $build->setAttribute('status', 'building'); - - $database->updateDocument('builds', $build->getId(), $build); - - // Check if runtime is active - $runtime = $runtimes[$build->getAttribute('runtime', '')] ?? null; - - if (\is_null($runtime)) { - throw new Exception('Runtime "' . $build->getAttribute('runtime', '') . '" is not supported'); - } - - // Grab Tag Files - $tagPath = $build->getAttribute('source', ''); - $sourceType = $build->getAttribute('sourceType', ''); - - $device = Storage::getDevice('builds'); - - $tagPathTarget = '/tmp/project-' . $projectID . '/' . $build->getId() . '/code.tar.gz'; - $tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME); - - $container = 'build-stage-' . $build->getId(); - - // Perform various checks - if (!\file_exists($tagPathTargetDir)) { - if (@\mkdir($tagPathTargetDir, 0777, true)) { - \chmod($tagPathTargetDir, 0777); - } else { - throw new Exception('Can\'t create directory ' . $tagPathTargetDir); - } - } - - if (!\file_exists($tagPathTarget)) { - if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { - if (!\copy($tagPath, $tagPathTarget)) { - throw new Exception('Can\'t create temporary code file ' . $tagPathTarget); - } - } else { - $buffer = $device->read($tagPath); - \file_put_contents($tagPathTarget, $buffer); - } - } - - if (!$device->exists($tagPath)) { - throw new Exception('Code is not readable: ' . $build->getAttribute('source', '')); - } - - $vars = $build->getAttribute('vars', []); - - // Start tracking time - $buildStart = \microtime(true); - $time = \time(); - - $orchestration - ->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0)) - ->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256)) - ->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256)); - - $vars = array_map(fn ($v) => strval($v), $vars); - $path = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode'; - - if (!\file_exists($path)) { - if (@\mkdir($path, 0777, true)) { - \chmod($path, 0777); - } else { - throw new Exception('Can\'t create directory /tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode'); - } - }; - - // Launch build container - $id = $orchestration->run( - image: $runtime['base'], - name: $container, - vars: $vars, - workdir: '/usr/code', - labels: [ - 'appwrite-type' => 'function', - 'appwrite-created' => strval($time), - 'appwrite-runtime' => $build->getAttribute('runtime', ''), - 'appwrite-project' => $projectID, - 'appwrite-build' => $build->getId(), - ], - command: [ - 'tail', - '-f', - '/dev/null' - ], - hostname: $container, - mountFolder: $tagPathTargetDir, - volumes: [ - '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode' . ':/usr/builtCode:rw' - ] - ); - - if (empty($id)) { - throw new Exception('Failed to start build container'); - } - - // Extract user code into build container - $untarStdout = ''; - $untarStderr = ''; - - $untarSuccess = $orchestration->execute( - name: $container, - command: [ - 'sh', - '-c', - 'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz' - ], - stdout: $untarStdout, - stderr: $untarStderr, - timeout: 60 - ); - - if (!$untarSuccess) { - throw new Exception('Failed to extract tar: ' . $untarStderr); - } - - // Build Code / Install Dependencies - $buildSuccess = $orchestration->execute( - name: $container, - command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'], - stdout: $buildStdout, - stderr: $buildStderr, - timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900) - ); - - if (!$buildSuccess) { - throw new Exception('Failed to build dependencies: ' . $buildStderr); - } - - // Repackage Code and Save. - $compressStdout = ''; - $compressStderr = ''; - - $builtCodePath = '/tmp/project-' . $projectID . '/' . $build->getId() . '/builtCode/code.tar.gz'; - - $compressSuccess = $orchestration->execute( - name: $container, - command: [ - 'tar', '-C', '/usr/code', '-czvf', '/usr/builtCode/code.tar.gz', './' - ], - stdout: $compressStdout, - stderr: $compressStderr, - timeout: 60 - ); - - if (!$compressSuccess) { - throw new Exception('Failed to compress built code: ' . $compressStderr); - } - - // Remove Container - $orchestration->remove($id, true); - - // Check if the build was successful by checking if file exists - if (!\file_exists($builtCodePath)) { - throw new Exception('Something went wrong during the build process.'); - } - - // Upload new code - $device = Storage::getDevice('builds'); - - $path = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - - if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists - if (@\mkdir(\dirname($path), 0777, true)) { - \chmod(\dirname($path), 0777); - } else { - throw new Exception('Can\'t create directory: ' . \dirname($path)); - } - } - - if (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) { - if (!$device->move($builtCodePath, $path)) { - throw new Exception('Failed to upload built code upload to storage', 500); - } - } else { - if (!$device->upload($builtCodePath, $path)) { - throw new Exception('Failed to upload built code upload to storage', 500); - } - } - - if ($buildStdout == '') { - $buildStdout = 'Build Successful!'; - } - - $build - ->setAttribute('outputPath', $path) - ->setAttribute('status', 'ready') - ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) - ->setAttribute('stderr', \utf8_encode(\mb_substr($buildStderr, -4096))) - ->setAttribute('time', $time); - - // Update build with built code attribute - $build = $database->updateDocument('builds', $buildId, $build); - - $buildEnd = \microtime(true); - - Console::info('Build Stage Ran in ' . ($buildEnd - $buildStart) . ' seconds'); - } catch (Exception $e) { - $build - ->setAttribute('status', 'failed') - ->setAttribute('stdout', \utf8_encode(\mb_substr($buildStdout, -4096))) - ->setAttribute('stderr', \utf8_encode(\mb_substr($e->getMessage(), -4096))); - - $build = $database->updateDocument('builds', $buildId, $build); - - // also remove the container if it exists - if (isset($id)) { - $orchestration->remove($id, true); - } - - $orchestrationPool->put($orchestration); - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - - throw new Exception('Build failed: ' . $e->getMessage()); - } finally { - $orchestrationPool->put($orchestration); - - $register->get('dbPool')->put($db); - $register->get('redisPool')->put($redis); - } - - return $build; -} - App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode -$http = new Server("0.0.0.0", 8080); +$http = new Server("0.0.0.0", 80); function handleShutdown() { @@ -1399,4 +1407,4 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo } }); -$http->start(); +$http->start(); \ No newline at end of file diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index b4b4b492f4..a4e8aeb0c0 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -194,10 +194,10 @@ services: - _APP_USAGE_STATS - _APP_STATSD_HOST - _APP_STATSD_PORT - - DOCKERHUB_PULL_USERNAME - - DOCKERHUB_PULL_PASSWORD - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG + - DOCKERHUB_PULL_USERNAME + - DOCKERHUB_PULL_PASSWORD appwrite-worker-database: image: /: @@ -324,13 +324,9 @@ services: depends_on: - redis - mariadb - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - appwrite-functions:/storage/functions:rw - - /tmp:/tmp:rw + - appwrite-executor environment: - _APP_ENV - - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -341,13 +337,7 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CONTAINERS - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY - - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_SECRET - - _APP_FUNCTIONS_RUNTIMES - _APP_USAGE_STATS appwrite-executor: diff --git a/app/workers/functions.php b/app/workers/functions.php index 537e4ddd93..a96a1fa7f2 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -1,10 +1,6 @@ list(['label' => 'appwrite-type=function']); -/** @var Container[] $list */ -$list = []; - -foreach ($response as $value) { - $list[$value->getName()] = $value; -} - -$executionEnd = \microtime(true); - -Console::info(count($list) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds'); - -/** - * 1. Get event args - DONE - * 2. Unpackage code in the isolated container - DONE - * 3. Execute in container with timeout - * + messure execution time - DONE - * + pass env vars - DONE - * + pass one-time api key - * 4. Update execution status - DONE - * 5. Update execution stdout & stderr - DONE - * 6. Trigger audit log - DONE - * 7. Trigger usage log - DONE - */ - -// TODO avoid scheduled execution if delay is bigger than X offest - class FunctionsV1 extends Worker { public array $args = []; @@ -266,7 +224,7 @@ class FunctionsV1 extends Worker public function execute(string $trigger, string $projectId, string $executionId, Database $database, Document $function, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): void { $ch = \curl_init(); - \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/execute"); + \curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor/v1/execute"); \curl_setopt($ch, CURLOPT_POST, true); \curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([ 'trigger' => $trigger, @@ -299,67 +257,6 @@ class FunctionsV1 extends Worker \curl_close($ch); } - /** - * Cleanup any hanging containers above the allowed max containers. - * - * @return void - */ - public function cleanup(): void - { - /** @var Container[] $list */ - global $list; - /** @var Orchestration $orchestration */ - global $orchestration; - - Console::success(count($list) . ' running containers counted'); - - $max = (int) App::getEnv('_APP_FUNCTIONS_CONTAINERS'); - - if (\count($list) > $max) { - Console::info('Starting containers cleanup'); - - \uasort($list, function (Container $item1, Container $item2) { - return (int)($item1->getLabels['appwrite-created'] ?? 0) <=> (int)($item2->getLabels['appwrite-created'] ?? 0); - }); - - while (\count($list) > $max) { - $first = \array_shift($list); - - try { - $orchestration->remove($first->getName(), true); - Console::info('Removed container: ' . $first->getName()); - } catch (Exception $e) { - Console::error('Failed to remove container: ' . $e); - } - } - } - } - - /** - * Filter ENV vars - * - * @param string $string - * - * @return string - */ - public function filterEnvKey(string $string): string - { - if (empty($this->allowed)) { - $this->allowed = array_fill_keys(\str_split('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_'), true); - } - - $string = \str_split($string); - $output = ''; - - foreach ($string as $char) { - if (\array_key_exists($char, $this->allowed)) { - $output .= $char; - } - } - - return $output; - } - public function shutdown(): void { } diff --git a/docker-compose.yml b/docker-compose.yml index 584c65fe89..4f7afc3d61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,6 @@ services: - traefik.http.routers.appwrite_api_https.service=appwrite_api - traefik.http.routers.appwrite_api_https.tls=true volumes: - - /var/run/docker.sock:/var/run/docker.sock - appwrite-uploads:/storage/uploads:rw - appwrite-cache:/storage/cache:rw - appwrite-config:/storage/config:rw @@ -326,17 +325,14 @@ services: networks: - appwrite volumes: - - /var/run/docker.sock:/var/run/docker.sock - - appwrite-functions:/storage/functions:rw - - /tmp:/tmp:rw - ./app:/usr/src/code/app - ./src:/usr/src/code/src depends_on: - redis - mariadb + - appwrite-executor environment: - _APP_ENV - - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -347,12 +343,6 @@ services: - _APP_DB_USER - _APP_DB_PASS - _APP_FUNCTIONS_TIMEOUT - - _APP_FUNCTIONS_BUILD_TIMEOUT - - _APP_FUNCTIONS_CONTAINERS - - _APP_FUNCTIONS_RUNTIMES - - _APP_FUNCTIONS_CPUS - - _APP_FUNCTIONS_MEMORY - - _APP_FUNCTIONS_MEMORY_SWAP - _APP_EXECUTOR_SECRET - _APP_USAGE_STATS - DOCKERHUB_PULL_USERNAME