CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-14 08:44:49 +12:00
Code Issues Projects Releases Wiki Activity

feat: extract build commands to the builds worker

Browse source
This commit is contained in:
Christy Jacob 2022-02-17 18:51:36 +04:00
parent 039d993730
commit 1d7a2cfc71
5 changed files with 123 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
app/executor.php
View file

@ -18,6 +18,7 @@ use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Swoole\Request;
use Utopia\Swoole\Response;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Range as ValidatorRange;
use Utopia\Validator\Text;
@ -118,13 +119,14 @@ App::post('/v1/runtimes')
->param('runtimeId', '', new Text(62), 'Unique runtime ID.')
->param('source', '', new Text(0), 'Path to source files.')
->param('destination', '', new Text(0), 'Destination folder to store build files into.')
->param('vars', '', new Assoc(), 'Environment Variables required for the build')
->param('vars', [], new Assoc(), 'Environment Variables required for the build')
->param('commands', [], new ArrayList(new Text(0)), 'Commands required to build the container')
->param('runtime', '', new Text(128), 'Runtime for the cloud function')
->param('baseImage', '', new Text(128), 'Base image name of the runtime')
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
->action(function (string $runtimeId, string $source, string $destination, array $vars, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) {
->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, $orchestrationPool, $activeRuntimes, Response $response) {
$container = 'r-' . $runtimeId;
@ -173,7 +175,7 @@ App::post('/v1/runtimes')
$container = 'b-' . $runtimeId;
$vars = array_map(fn ($v) => strval($v), $vars);
$orchestration
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 0))
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1))
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
@ -207,61 +209,21 @@ App::post('/v1/runtimes')
/**
* Extract user code into build container
*/
$untarStdout = '';
$untarStderr = '';
$untarSuccess = $orchestration->execute(
$status = $orchestration->execute(
name: $container,
command: [
'sh',
'-c',
'mkdir -p /usr/code && cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && cd /usr/workspace/ && tar -zxf /usr/workspace/code.tar.gz -C /usr/code && rm /usr/workspace/code.tar.gz'
],
stdout: $untarStdout,
stderr: $untarStderr,
timeout: 60
);
if (!$untarSuccess) {
throw new Exception('Failed to extract tarfile ' . $untarStderr, 500);
}
/**
* Build code and install dependenices
*/
$buildSuccess = $orchestration->execute(
name: $container,
command: ['sh', '-c', 'cd /usr/local/src && ./build.sh'],
command: $commands,
stdout: $buildStdout,
stderr: $buildStderr,
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
timeout: App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)
);
if (!$buildSuccess) {
throw new Exception('Failed to build dependencies: ' . $buildStderr, 500);
}
/**
* Repackage code and save
*/
$compressStdout = '';
$compressStderr = '';
$compressSuccess = $orchestration->execute(
name: $container,
command: [
'tar', '-C', '/usr/code', '-czvf', '/usr/builds/code.tar.gz', './'
],
stdout: $compressStdout,
stderr: $compressStderr,
timeout: 60
);
if (!$compressSuccess) {
throw new Exception('Failed to compress built code: ' . $compressStderr, 500);
if (!$status) {
throw new Exception('Failed to build dependenices ' . $buildStderr, 500);
}
// Check if the build was successful by checking if file exists
if (!\file_exists($tmpBuild)) {
throw new Exception('Something went wrong during the build process');
throw new Exception('Something went wrong during the build process', 500);
}
/**
@ -287,8 +249,8 @@ App::post('/v1/runtimes')
$build = [
'outputPath' => $outputPath,
'status' => 'ready',
'stdout' => \utf8_encode(\mb_substr($buildStdout, -4096)),
'stderr' => \utf8_encode(\mb_substr($buildStderr, -4096)),
'stdout' => \utf8_encode($buildStdout),
'stderr' => \utf8_encode($buildStderr),
'startTime' => $buildStart,
'endTime' => $buildEnd,
'duration' => $buildEnd - $buildStart,
@ -332,9 +294,9 @@ App::post('/v1/runtimes')
$vars = array_map(fn ($v) => strval($v), $vars);
$orchestration
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'))
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'))
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', 1))
->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', 256))
->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 256));
$id = $orchestration->run(
image: $baseImage,
@ -463,7 +425,7 @@ App::post('/v1/execution')
->desc('Create an execution')
->param('runtimeId', '', new Text(62), 'The runtimeID to execute')
->param('path', '', new Text(0), 'Path containing the built files.', false)
->param('vars', '', new Assoc(), 'Environment variables required for the build', false)
->param('vars', [], new Assoc(), 'Environment variables required for the build', false)
->param('data', '', new Text(8192), 'Data to be forwarded to the function, this is user specified.', true)
->param('runtime', '', new Text(128), 'Runtime for the cloud function', false)
->param('entrypoint', '', new Text(256), 'Entrypoint of the code file')

13
app/workers/builds.php
View file

@ -119,7 +119,18 @@ class BuildsV1 extends Worker
source: $source,
vars: $vars,
runtime: $key,
baseImage: $baseImage
baseImage: $baseImage,
commands: [
'sh', '-c',
'mkdir -p /usr/code && \
cp /tmp/code.tar.gz /usr/workspace/code.tar.gz && \
cd /usr/workspace/ && \
tar -zxf /usr/workspace/code.tar.gz -C /usr/code && \
rm /usr/workspace/code.tar.gz && \
cd /usr/local/src && \
./build.sh && \
tar -C /usr/code -czf /usr/builds/code.tar.gz ./'
]
);
/** Update the build document */

2
composer.json
View file

@ -60,7 +60,7 @@
"utopia-php/storage": "0.5.*",
"utopia-php/websocket": "0.1.0",
"utopia-php/image": "0.5.*",
"utopia-php/orchestration": "0.3.*",
"utopia-php/orchestration": "0.4.*",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "5.0.1",
"dragonmantank/cron-expression": "3.1.0",

176
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "1a5d84f96eb76e59f7ad0ff7bcd4a8d8",
"content-hash": "e28c5946b102f637cc247b026f42ae0e",
"packages": [
{
"name": "adhocore/jwt",
@ -119,11 +119,11 @@
"source": {
"type": "git",
"url": "https://github.com/appwrite/runtimes.git",
"reference": "991a8674c75f78644b557c9723869c370b3290b5"
"reference": "ea44eb19af851a3fe4ecf73e789f374f70d778c5"
},
"require": {
"php": ">=8.0",
"utopia-php/orchestration": "0.3.x",
"utopia-php/orchestration": "0.4.*",
"utopia-php/system": "0.4.*"
},
"require-dev": {
@ -160,7 +160,7 @@
"php",
"runtimes"
],
"time": "2022-02-01T10:58:43+00:00"
"time": "2022-02-17T12:06:12+00:00"
},
{
"name": "chillerlan/php-qrcode",
@ -525,12 +525,12 @@
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\": "src/"
},
"files": [
"src/functions_include.php"
]
],
"psr-4": {
"GuzzleHttp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -1027,12 +1027,12 @@
}
},
"autoload": {
"psr-4": {
"MongoDB\\": "src/"
},
"files": [
"src/functions.php"
]
],
"psr-4": {
"MongoDB\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -1760,12 +1760,12 @@
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Php80\\": ""
},
"files": [
"bootstrap.php"
],
"psr-4": {
"Symfony\\Polyfill\\Php80\\": ""
},
"classmap": [
"Resources/stubs"
]
@ -2252,16 +2252,16 @@
},
{
"name": "utopia-php/framework",
"version": "0.19.5",
"version": "0.19.6",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/framework.git",
"reference": "1c28ba9a5b491cf7c90c535fefee5832c7133623"
"reference": "7d9b28365fb794001cb34dd028659452d4e71b7d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/1c28ba9a5b491cf7c90c535fefee5832c7133623",
"reference": "1c28ba9a5b491cf7c90c535fefee5832c7133623",
"url": "https://api.github.com/repos/utopia-php/framework/zipball/7d9b28365fb794001cb34dd028659452d4e71b7d",
"reference": "7d9b28365fb794001cb34dd028659452d4e71b7d",
"shasum": ""
},
"require": {
@ -2295,9 +2295,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/framework/issues",
"source": "https://github.com/utopia-php/framework/tree/0.19.5"
"source": "https://github.com/utopia-php/framework/tree/0.19.6"
},
"time": "2022-01-04T14:40:23+00:00"
"time": "2022-02-10T17:05:22+00:00"
},
{
"name": "utopia-php/image",
@ -2470,16 +2470,16 @@
},
{
"name": "utopia-php/orchestration",
"version": "0.3.0",
"version": "0.4.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/orchestration.git",
"reference": "5398361b39fdb7a15b561ebf243c4cb75a68432a"
"reference": "6f25a47e6d6b14540853b62db43c676aecf80519"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/5398361b39fdb7a15b561ebf243c4cb75a68432a",
"reference": "5398361b39fdb7a15b561ebf243c4cb75a68432a",
"url": "https://api.github.com/repos/utopia-php/orchestration/zipball/6f25a47e6d6b14540853b62db43c676aecf80519",
"reference": "6f25a47e6d6b14540853b62db43c676aecf80519",
"shasum": ""
},
"require": {
@ -2519,9 +2519,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/orchestration/issues",
"source": "https://github.com/utopia-php/orchestration/tree/0.3.0"
"source": "https://github.com/utopia-php/orchestration/tree/0.4.0"
},
"time": "2021-12-20T14:08:08+00:00"
"time": "2022-02-17T11:48:37+00:00"
},
{
"name": "utopia-php/preloader",
@ -2682,16 +2682,16 @@
},
{
"name": "utopia-php/swoole",
"version": "0.3.2",
"version": "0.3.3",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/swoole.git",
"reference": "2b714eddf77cd5eda1889219c9656d7c0a63ce73"
"reference": "8312df69233b5dcd3992de88f131f238002749de"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/swoole/zipball/2b714eddf77cd5eda1889219c9656d7c0a63ce73",
"reference": "2b714eddf77cd5eda1889219c9656d7c0a63ce73",
"url": "https://api.github.com/repos/utopia-php/swoole/zipball/8312df69233b5dcd3992de88f131f238002749de",
"reference": "8312df69233b5dcd3992de88f131f238002749de",
"shasum": ""
},
"require": {
@ -2732,9 +2732,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/swoole/issues",
"source": "https://github.com/utopia-php/swoole/tree/0.3.2"
"source": "https://github.com/utopia-php/swoole/tree/0.3.3"
},
"time": "2021-12-13T15:37:41+00:00"
"time": "2022-01-20T09:58:43+00:00"
},
{
"name": "utopia-php/system",
@ -3031,12 +3031,12 @@
}
},
"autoload": {
"psr-4": {
"Amp\\ByteStream\\": "lib"
},
"files": [
"lib/functions.php"
]
],
"psr-4": {
"Amp\\ByteStream\\": "lib"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -3197,23 +3197,23 @@
},
{
"name": "composer/semver",
"version": "3.2.7",
"version": "3.2.9",
"source": {
"type": "git",
"url": "https://github.com/composer/semver.git",
"reference": "deac27056b57e46faf136fae7b449eeaa71661ee"
"reference": "a951f614bd64dcd26137bc9b7b2637ddcfc57649"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/semver/zipball/deac27056b57e46faf136fae7b449eeaa71661ee",
"reference": "deac27056b57e46faf136fae7b449eeaa71661ee",
"url": "https://api.github.com/repos/composer/semver/zipball/a951f614bd64dcd26137bc9b7b2637ddcfc57649",
"reference": "a951f614bd64dcd26137bc9b7b2637ddcfc57649",
"shasum": ""
},
"require": {
"php": "^5.3.2 || ^7.0 || ^8.0"
},
"require-dev": {
"phpstan/phpstan": "^0.12.54",
"phpstan/phpstan": "^1.4",
"symfony/phpunit-bridge": "^4.2 || ^5"
},
"type": "library",
@ -3258,7 +3258,7 @@
"support": {
"irc": "irc://irc.freenode.org/composer",
"issues": "https://github.com/composer/semver/issues",
"source": "https://github.com/composer/semver/tree/3.2.7"
"source": "https://github.com/composer/semver/tree/3.2.9"
},
"funding": [
{
@ -3274,7 +3274,7 @@
"type": "tidelift"
}
],
"time": "2022-01-04T09:57:54+00:00"
"time": "2022-02-04T13:58:43+00:00"
},
{
"name": "composer/xdebug-handler",
@ -3704,12 +3704,12 @@
},
"type": "library",
"autoload": {
"psr-4": {
"DeepCopy\\": "src/DeepCopy/"
},
"files": [
"src/DeepCopy/deep_copy.php"
]
],
"psr-4": {
"DeepCopy\\": "src/DeepCopy/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -3957,16 +3957,16 @@
},
{
"name": "phar-io/version",
"version": "3.1.0",
"version": "3.1.1",
"source": {
"type": "git",
"url": "https://github.com/phar-io/version.git",
"reference": "bae7c545bef187884426f042434e561ab1ddb182"
"reference": "15a90844ad40f127afd244c0cad228de2a80052a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phar-io/version/zipball/bae7c545bef187884426f042434e561ab1ddb182",
"reference": "bae7c545bef187884426f042434e561ab1ddb182",
"url": "https://api.github.com/repos/phar-io/version/zipball/15a90844ad40f127afd244c0cad228de2a80052a",
"reference": "15a90844ad40f127afd244c0cad228de2a80052a",
"shasum": ""
},
"require": {
@ -4002,9 +4002,9 @@
"description": "Library for handling version information and constraints",
"support": {
"issues": "https://github.com/phar-io/version/issues",
"source": "https://github.com/phar-io/version/tree/3.1.0"
"source": "https://github.com/phar-io/version/tree/3.1.1"
},
"time": "2021-02-23T14:00:09+00:00"
"time": "2022-02-07T21:56:48+00:00"
},
{
"name": "phpdocumentor/reflection-common",
@ -4613,11 +4613,11 @@
}
},
"autoload": {
"classmap": [
"src/"
],
"files": [
"src/Framework/Assert/Functions.php"
],
"classmap": [
"src/"
]
},
"notification-url": "https://packagist.org/downloads/",
@ -5213,16 +5213,16 @@
},
{
"name": "sebastian/global-state",
"version": "5.0.3",
"version": "5.0.5",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/global-state.git",
"reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49"
"reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/23bd5951f7ff26f12d4e3242864df3e08dec4e49",
"reference": "23bd5951f7ff26f12d4e3242864df3e08dec4e49",
"url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2",
"reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2",
"shasum": ""
},
"require": {
@ -5265,7 +5265,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/global-state/issues",
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.3"
"source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5"
},
"funding": [
{
@ -5273,7 +5273,7 @@
"type": "github"
}
],
"time": "2021-06-11T13:31:12+00:00"
"time": "2022-02-14T08:28:10+00:00"
},
{
"name": "sebastian/lines-of-code",
@ -5839,12 +5839,12 @@
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Intl\\Grapheme\\": ""
},
"files": [
"bootstrap.php"
]
],
"psr-4": {
"Symfony\\Polyfill\\Intl\\Grapheme\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -5920,12 +5920,12 @@
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Intl\\Normalizer\\": ""
},
"files": [
"bootstrap.php"
],
"psr-4": {
"Symfony\\Polyfill\\Intl\\Normalizer\\": ""
},
"classmap": [
"Resources/stubs"
]
@ -6007,12 +6007,12 @@
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Mbstring\\": ""
},
"files": [
"bootstrap.php"
]
],
"psr-4": {
"Symfony\\Polyfill\\Mbstring\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -6084,12 +6084,12 @@
}
},
"autoload": {
"psr-4": {
"Symfony\\Polyfill\\Php72\\": ""
},
"files": [
"bootstrap.php"
]
],
"psr-4": {
"Symfony\\Polyfill\\Php72\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
@ -6400,16 +6400,16 @@
},
{
"name": "twig/twig",
"version": "v2.14.10",
"version": "v2.14.11",
"source": {
"type": "git",
"url": "https://github.com/twigphp/Twig.git",
"reference": "95fb194cd4dd6ac373a27af2bde2bad5d3f27aba"
"reference": "66baa66f29ee30e487e05f1679903e36eb01d727"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/twigphp/Twig/zipball/95fb194cd4dd6ac373a27af2bde2bad5d3f27aba",
"reference": "95fb194cd4dd6ac373a27af2bde2bad5d3f27aba",
"url": "https://api.github.com/repos/twigphp/Twig/zipball/66baa66f29ee30e487e05f1679903e36eb01d727",
"reference": "66baa66f29ee30e487e05f1679903e36eb01d727",
"shasum": ""
},
"require": {
@ -6464,7 +6464,7 @@
],
"support": {
"issues": "https://github.com/twigphp/Twig/issues",
"source": "https://github.com/twigphp/Twig/tree/v2.14.10"
"source": "https://github.com/twigphp/Twig/tree/v2.14.11"
},
"funding": [
{
@ -6476,7 +6476,7 @@
"type": "tidelift"
}
],
"time": "2022-01-03T21:13:26+00:00"
"time": "2022-02-04T06:57:25+00:00"
},
{
"name": "vimeo/psalm",
@ -6555,13 +6555,13 @@
}
},
"autoload": {
"psr-4": {
"Psalm\\": "src/Psalm/"
},
"files": [
"src/functions.php",
"src/spl_object_id.php"
]
],
"psr-4": {
"Psalm\\": "src/Psalm/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [

8
src/Executor/Executor.php
View file

@ -37,8 +37,9 @@ class Executor
string $source,
array $vars,
string $runtime,
string $baseImage)
{
string $baseImage,
array $commands
) {
$route = "/runtimes";
$headers = [
'content-type' => 'application/json',
@ -51,7 +52,8 @@ class Executor
'destination' => APP_STORAGE_BUILDS . "/app-$projectId",
'vars' => $vars,
'runtime' => $runtime,
'baseImage' => $baseImage
'baseImage' => $baseImage,
'commands' => $commands
];
$response = $this->call(self::METHOD_POST, $route, $headers, $params, true, 30);