CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-04 14:10:33 +12:00
Code Issues Projects Releases Wiki Activity

Add build & install commands

Browse source
This commit is contained in:
Matej Bačo 2023-03-28 15:21:42 +02:00
parent 964428944a
commit 5389afc80e
15 changed files with 214 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
app/config/collections.php
View file

@ -2411,6 +2411,28 @@ $collections = [
'default' => null,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('buildCommand'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('installCommand'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'filters' => [],
],
],
'indexes' => [
[
@ -2462,6 +2484,27 @@ $collections = [
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_entrypoint'),
'type' => Database::INDEX_KEY,
'attributes' => ['entrypoint'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_build_command'),
'type' => Database::INDEX_KEY,
'attributes' => ['buildCommand'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_install_command'),
'type' => Database::INDEX_KEY,
'attributes' => ['installCommand'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
],
],
@ -2536,6 +2579,28 @@ $collections = [
'default' => null,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('buildCommand'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'filters' => [],
],
[
'array' => false,
'$id' => ID::custom('installCommand'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
'signed' => true,
'required' => false,
'default' => null,
'filters' => [],
],
[
'$id' => ID::custom('path'),
'type' => Database::VAR_STRING,
@ -2640,7 +2705,21 @@ $collections = [
'$id' => ID::custom('_key_entrypoint'),
'type' => Database::INDEX_KEY,
'attributes' => ['entrypoint'],
'lengths' => [768],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_build_command'),
'type' => Database::INDEX_KEY,
'attributes' => ['buildCommand'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_install_command'),
'type' => Database::INDEX_KEY,
'attributes' => ['installCommand'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
@ -3009,6 +3088,27 @@ $collections = [
'lengths' => [128],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_agent'),
'type' => Database::INDEX_KEY,
'attributes' => ['agent'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_path'),
'type' => Database::INDEX_KEY,
'attributes' => ['path'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_method'),
'type' => Database::INDEX_KEY,
'attributes' => ['method'],
'lengths' => [128],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_status'),
'type' => Database::INDEX_KEY,

2
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

71
app/controllers/api/functions.php
View file

@ -70,14 +70,16 @@ App::post('/v1/functions')
->param('timeout', 15, new Range(1, (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)), 'Function maximum execution time in seconds.', true)
->param('enabled', true, new Boolean(), 'Is function enabled?', true)
->param('logging', true, new Boolean(), 'Do executions get logged?', true)
->param('entrypoint', null, new Text('1028'), 'Entrypoint File.', true)
->param('entrypoint', '', new Text('1028'), 'Entrypoint File.', true)
->param('buildCommand', '', new Text('1028'), 'Build Command.', true)
->param('installCommand', '', new Text('1028'), 'Install Command.', true)
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('user')
->inject('events')
->inject('dbForConsole')
->action(function (string $functionId, string $name, array $execute, string $runtime, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, ?string $entrypoint, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) {
->action(function (string $functionId, string $name, array $execute, string $runtime, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $buildCommand, string $installCommand, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) {
$functionId = ($functionId == 'unique()') ? ID::unique() : $functionId;
$function = $dbForProject->createDocument('functions', new Document([
@ -94,12 +96,14 @@ App::post('/v1/functions')
'scheduleUpdatedAt' => DateTime::now(),
'timeout' => $timeout,
'entrypoint' => $entrypoint,
'buildCommand' => $buildCommand,
'installCommand' => $installCommand,
'search' => implode(' ', [$functionId, $name, $runtime]),
'version' => 'v3'
]));
$schedule = Authorization::skip(
fn() => $dbForConsole->createDocument('schedules', new Document([
fn () => $dbForConsole->createDocument('schedules', new Document([
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $function->getId(),
@ -117,7 +121,7 @@ App::post('/v1/functions')
$domain = "{$routeSubdomain}.{$functionsDomain}";
$rule = Authorization::skip(
fn() => $dbForConsole->createDocument('rules', new Document([
fn () => $dbForConsole->createDocument('rules', new Document([
'$id' => $ruleId,
'projectId' => $project->getId(),
'projectInternalId' => $project->getInternalId(),
@ -127,7 +131,7 @@ App::post('/v1/functions')
'resourceInternalId' => $function->getInternalId(),
'status' => 'verified',
'certificateId' => '',
'search' => implode(' ', [ $domain, $ruleId, $function->getId(), 'function' ]),
'search' => implode(' ', [$domain, $ruleId, $function->getId(), 'function']),
]))
);
@ -516,14 +520,16 @@ App::put('/v1/functions/:functionId')
->param('timeout', 15, new Range(1, (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)), 'Maximum execution time in seconds.', true)
->param('enabled', true, new Boolean(), 'Is function enabled?', true)
->param('logging', true, new Boolean(), 'Do executions get logged?', true)
->param('entrypoint', null, new Text('1028'), 'Entrypoint File.', true)
->param('entrypoint', '', new Text('1028'), 'Entrypoint File.', true)
->param('buildCommand', '', new Text('1028'), 'Build Command.', true)
->param('installCommand', '', new Text('1028'), 'Install Command.', true)
->inject('response')
->inject('dbForProject')
->inject('project')
->inject('user')
->inject('events')
->inject('dbForConsole')
->action(function (string $functionId, string $name, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, ?string $entrypoint, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) {
->action(function (string $functionId, string $name, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $buildCommand, string $installCommand, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -543,6 +549,8 @@ App::put('/v1/functions/:functionId')
'enabled' => $enabled,
'logging' => $logging,
'entrypoint' => $entrypoint,
'buildCommand' => $buildCommand,
'installCommand' => $installCommand,
'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]),
])));
@ -670,8 +678,7 @@ App::delete('/v1/functions/:functionId')
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('active', false)
;
->setAttribute('active', false);
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
@ -704,6 +711,8 @@ App::post('/v1/functions/:functionId/deployments')
->param('entrypoint', null, new Text('1028'), 'Entrypoint File.', false)
->param('code', [], new File(), 'Gzip file with your code package. When used with the Appwrite CLI, pass the path to your code directory, and the CLI will automatically package your code. Use a path that is within the current directory.', false)
->param('activate', false, new Boolean(true), 'Automatically activate the deployment when it is finished building.', false)
->param('buildCommand', null, new Text('1028'), 'Build Command.', false)
->param('installCommand', null, new Text('1028'), 'Install Command.', false)
->inject('request')
->inject('response')
->inject('dbForProject')
@ -712,19 +721,26 @@ App::post('/v1/functions/:functionId/deployments')
->inject('deviceFunctions')
->inject('deviceLocal')
->inject('dbForConsole')
->action(function (string $functionId, ?string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal, Database $dbForConsole) {
->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, string $buildCommand, string $installCommand, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
if ($entrypoint === null || empty($entrypoint)) {
$entrypoint = $function->getAttribute('entrypoint', null);
if ($entrypoint === 'fromFunction()') {
$entrypoint = $function->getAttribute('entrypoint', '');
}
if ($entrypoint === null || empty($entrypoint)) {
if ($buildCommand === 'fromFunction()') {
$buildCommand = $function->getAttribute('buildCommand', '');
}
if ($installCommand === 'fromFunction()') {
$installCommand = $function->getAttribute('installCommand', '');
}
if (empty($entrypoint)) {
throw new Exception(Exception::FUNCTION_ENTRYPOINT_MISSING);
}
@ -830,6 +846,8 @@ App::post('/v1/functions/:functionId/deployments')
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'entrypoint' => $entrypoint,
'buildCommand' => $buildCommand,
'installCommand' => $installCommand,
'path' => $path,
'size' => $fileSize,
'search' => implode(' ', [$deploymentId, $entrypoint]),
@ -1148,7 +1166,7 @@ App::post('/v1/functions/:functionId/executions')
->param('body', '', new Text(8192), 'HTTP body of execution. Default value is empty string.', true)
->param('async', false, new Boolean(), 'Execute code in the background. Default value is false.', true)
->param('path', '/', new Text(2048), 'HTTP path of execution. Path can include query params. Default value is /', true)
->param('method', 'GET', new Whitelist([ 'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS' ], true), 'HTTP method of execution. Default value is GET.', true)
->param('method', 'GET', new Whitelist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], true), 'HTTP method of execution. Default value is GET.', true)
->param('headers', [], new Assoc(), 'HTP headers of execution. Defaults to empty.', true)
->inject('response')
->inject('project')
@ -1281,6 +1299,8 @@ App::post('/v1/functions/:functionId/executions')
->dynamic($execution, Response::MODEL_EXECUTION);
}
$durationStart = \microtime(true);
$vars = [];
// Shared vars
@ -1315,6 +1335,7 @@ App::post('/v1/functions/:functionId/executions')
/** Execute function */
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
try {
$command = 'npm start';
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
@ -1328,6 +1349,10 @@ App::post('/v1/functions/:functionId/executions')
path: $path,
method: $method,
headers: $headers,
startCommands: [
'sh', '-c',
'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && helpers/start.sh "' . $command . '"'
]
);
/** Update execution status */
@ -1338,9 +1363,10 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('errors', $executionResponse['errors']);
$execution->setAttribute('duration', $executionResponse['duration']);
} catch (\Throwable $th) {
$interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
$durationEnd = \microtime(true);
$execution
->setAttribute('duration', (float)$interval->format('%s.%f'))
->setAttribute('duration', $durationEnd - $durationStart)
->setAttribute('status', 'failed')
->setAttribute('statusCode', 500)
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
@ -1353,10 +1379,10 @@ App::post('/v1/functions/:functionId/executions')
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
@ -1663,8 +1689,7 @@ App::put('/v1/functions/:functionId/variables/:variableId')
$variable
->setAttribute('key', $key)
->setAttribute('value', $value ?? $variable->getAttribute('value'))
->setAttribute('search', implode(' ', [$variableId, $function->getId(), $key, 'function']))
;
->setAttribute('search', implode(' ', [$variableId, $function->getId(), $key, 'function']));
try {
$dbForProject->updateDocument('variables', $variable->getId(), $variable);

22
app/workers/builds.php
View file

@ -91,6 +91,7 @@ class BuildsV1 extends Worker
$buildId = $deployment->getAttribute('buildId', '');
$startTime = DateTime::now();
$durationStart = \microtime(true);
if (empty($buildId)) {
$buildId = ID::unique();
$build = $dbForProject->createDocument('builds', new Document([
@ -172,6 +173,19 @@ class BuildsV1 extends Worker
}, []);
try {
$command = '';
if(!empty($deployment->getAttribute('installCommand', ''))) {
$command .= $deployment->getAttribute('installCommand', '');
}
if(!empty($deployment->getAttribute('buildCommand', ''))) {
$separator = empty($command) ? '' : ' && ';
$command .= $separator . $deployment->getAttribute('buildCommand', '');
}
$command = \str_replace('"', '\\"', $command);
$response = $this->executor->createRuntime(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
@ -180,13 +194,11 @@ class BuildsV1 extends Worker
image: $runtime['image'],
remove: true,
entrypoint: $deployment->getAttribute('entrypoint'),
workdir: '/usr/code',
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
variables: $vars,
commands: [
'sh', '-c',
'tar -zxf /tmp/code.tar.gz -C /usr/code && \
cd /usr/local/src/ && ./build.sh'
'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'
]
);
@ -229,9 +241,9 @@ class BuildsV1 extends Worker
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
} catch (\Throwable $th) {
$endTime = DateTime::now();
$interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
$durationEnd = \microtime(true);
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', $interval->format('%s') + 0);
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
$build->setAttribute('status', 'failed');
$build->setAttribute('stderr', $th->getMessage());
Console::error($th->getMessage());

13
app/workers/functions.php
View file

@ -14,6 +14,7 @@ use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\ID;
use Utopia\Database\Permission;
@ -124,6 +125,8 @@ Server::setResource('execute', function () {
}
}
$durationStart = \microtime(true);
$vars = [];
// global vars
@ -164,6 +167,7 @@ Server::setResource('execute', function () {
/** Execute function */
try {
$command = 'npm start';
$client = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
$executionResponse = $client->createExecution(
projectId: $project->getId(),
@ -178,6 +182,10 @@ Server::setResource('execute', function () {
path: $path,
method: $method,
headers: $headers,
startCommands: [
'sh', '-c',
'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && helpers/start.sh "' . $command . '"'
]
);
$status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed';
@ -190,9 +198,10 @@ Server::setResource('execute', function () {
->setAttribute('errors', $executionResponse['errors'])
->setAttribute('duration', $executionResponse['duration']);
} catch (\Throwable $th) {
$interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
$durationEnd = \microtime(true);
$execution
->setAttribute('duration', (float)$interval->format('%s.%f'))
->setAttribute('duration', $durationEnd - $durationStart)
->setAttribute('status', 'failed')
->setAttribute('statusCode', 500)
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());

4
src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php
View file

@ -5,10 +5,12 @@ namespace Appwrite\Utopia\Database\Validator\Queries;
class Deployments extends Base
{
public const ALLOWED_ATTRIBUTES = [
'entrypoint',
'size',
'buildId',
'activate',
'entrypoint',
'buildCommand',
'installCommand'
];
/**

3
src/Appwrite/Utopia/Database/Validator/Queries/Executions.php
View file

@ -6,6 +6,9 @@ class Executions extends Base
{
public const ALLOWED_ATTRIBUTES = [
'trigger',
'method',
'path',
'agent',
'status',
'statusCode',
'duration'

5
src/Appwrite/Utopia/Database/Validator/Queries/Functions.php
View file

@ -12,7 +12,10 @@ class Functions extends Base
'schedule',
'scheduleNext',
'schedulePrevious',
'timeout'
'timeout',
'entrypoint',
'buildCommand',
'installCommand'
];
/**

14
src/Appwrite/Utopia/Response/Model/Func.php
View file

@ -95,10 +95,22 @@ class Func extends Model
])
->addRule('entrypoint', [
'type' => self::TYPE_STRING,
'description' => 'The entrypoint file to use to execute the deployment code.',
'description' => 'The entrypoint file used to execute the deployment.',
'default' => '',
'example' => 'index.js',
])
->addRule('buildCommand', [
'type' => self::TYPE_STRING,
'description' => 'The build command used to build the deployment.',
'default' => '',
'example' => 'npm run build',
])
->addRule('installCommand', [
'type' => self::TYPE_STRING,
'description' => 'The install command used to build the deployment.',
'default' => '',
'example' => 'npm install',
])
;
}

12
src/Executor/Executor.php
View file

@ -55,7 +55,6 @@ class Executor
* @param string $image
* @param bool $remove
* @param string $entrypoint
* @param string $workdir
* @param string $destination
* @param array $variables
* @param array $commands
@ -68,10 +67,10 @@ class Executor
string $version,
bool $remove = false,
string $entrypoint = '',
string $workdir = '',
string $destination = '',
array $variables = [],
array $commands = []
array $commands = null,
array $startCommands = null,
) {
$runtimeId = "$projectId-$deploymentId";
$route = "/runtimes";
@ -81,10 +80,10 @@ class Executor
'destination' => $destination,
'image' => $image,
'entrypoint' => $entrypoint,
'workdir' => $workdir,
'variables' => $variables,
'remove' => $remove,
'commands' => $commands,
'startCommands' => $startCommands,
'cpus' => $this->cpus,
'memory' => $this->memory,
'version' => $version,
@ -140,6 +139,7 @@ class Executor
* @param string $image
* @param string $source
* @param string $entrypoint
* @param array $commands
*
* @return array
*/
@ -156,6 +156,8 @@ class Executor
string $path,
string $method,
array $headers,
array $commands = null,
array $startCommands = null
) {
$headers['host'] = App::getEnv('_APP_DOMAIN', '');
@ -176,6 +178,8 @@ class Executor
'cpus' => $this->cpus,
'memory' => $this->memory,
'version' => $version,
'commands' => $commands,
'startCommands' => $startCommands
];
$timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);