CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-05-20 04:32:37 +12:00
Code Issues Projects Releases Wiki Activity

feat: add build worker

Browse source
This commit is contained in:
Christy Jacob 2022-01-24 02:25:46 +04:00
parent 12ed89b49c
commit abd54938ef
9 changed files with 342 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env
View file

@ -39,7 +39,7 @@ _APP_FUNCTIONS_CONTAINERS=10
_APP_FUNCTIONS_CPUS=4
_APP_FUNCTIONS_MEMORY=2000
_APP_FUNCTIONS_MEMORY_SWAP=2000
_APP_EXECUTOR_SECRET=a-randomly-generated-key
_APP_EXECUTOR_SECRET=a-random-secret
_APP_MAINTENANCE_INTERVAL=86400
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400

1
Dockerfile
View file

@ -269,6 +269,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-database && \
chmod +x /usr/local/bin/worker-deletes && \
chmod +x /usr/local/bin/worker-functions && \
chmod +x /usr/local/bin/worker-builds && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-webhooks

40
app/controllers/api/functions.php
View file

@ -566,6 +566,8 @@ App::post('/v1/functions/:functionId/tags')
}
$tagId = $dbForProject->getId();
// TODO : What should be the read and write permissons of a tag?
$tag = $dbForProject->createDocument('tags', new Document([
'$id' => $tagId,
'$read' => ['role:all'],
@ -586,42 +588,14 @@ App::post('/v1/functions/:functionId/tags')
->setParam('storage', $tag->getAttribute('size', 0))
;
// Send start build reqeust to executor using /v1/tag
$function = $dbForProject->getDocument('functions', $functionId);
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/tag");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
// Enqueue a message to start the build
Resque::enqueue('v1-builds', 'BuildsV1', [
'projectId' => $project->getId(),
'functionId' => $function->getId(),
'tagId' => $tag->getId(),
'userId' => $user->getId(),
]));
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'x-appwrite-project: '.$project->getId(),
'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '')
'tagId' => $tagId,
'type' => 'tag'
]);
$executorResponse = \curl_exec($ch);
$error = \curl_error($ch);
if (!empty($error)) {
throw new Exception('Executor Communication Error: ' . $error, 500);
}
// Check status code
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (200 !== $statusCode) {
throw new Exception('Executor error: ' . $executorResponse, $statusCode);
}
\curl_close($ch);
$response->setStatusCode(Response::STATUS_CODE_CREATED);
$response->dynamic($tag, Response::MODEL_TAG);
});

140
app/executor.php
View file

@ -151,15 +151,22 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI
global $runtimes;
global $activeFunctions;
var_dump("Im here 1");
try {
$orchestration = $orchestrationPool->get();
var_dump("Im here 2");
$function = $database->getDocument('functions', $functionId);
var_dump("Im here 3");
$tag = $database->getDocument('tags', $tagId);
var_dump("Im here 4");
if ($tag->getAttribute('buildId') === null) {
throw new Exception('Tag has no buildId');
}
var_dump("Im here 5");
// Grab Build Document
$build = $database->getDocument('builds', $tag->getAttribute('buildId'));
@ -170,6 +177,8 @@ function createRuntimeServer(string $functionId, string $projectId, string $tagI
return;
}
var_dump("Im here");
// Generate random secret key
$secret = \bin2hex(\random_bytes(16));
@ -787,122 +796,27 @@ App::post('/v1/cleanup/tag')
return $response->json(['success' => true]);
});
App::post('/v1/tag')
App::post('/v1/create/runtime')
->desc('Create a new runtime server')
->param('functionId', '', new UID(), 'Function unique ID.')
->param('tagId', '', new UID(), 'Tag unique ID.')
->param('userId', '', new UID(), 'User unique ID.', true)
->inject('projectID')
->inject('response')
->inject('dbForProject')
->inject('projectID')
->inject('register')
->action(function (string $functionId, string $tagId, string $userId, Response $response, Database $dbForProject, string $projectID, Registry $register) use ($runtimes) {
// Get function document
$function = $dbForProject->getDocument('functions', $functionId);
// Get tag document
$tag = $dbForProject->getDocument('tags', $tagId);
// Check if both documents exist
if ($function->isEmpty()) {
throw new Exception('Function not found', 404);
}
if ($tag->isEmpty()) {
throw new Exception('Tag not found', 404);
}
$runtime = $runtimes[$function->getAttribute('runtime')] ?? null;
if (\is_null($runtime)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
// Create a new build entry
$buildId = $dbForProject->getId();
if ($tag->getAttribute('buildId')) {
$buildId = $tag->getAttribute('buildId');
} else {
try {
$dbForProject->createDocument('builds', new Document([
'$id' => $buildId,
'$read' => (!empty($userId)) ? ['user:' . $userId] : [],
'$write' => ['role:all'],
'dateCreated' => time(),
'status' => 'processing',
'runtime' => $function->getAttribute('runtime'),
'outputPath' => '',
'source' => $tag->getAttribute('path'),
'sourceType' => Storage::DEVICE_LOCAL,
'stdout' => '',
'stderr' => '',
'buildTime' => 0,
'envVars' => [
'ENTRYPOINT_NAME' => $tag->getAttribute('entrypoint'),
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
'APPWRITE_FUNCTION_PROJECT_ID' => $projectID,
]
]));
$tag->setAttribute('buildId', $buildId);
$dbForProject->updateDocument('tags', $tag->getId(), $tag);
} catch (\Throwable $th) {
var_dump($tag->getArrayCopy());
throw $th;
}
}
// Build Code
go(function () use ($projectID, $tagId, $buildId, $functionId, $function, $register) {
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$cache = new Cache(new RedisCache($redis));
$dbForProject = new Database(new MariaDB($db), $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace('_project_' . $projectID);
// Build Code
runBuildStage($buildId, $projectID);
// Update the schedule
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0;
// Grab tag
$tag = $dbForProject->getDocument('tags', $tagId);
// Grab build
$build = $dbForProject->getDocument('builds', $buildId);
// If the build failed, it won't be possible to deploy
if ($build->getAttribute('status') !== 'ready') {
return;
}
if ($tag->getAttribute('automaticDeploy') === true) {
// Update the function document setting the tag as the active one
$function
->setAttribute('tag', $tag->getId())
->setAttribute('scheduleNext', (int)$next);
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
// Deploy Runtime Server
->action(function (string $functionId, string $tagId, string $projectID, Response $response, Database $dbForProject) {
try {
Console::success('Creating runtime for tag ' . $tagId);
createRuntimeServer($functionId, $projectID, $tagId, $dbForProject);
} catch (\Throwable $th) {
$response
->setStatusCode(400)
->json(['error' => $th->getMessage()]);
};
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
});
if (false === $function) {
throw new Exception('Failed saving function to DB', 500);
}
$response->dynamic($function, Response::MODEL_FUNCTION);
$response
->setStatusCode(201)
->noContent();
});
App::get('/v1/')
@ -943,10 +857,8 @@ App::post('/v1/build/:buildId') // Start a Build
throw new Exception('Build is already finished', 409);
}
go(function () use ($buildId, $dbForProject, $projectID) {
// Build Code
runBuildStage($buildId, $projectID, $dbForProject);
});
// Build Code
runBuildStage($buildId, $projectID, $dbForProject);
// return success
return $response->json(['success' => true]);

253
app/workers/builds.php Normal file
View file

@ -0,0 +1,253 @@
<?php
use Appwrite\Resque\Worker;
use Cron\CronExpression;
use Utopia\Database\Validator\Authorization;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Storage\Storage;
use Utopia\Database\Document;
use Utopia\Config\Config;
require_once __DIR__.'/../init.php';
Console::title('Builds V1 Worker');
Console::success(APP_NAME.' build worker v1 has started');
class BuildsV1 extends Worker
{
public function getName(): string {
return "builds";
}
public function init(): void
{
Console::success("Initializing...");
}
public function run(): void
{
$type = $this->args['type'] ?? '';
$projectId = $this->args['projectId'] ?? '';
switch ($type) {
case 'tag':
$functionId = $this->args['functionId'] ?? '';
$tagId = $this->args['tagId'] ?? '';
Console::success("Creating build for tag: $tagId");
$this->buildTag($projectId, $functionId, $tagId);
break;
default:
throw new \Exception('Invalid trigger');
break;
}
}
protected function triggerBuildStage(string $projectId, string $buildId)
{
// TODO: What is a reasonable time to wait for a build to complete?
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/build/$buildId");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'x-appwrite-project: '.$projectId,
'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '')
]);
$response = \curl_exec($ch);
$responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$error = \curl_error($ch);
if (!empty($error)) {
throw new \Exception($error);
}
\curl_close($ch);
if ($responseStatus !== 200) {
throw new \Exception("Build failed with status code: $responseStatus");
}
$response = json_decode($response, true);
if (isset($response['error'])) {
throw new \Exception($response['error']);
}
if (isset($response['success']) && $response['success'] === true) {
return;
} else {
throw new \Exception("Build failed");
}
}
protected function triggerCreateRuntimeServer(string $projectId, string $functionId, string $tagId)
{
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/create/runtime");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'x-appwrite-project: '.$projectId,
'x-appwrite-executor-key: '. App::getEnv('_APP_EXECUTOR_SECRET', '')
]);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'functionId' => $functionId,
'tagId' => $tagId
]));
$response = \curl_exec($ch);
$responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$error = \curl_error($ch);
if (!empty($error)) {
throw new \Exception($error);
}
\curl_close($ch);
if ($responseStatus !== 200) {
throw new \Exception("Build failed with status code: $responseStatus");
}
$response = json_decode($response, true);
if (isset($response['error'])) {
throw new \Exception($response['error']);
}
if (isset($response['success']) && $response['success'] === true) {
return;
} else {
throw new \Exception("Build failed");
}
}
protected function buildTag(string $projectId, string $functionId, string $tagId)
{
$dbForProject = $this->getProjectDB($projectId);
// TODO: Why does it need to skip authorization?
$function = Authorization::skip(fn() => $dbForProject->getDocument('functions', $functionId));
if ($function->isEmpty()) {
throw new Exception('Function not found', 404);
}
// Get tag document
$tag = $dbForProject->getDocument('tags', $tagId);
if ($tag->isEmpty()) {
throw new Exception('Tag not found', 404);
}
$runtimes = Config::getParam('runtimes', []);
$key = $function->getAttribute('runtime');
$runtime = isset($runtimes[$key]) ? $runtimes[$key] : null;
if (\is_null($runtime)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
$buildId = $tag->getAttribute('buildId', '');
// If build ID is empty, create a new build
if (empty($buildId)) {
try {
$buildId = $dbForProject->getId();
// TODO : There is no way to associate a build with a tag. So we need to add a tagId attribute to the build document
// TODO : What should be the read and write permissions for a build ?
$dbForProject->createDocument('builds', new Document([
'$id' => $buildId,
'$read' => ['role:all'],
'$write' => ['role:all'],
'dateCreated' => time(),
'status' => 'processing',
'runtime' => $function->getAttribute('runtime'),
'outputPath' => '',
'source' => $tag->getAttribute('path'),
'sourceType' => Storage::DEVICE_LOCAL,
'stdout' => '',
'stderr' => '',
'buildTime' => 0,
'envVars' => [
'ENTRYPOINT_NAME' => $tag->getAttribute('entrypoint'),
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
'APPWRITE_FUNCTION_PROJECT_ID' => $projectId,
]
]));
$tag->setAttribute('buildId', $buildId);
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
} catch (\Throwable $th) {
Console::error($th->getMessage());
$tag->setAttribute('status', 'failed');
$tag->setAttribute('buildId', '');
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
return;
}
}
// Build the Code
try {
Console::success("Creating Build with id: $buildId");
$tag->setAttribute('status', 'building');
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
$this->triggerBuildStage($projectId, $buildId);
} catch (\Throwable $th) {
Console::error($th->getMessage());
$tag->setAttribute('status', 'failed');
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
return;
}
Console::success("Build id: $buildId completed successfully");
// Update the schedule
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('tag')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('tag')) && !empty($schedule)) ? $cron->getNextRunDate()->format('U') : 0;
// Grab build
$build = $dbForProject->getDocument('builds', $buildId);
// If the build failed, it won't be possible to deploy
if ($build->getAttribute('status') !== 'ready') {
throw new Exception('Build failed', 500);
}
if ($tag->getAttribute('automaticDeploy') === true) {
// Update the function document setting the tag as the active one
$function
->setAttribute('tag', $tag->getId())
->setAttribute('scheduleNext', (int)$next);
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
// Deploy Runtime Server
try {
Console::success("Creating Runtime Server");
$this->triggerCreateRuntimeServer($functionId, $projectId, $tagId, $dbForProject);
} catch (\Throwable $th) {
Console::error($th->getMessage());
$tag->setAttribute('status', 'failed');
$tag = $dbForProject->updateDocument('tags', $tagId, $tag);
return;
}
Console::success("Runtime Server created successfully");
}
public function shutdown(): void
{
Console::success("Shutting Down...");
}
}

10
bin/worker-builds Normal file
View file

@ -0,0 +1,10 @@
#!/bin/sh
if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ]
then
REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
else
REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
fi
INTERVAL=0.1 QUEUE='v1-builds' APP_INCLUDE='/usr/src/code/app/workers/builds.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php

2
composer.json
View file

@ -50,7 +50,7 @@
"utopia-php/cache": "0.4.*",
"utopia-php/cli": "0.11.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.13.*",
"utopia-php/database": "0.14.*",
"utopia-php/locale": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/preloader": "0.2.*",

33
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "cba39f50398d5ae2b121db34c9e4c529",
"content-hash": "1a5d84f96eb76e59f7ad0ff7bcd4a8d8",
"packages": [
{
"name": "adhocore/jwt",
@ -2135,16 +2135,16 @@
},
{
"name": "utopia-php/database",
"version": "0.13.2",
"version": "0.14.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "bf92279b707b3a10ee5ec5df5c065023b2221357"
"reference": "2f2527bb080cf578fba327ea2ec637064561d403"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/bf92279b707b3a10ee5ec5df5c065023b2221357",
"reference": "bf92279b707b3a10ee5ec5df5c065023b2221357",
"url": "https://api.github.com/repos/utopia-php/database/zipball/2f2527bb080cf578fba327ea2ec637064561d403",
"reference": "2f2527bb080cf578fba327ea2ec637064561d403",
"shasum": ""
},
"require": {
@ -2192,9 +2192,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.13.2"
"source": "https://github.com/utopia-php/database/tree/0.14.0"
},
"time": "2022-01-04T10:51:22+00:00"
"time": "2022-01-21T16:34:34+00:00"
},
{
"name": "utopia-php/domains",
@ -3126,23 +3126,23 @@
},
{
"name": "composer/pcre",
"version": "1.0.0",
"version": "1.0.1",
"source": {
"type": "git",
"url": "https://github.com/composer/pcre.git",
"reference": "3d322d715c43a1ac36c7fe215fa59336265500f2"
"reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/composer/pcre/zipball/3d322d715c43a1ac36c7fe215fa59336265500f2",
"reference": "3d322d715c43a1ac36c7fe215fa59336265500f2",
"url": "https://api.github.com/repos/composer/pcre/zipball/67a32d7d6f9f560b726ab25a061b38ff3a80c560",
"reference": "67a32d7d6f9f560b726ab25a061b38ff3a80c560",
"shasum": ""
},
"require": {
"php": "^5.3.2 || ^7.0 || ^8.0"
},
"require-dev": {
"phpstan/phpstan": "^1",
"phpstan/phpstan": "^1.3",
"phpstan/phpstan-strict-rules": "^1.1",
"symfony/phpunit-bridge": "^4.2 || ^5"
},
@ -3177,7 +3177,7 @@
],
"support": {
"issues": "https://github.com/composer/pcre/issues",
"source": "https://github.com/composer/pcre/tree/1.0.0"
"source": "https://github.com/composer/pcre/tree/1.0.1"
},
"funding": [
{
@ -3193,7 +3193,7 @@
"type": "tidelift"
}
],
"time": "2021-12-06T15:17:27+00:00"
"time": "2022-01-21T20:24:37+00:00"
},
{
"name": "composer/semver",
@ -3697,9 +3697,6 @@
"require": {
"php": "^7.1 || ^8.0"
},
"replace": {
"myclabs/deep-copy": "self.version"
},
"require-dev": {
"doctrine/collections": "^1.0",
"doctrine/common": "^2.6",
@ -6665,5 +6662,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.1.0"
"plugin-api-version": "2.2.0"
}

28
docker-compose.yml
View file

@ -286,6 +286,34 @@ services:
- _APP_DB_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-worker-builds:
entrypoint: worker-builds
container_name: appwrite-worker-builds
build:
context: .
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_EXECUTOR_SECRET
appwrite-worker-certificates:
entrypoint: worker-certificates