CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-27 02:31:04 +12:00
Code Issues Projects Releases Wiki Activity

Add build step and more function cleanup

Browse source
This commit is contained in:
Bradley Schofield 2021-09-13 11:50:45 +01:00
parent 66d8b0599c
commit da8b4bb9f9
12 changed files with 508 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
app/config/collections.php
View file

@ -1622,6 +1622,24 @@ $collections = [
'default' => '',
'required' => false,
'array' => false,
],
[
'$collection' => Database::SYSTEM_COLLECTION_RULES,
'label' => 'Build Status',
'key' => 'status',
'type' => Database::SYSTEM_VAR_TYPE_TEXT,
'default' => '',
'required' => false,
'array' => false,
],
[
'$collection' => Database::SYSTEM_COLLECTION_RULES,
'label' => 'Build Path',
'key' => 'builtPath',
'type' => Database::SYSTEM_VAR_TYPE_TEXT,
'default' => '',
'required' => false,
'array' => false,
]
],
],

72
app/controllers/api/functions.php
View file

@ -403,15 +403,47 @@ App::delete('/v1/functions/:functionId')
->label('sdk.response.model', Response::MODEL_NONE)
->param('functionId', '', new UID(), 'Function unique ID.')
->inject('response')
->inject('project')
->inject('projectDB')
->inject('deletes')
->action(function ($functionId, $response, $projectDB, $deletes) {
->action(function ($functionId, $response, $project, $projectDB, $deletes) {
/** @var Appwrite\Utopia\Response $response */
/** @var Appwrite\Database\Database $projectDB */
/** @var Appwrite\Event\Event $deletes */
$function = $projectDB->getDocument($functionId);
// Request executor to delete tag containers
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/function");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'functionId' => $functionId
]));
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'X-Appwrite-Project: '.$project->getId(),
]);
$executorResponse = \curl_exec($ch);
$error = \curl_error($ch);
if (!empty($error)) {
throw new Exception('Curl error: ' . $error, 500);
}
// Check status code
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (200 !== $statusCode) {
throw new Exception('Executor error: ' . $executorResponse, $statusCode);
}
\curl_close($ch);
if (empty($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
throw new Exception('Function not found', 404);
}
@ -506,7 +538,9 @@ App::post('/v1/functions/:functionId/tags')
'dateCreated' => time(),
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $size
'size' => $size,
'status' => 'pending',
'builtPath' => ''
]);
if (false === $tag) {
@ -620,9 +654,10 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
->param('functionId', '', new UID(), 'Function unique ID.')
->param('tagId', '', new UID(), 'Tag unique ID.')
->inject('response')
->inject('project')
->inject('projectDB')
->inject('usage')
->action(function ($functionId, $tagId, $response, $projectDB, $usage) {
->action(function ($functionId, $tagId, $response, $project, $projectDB, $usage) {
/** @var Appwrite\Utopia\Response $response */
/** @var Appwrite\Database\Database $projectDB */
/** @var Appwrite\Event\Event $usage */
@ -643,6 +678,37 @@ App::delete('/v1/functions/:functionId/tags/:tagId')
throw new Exception('Tag not found', 404);
}
// Request executor to delete tag containers
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://appwrite-executor:8080/v1/cleanup/tag");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'tagId' => $tagId
]));
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, 900);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'X-Appwrite-Project: '.$project->getId(),
]);
$executorResponse = \curl_exec($ch);
$error = \curl_error($ch);
if (!empty($error)) {
throw new Exception('Curl error: ' . $error, 500);
}
// Check status code
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (200 !== $statusCode) {
throw new Exception('Executor error: ' . $executorResponse, $statusCode);
}
\curl_close($ch);
$device = Storage::getDevice('functions');
if ($device->delete($tag->getAttribute('path', ''))) {

487
app/executor.php
View file

@ -26,6 +26,12 @@ use Utopia\Validator\ArrayList;
use Utopia\Validator\JSON;
use Utopia\Validator\Text;
use Cron\CronExpression;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\FileExt;
use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
use Swoole\Coroutine as Co;
require_once __DIR__ . '/workers.php';
@ -39,12 +45,12 @@ $runtimes = Config::getParam('runtimes');
Swoole\Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_CURL);
// Warmup: make sure images are ready to run fast 🚀
Co\run(function() use ($runtimes, $orchestration) {
foreach($runtimes as $runtime) {
go(function() use ($runtime, $orchestration) {
Console::info('Warming up '.$runtime['name'].' '.$runtime['version'].' environment...');
$response = $orchestration->pull($runtime['image']);
Co\run(function () use ($runtimes, $orchestration) {
foreach ($runtimes as $runtime) {
go(function () use ($runtime, $orchestration) {
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
$response = $orchestration->pull($runtime['image']);
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
@ -70,7 +76,7 @@ foreach ($response as $value) {
$executionEnd = \microtime(true);
Console::info(count($activeFunctions).' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
Console::info(count($activeFunctions) . ' functions listed in ' . ($executionEnd - $executionStart) . ' seconds');
App::post('/v1/execute') // Define Route
->inject('request')
@ -87,19 +93,127 @@ App::post('/v1/execute') // Define Route
->inject('response')
->action(
function ($trigger, $projectId, $executionId, $functionId, $event, $eventData, $data, $webhooks, $userId, $jwt, $request, $response) {
global $register;
$db = $register->get('dbPool')->get();
$cache = $register->get('redisPool')->get();
// Create new Database Instance
$database = new Database();
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$database->setNamespace('app_' . $projectId);
$database->setMocks(Config::getParam('collections', []));
try {
$data = execute($trigger, $projectId, $executionId, $functionId, $event, $eventData, $data, $webhooks, $userId, $jwt);
return $response->json($data);
$data = execute($trigger, $projectId, $executionId, $functionId, $database, $event, $eventData, $data, $webhooks, $userId, $jwt);
$response->json($data);
} catch (Exception $e) {
return $response
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->json(['error' => $e->getMessage()]);
} finally {
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($cache);
}
}
);
// Cleanup Endpoints used internally by appwrite when a function or tag gets deleted to also clean up their containers
App::post('/v1/cleanup/function')
->param('functionId', '', new UID())
->inject('response')
->inject('projectDB')
->inject('projectID')
->action(function ($functionId, $response, $projectDB, $projectID) {
/** @var string $functionId */
/** @var Appwrite\Utopia\Response $response */
/** @var Appwrite\Database\Database $projectDB */
/** @var string $projectID */
global $orchestration;
try {
Authorization::disable();
$function = $projectDB->getDocument($functionId);
Authorization::reset();
if (\is_null($function->getId()) || Database::SYSTEM_COLLECTION_FUNCTIONS != $function->getCollection()) {
throw new Exception('Function not found', 404);
}
Authorization::disable();
$results = $projectDB->getCollection([
'limit' => 999,
'offset' => 0,
'orderType' => 'ASC',
'filters' => [
'$collection='.Database::SYSTEM_COLLECTION_TAGS,
'functionId='.$functionId,
],
]);
Authorization::reset();
// If amount is 0 then we simply return true
if (count($results) === 0) {
return $response->json(['success' => true]);
}
// Delete the containers of all tags
foreach ($results as $tag) {
try {
$orchestration->remove('appwrite-function-'.$tag['$id'], true);
Console::info('Removed container for tag ' . $tag['$id']);
} catch (Exception $e) {
// Do nothing, we don't care that much if it fails
}
}
return $response->json(['success' => true]);
} catch (Exception $e) {
Console::error($e->getMessage());
return $response->json(['error' => $e->getMessage()]);
}
});
App::post('/v1/cleanup/tag')
->param('tagId', '', new UID(), 'Tag unique ID.')
->inject('response')
->inject('projectDB')
->inject('projectID')
->action(function ($tagId, $response, $projectDB, $projectID) {
/** @var string $tagId */
/** @var Appwrite\Utopia\Response $response */
/** @var Appwrite\Database\Database $projectDB */
/** @var string $projectID */
global $orchestration;
try {
Authorization::disable();
$tag = $projectDB->getDocument($tagId);
Authorization::reset();
if (\is_null($tag->getId()) || Database::SYSTEM_COLLECTION_TAGS != $tag->getCollection()) {
throw new Exception('Tag not found', 404);
}
try {
$orchestration->remove('appwrite-function-'.$tag['$id'], true);
Console::info('Removed container for tag ' . $tag['$id']);
} catch (Exception $e) {
// Do nothing, we don't care that much if it fails
}
} catch (Exception $e) {
Console::error($e->getMessage());
return $response->json(['error' => $e->getMessage()]);
}
return $response->json(['success' => true]);
});
App::post('/v1/tag')
->param('functionId', '', new UID(), 'Function unique ID.')
->param('tagId', '', new UID(), 'Tag unique ID.')
@ -132,8 +246,14 @@ App::post('/v1/tag')
]));
Authorization::reset();
// Deploy Runtime Server
createRuntimeServer($functionId, $projectID, $tag);
// Build Code
go(function() use ($projectDB, $projectID, $function, $tagId, $functionId) {
// Build Code
$tag = runBuildStage($tagId, $function, $projectID, $projectDB);
// Deploy Runtime Server
createRuntimeServer($functionId, $projectID, $tag, $projectDB);
});
if ($next) { // Init first schedule
ResqueScheduler::enqueueAt($next, 'v1-functions', 'FunctionsV1', [
@ -165,27 +285,210 @@ App::get('/v1/healthz')
}
);
function createRuntimeServer(string $functionId, string $projectId, Document $tag) {
function runBuildStage(string $tagID, Document $function, string $projectID, Database $database): Document
{
global $runtimes;
global $orchestration;
// Update Tag Status
Authorization::disable();
$tag = $database->getDocument($tagID);
$tag = $database->updateDocument(array_merge($tag->getArrayCopy(), [
'status' => 'building'
]));
Authorization::reset();
// Check if runtime is active
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')]))
? $runtimes[$function->getAttribute('runtime', '')]
: null;
if ($tag->getAttribute('functionId') !== $function->getId()) {
throw new Exception('Tag not found', 404);
}
if (\is_null($runtime)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
// Grab Tag Files
$tagPath = $tag->getAttribute('path', '');
$tagPathTarget = '/tmp/project-' . $projectID . '/' . $tag->getId() . '/code.tar.gz';
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
$container = 'build-stage-' . $tag->getId();
if (!\is_readable($tagPath)) {
throw new Exception('Code is not readable: ' . $tag->getAttribute('path', ''));
}
if (!\file_exists($tagPathTargetDir)) {
if (!\mkdir($tagPathTargetDir, 0755, true)) {
throw new Exception('Can\'t create directory ' . $tagPathTargetDir);
}
}
if (!\file_exists($tagPathTarget)) {
if (!\copy($tagPath, $tagPathTarget)) {
throw new Exception('Can\'t create temporary code file ' . $tagPathTarget);
}
}
$vars = \array_merge($function->getAttribute('vars', []), [
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name', ''),
'APPWRITE_FUNCTION_TAG' => $tag->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'],
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'],
'APPWRITE_FUNCTION_PROJECT_ID' => $projectID,
]);
$buildStart = \microtime(true);
$buildTime = \time();
$orchestration->setCpus(App::getEnv('_APP_FUNCTIONS_CPUS', '1'));
$orchestration->setMemory(App::getEnv('_APP_FUNCTIONS_MEMORY', '256'));
$orchestration->setSwap(App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', '256'));
foreach ($vars as &$value) {
$value = strval($value);
}
$id = $orchestration->run(
image: $runtime['base'],
name: $container,
vars: $vars,
workdir: '/usr/code',
labels: [
'appwrite-type' => 'function',
'appwrite-created' => strval($buildTime),
'appwrite-runtime' => $function->getAttribute('runtime', ''),
],
command: [
'tail',
'-f',
'/dev/null'
],
hostname: $container,
mountFolder: $tagPathTargetDir,
volumes: [
'/tmp/project-' . $projectID . '/' . $tag->getId() . '/builtCode'. ':/usr/builtCode:rw'
]
);
$untarStdout = '';
$untarStderr = '';
$untarSuccess = $orchestration->execute(
name: $container,
command: [
'sh',
'-c',
'mkdir /usr/code -p && cp /tmp/code.tar.gz /usr/code.tar.gz && cd /usr && tar -zxf /usr/code.tar.gz -C /usr/code && rm /usr/code.tar.gz'
],
stdout: $untarStdout,
stderr: $untarStderr,
timeout: 60
);
if (!$untarSuccess) {
throw new Exception('Failed to extract tar: ' . $untarStderr);
}
// Build Code / Install Dependencies
$buildStdout = '';
$buildStderr = '';
$buildSuccess = $orchestration->execute(
name: $container,
command: $runtime['buildCommand'],
stdout: $buildStdout,
stderr: $buildStderr,
timeout: 60
);
if (!$buildSuccess) {
throw new Exception('Failed to build dependencies: ' . $buildStderr);
}
// Repackage Code and Save.
$compressStdout = '';
$compressStderr = '';
$builtCodePath = '/tmp/project-' . $projectID . '/' . $tag->getId() . '/builtCode/code.tar.gz';
$compressSuccess = $orchestration->execute(
name: $container,
command: [
'sh',
'-c',
'tar -czvf /usr/builtCode/code.tar.gz /usr/code'
],
stdout: $compressStdout,
stderr: $compressStderr,
timeout: 60
);
if (!$compressSuccess) {
throw new Exception('Failed to compress built code: ' . $compressStderr);
}
// Remove Container
$orchestration->remove($id, true);
// Check if the build was successful by checking if file exists
if (!\file_exists($builtCodePath)) {
throw new Exception('Something went wrong during the build process.');
}
// Upload new code
$device = Storage::getDevice('functions');
$path = $device->getPath(\uniqid().'.'.\pathinfo('code.tar.gz', PATHINFO_EXTENSION));
if (!\file_exists(\dirname($path))) { // Checks if directory path to file exists
if (!@\mkdir(\dirname($path), 0755, true)) {
throw new Exception('Can\'t create directory: ' . \dirname($path));
}
}
if (!\rename($builtCodePath, $path)) {
throw new Exception('Failed moving file', 500);
}
// Update tag with built code attribute
Authorization::disable();
$tag = $database->updateDocument(array_merge($tag->getArrayCopy(), [
'builtPath' => $path,
'status' => 'ready'
]));
Authorization::enable();
$buildEnd = \microtime(true);
Console::info('Tag Built in ' . ($buildEnd - $buildStart) . ' seconds');
return $tag;
}
function createRuntimeServer(string $functionId, string $projectId, Document $tag, Database $database)
{
global $register;
global $orchestration;
global $runtimes;
global $activeFunctions;
$db = $register->get('db');
$cache = $register->get('cache');
// Create new Database Instance
$database = new Database();
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$database->setNamespace('app_' . $projectId);
$database->setMocks(Config::getParam('collections', []));
// Grab Tag Document
Authorization::disable();
$function = $database->getDocument($functionId);
$tag = $database->getDocument($function->getAttribute('tag', ''));
Authorization::reset();
// Check if function isn't already created
$functions = $orchestration->list(['label' => 'appwrite-type=function', 'name' => 'appwrite-function-' . $tag->getId()]);
if (\count($functions) > 0) {
return;
}
// Check if runtime is active
$runtime = (isset($runtimes[$function->getAttribute('runtime', '')]))
? $runtimes[$function->getAttribute('runtime', '')]
@ -211,7 +514,7 @@ function createRuntimeServer(string $functionId, string $projectId, Document $ta
$container = 'appwrite-function-' . $tag->getId();
if (isset($activeFunctions[$container]) && !(\substr($activeFunctions[$container]->getStatus(), 0, 2) === 'Up')) { // Remove conatiner if not online
if (isset($activeFunctions[$container]) && !(\substr($activeFunctions[$container]->getStatus(), 0, 2) === 'Up')) { // Remove container if not online
// If container is online then stop and remove it
try {
$orchestration->remove($container);
@ -222,8 +525,14 @@ function createRuntimeServer(string $functionId, string $projectId, Document $ta
unset($activeFunctions[$container]);
}
// Check if tag is built yet.
if ($tag->getAttribute('status') !== 'ready') {
throw new Exception('Tag is not built yet', 500);
}
// Grab Tag Files
$tagPath = $tag->getAttribute('path', '');
$tagPath = $tag->getAttribute('builtPath', '');
$tagPathTarget = '/tmp/project-' . $projectId . '/' . $tag->getId() . '/code.tar.gz';
$tagPathTargetDir = \pathinfo($tagPathTarget, PATHINFO_DIRNAME);
$container = 'appwrite-function-' . $tag->getId();
@ -318,22 +627,13 @@ function createRuntimeServer(string $functionId, string $projectId, Document $ta
}
};
function execute(string $trigger, string $projectId, string $executionId, string $functionId, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
function execute(string $trigger, string $projectId, string $executionId, string $functionId, Database $database, string $event = '', string $eventData = '', string $data = '', array $webhooks = [], string $userId = '', string $jwt = ''): array
{
Console::info('Executing function: ' . $functionId);
global $activeFunctions;
global $runtimes;
global $register;
$db = $register->get('db');
$cache = $register->get('cache');
// Create new Database Instance
$database = new Database();
$database->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$database->setNamespace('app_' . $projectId);
$database->setMocks(Config::getParam('collections', []));
// Grab Tag Document
Authorization::disable();
$function = $database->getDocument($functionId);
@ -396,10 +696,41 @@ function execute(string $trigger, string $projectId, string $executionId, string
$container = 'appwrite-function-' . $tag->getId();
if (!isset($activeFunctions[$container])) { // Create contianer if not ready
createRuntimeServer($functionId, $projectId, $tag);
} else {
Console::info('Container is ready to run');
// Check if code is built
try {
if ($tag->getAttribute('status') !== 'ready') {
runBuildStage($tag->getId(), $function, $projectId, $database);
sleep(1);
}
} catch (Exception $e) {
Console::error('Something went wrong building the code. ' . $e->getMessage());
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
'tagId' => $tag->getId(),
'status' => 'failed',
'exitCode' => 1,
'stderr' => \utf8_encode(\mb_substr($e->getMessage(), -4000)), // log last 4000 chars output
'time' => 0
]));
}
try {
if (!isset($activeFunctions[$container])) { // Create contianer if not ready
createRuntimeServer($functionId, $projectId, $tag, $database);
} else if ($activeFunctions[$container]->getStatus() === 'Down') {
sleep(1);
} else {
Console::info('Container is ready to run');
}
} catch (Exception $e) {
Console::error('Something went wrong building the runtime server. ' . $e->getMessage());
$execution = $database->updateDocument(array_merge($execution->getArrayCopy(), [
'tagId' => $tag->getId(),
'status' => 'failed',
'exitCode' => 1,
'stderr' => \utf8_encode(\mb_substr($e->getMessage(), -4000)), // log last 4000 chars output
'time' => 0
]));
}
$stdout = '';
@ -419,7 +750,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
do {
$attempts++;
$ch = \curl_init();
\curl_setopt($ch, CURLOPT_URL, "http://".$container.":3000/");
\curl_setopt($ch, CURLOPT_URL, "http://" . $container . ":3000/");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
'path' => '/usr/code',
@ -432,11 +763,11 @@ function execute(string $trigger, string $projectId, string $executionId, string
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, $function->getAttribute('timeout', (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)));
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
$executorResponse = \curl_exec($ch);
$error = \curl_error($ch);
$errNo = \curl_errno($ch);
\curl_close($ch);
@ -456,7 +787,7 @@ function execute(string $trigger, string $projectId, string $executionId, string
if ($errNo == CURLE_OPERATION_TIMEDOUT) {
$exitCode = 124;
}
if ($errNo !== 0 && $errNo != CURLE_COULDNT_CONNECT && $errNo != CURLE_OPERATION_TIMEDOUT) {
throw new Exception('Curl error: ' . $error, 500);
}
@ -487,8 +818,8 @@ function execute(string $trigger, string $projectId, string $executionId, string
'tagId' => $tag->getId(),
'status' => $functionStatus,
'exitCode' => $exitCode,
'stdout' => \mb_substr($stdout, -4000), // log last 4000 chars output
'stderr' => \mb_substr($stderr, -4000), // log last 4000 chars output
'stdout' => \utf8_encode(\mb_substr($stdout, -4000)), // log last 4000 chars output
'stderr' => \utf8_encode(\mb_substr($stderr, -4000)), // log last 4000 chars output
'time' => $executionTime
]));
@ -578,12 +909,14 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$projectId = $request->getHeader('x-appwrite-project', '');
App::setResource('projectDB', function($db, $cache) use ($projectId) {
Storage::setDevice('functions', new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId));
App::setResource('projectDB', function ($db, $cache) use ($projectId) {
$projectDB = new Database();
$projectDB->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache));
$projectDB->setNamespace('app_'.$projectId);
$projectDB->setNamespace('app_' . $projectId);
$projectDB->setMocks(Config::getParam('collections', []));
return $projectDB;
}, ['db', 'cache']);
@ -592,31 +925,31 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
/** @var Utopia\App $utopia */
/** @var Utopia\Swoole\Request $request */
/** @var Appwrite\Utopia\Response $response */
if ($error instanceof PDOException) {
throw $error;
}
$route = $utopia->match($request);
Console::error('[Error] Timestamp: '.date('c', time()));
if($route) {
Console::error('[Error] Method: '.$route->getMethod());
Console::error('[Error] Timestamp: ' . date('c', time()));
if ($route) {
Console::error('[Error] Method: ' . $route->getMethod());
}
Console::error('[Error] Type: '.get_class($error));
Console::error('[Error] Message: '.$error->getMessage());
Console::error('[Error] File: '.$error->getFile());
Console::error('[Error] Line: '.$error->getLine());
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $error->getMessage());
Console::error('[Error] File: ' . $error->getFile());
Console::error('[Error] Line: ' . $error->getLine());
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$code = $error->getCode();
$message = $error->getMessage();
//$_SERVER = []; // Reset before reporting to error log to avoid keys being compromised
$output = ((App::isDevelopment())) ? [
'message' => $error->getMessage(),
'code' => $error->getCode(),
@ -629,19 +962,20 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
'code' => $code,
'version' => $version,
];
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->setStatusCode($code)
;
$response->dynamic(new Document($output),
$utopia->isDevelopment() ? Response::MODEL_ERROR_DEV : Response::MODEL_ERROR);
->setStatusCode($code);
$response->dynamic(
new Document($output),
$utopia->isDevelopment() ? Response::MODEL_ERROR_DEV : Response::MODEL_ERROR
);
}, ['error', 'utopia', 'request', 'response']);
App::setResource('projectID', function() use ($projectId) {
App::setResource('projectID', function () use ($projectId) {
return $projectId;
});
@ -655,7 +989,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$http->start();
function handleShutdown() {
function handleShutdown()
{
Console::info('Cleaning up containers before shutdown...');
// Remove all containers.
@ -666,9 +1001,9 @@ function handleShutdown() {
foreach ($functionsToRemove as $container) {
try {
$orchestration->remove($container->getId(), true);
Console::info('Removed container '.$container->getName());
Console::info('Removed container ' . $container->getName());
} catch (Exception $e) {
Console::error('Failed to remove container: '.$container->getName());
Console::error('Failed to remove container: ' . $container->getName());
}
}
}
}

21
app/workers/functions.php
View file

@ -34,27 +34,6 @@ $dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
$dockerEmail = App::getEnv('DOCKERHUB_PULL_EMAIL', null);
$orchestration = new Orchestration(new DockerAPI($dockerUser, $dockerPass, $dockerEmail));
/**
* Warmup Docker Images
*/
$warmupStart = \microtime(true);
Co\run(function () use ($runtimes, $orchestration) { // Warmup: make sure images are ready to run fast 🚀
foreach ($runtimes as $runtime) {
go(function () use ($runtime, $orchestration) {
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
$response = $orchestration->pull($runtime['image']);
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
} else {
Console::error("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
}
});
}
});
$warmupEnd = \microtime(true);
$warmupTime = $warmupEnd - $warmupStart;

1
composer.json
View file

@ -66,7 +66,6 @@
"utopia-php/orchestration": "dev-exp1",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "4.2.3",
"utopia-php/orchestration": "0.2.1",
"dragonmantank/cron-expression": "3.1.0",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.5.0",

5
composer.lock generated
View file

@ -119,7 +119,7 @@
"source": {
"type": "git",
"url": "https://github.com/PineappleIOnic/php-runtimes.git",
"reference": "d633a896c4e5c20fd166f4b5461a869b0db2616e"
"reference": "147e76f4a72bc925d9d1613494417d583bd5de34"
},
"require": {
"php": ">=8.0",
@ -155,7 +155,7 @@
"php",
"runtimes"
],
"time": "2021-09-02T09:13:23+00:00"
"time": "2021-09-06T14:46:02+00:00"
},
{
"name": "chillerlan/php-qrcode",
@ -4920,6 +4920,7 @@
"type": "github"
}
],
"abandoned": true,
"time": "2020-09-28T06:45:17+00:00"
},
{

4
docker-compose.yml
View file

@ -368,7 +368,7 @@ services:
entrypoint:
- php
- -e
- app/executor.php
- /usr/src/code/app/executor.php
- -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
stop_signal: SIGINT
ports:
@ -376,7 +376,7 @@ services:
build:
context: .
args:
- DEBUG=false
- DEBUG=true
- TESTING=true
- VERSION=dev
networks:

2
tests/e2e/Services/Functions/FunctionsCustomClientTest.php
View file

@ -186,7 +186,7 @@ class FunctionsCustomClientTest extends Scope
$this->assertEquals(201, $execution['headers']['status-code']);
$executionId = $execution['body']['$id'] ?? '';
sleep(10);
$executions = $this->client->call(Client::METHOD_GET, '/functions/'.$functionId.'/executions/'.$executionId, [

3
tests/e2e/Services/Functions/FunctionsCustomServerTest.php
View file

@ -492,6 +492,9 @@ class FunctionsCustomServerTest extends Scope
]);
$this->assertEquals(200, $tag['headers']['status-code']);
// Allow build step to run
sleep(20);
$execution = $this->client->call(Client::METHOD_POST, '/functions/'.$functionId.'/executions', array_merge([
'content-type' => 'application/json',

BIN
tests/resources/functions/php-fn.tar.gz
View file

Binary file not shown.

1
tests/resources/functions/php-fn/index.php
View file

@ -13,5 +13,6 @@ return function ($request, $response) {
'APPWRITE_FUNCTION_DATA' => $request->env['APPWRITE_FUNCTION_DATA'],
'APPWRITE_FUNCTION_USER_ID' => $request->env['APPWRITE_FUNCTION_USER_ID'],
'APPWRITE_FUNCTION_JWT' => $request->env['APPWRITE_FUNCTION_JWT'],
'APPWRITE_FUNCTION_PROJECT_ID' => $request->env['APPWRITE_FUNCTION_PROJECT_ID'],
]);
};

BIN
tests/resources/functions/tests/resources/functions/packages/php-fn/code.tar.gz Normal file
View file

Binary file not shown.