1
0
Fork 0
mirror of synced 2024-05-17 11:12:41 +12:00

removed some unnecessary files

This commit is contained in:
shimon 2023-01-31 16:50:52 +02:00
parent be65f89eff
commit e278732481
21 changed files with 95 additions and 7884 deletions

View file

@ -305,7 +305,6 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/realtime && \
chmod +x /usr/local/bin/executor && \
chmod +x /usr/local/bin/schedule && \
chmod +x /usr/local/bin/sdks && \
chmod +x /usr/local/bin/specs && \

@ -1 +1 @@
Subproject commit af3d741ae8f02c2e16b8b4ea4664a3f8970290fd
Subproject commit 184ffdc0225afa33772a71ab25dfaef2ad8b386e

View file

@ -1,802 +0,0 @@
<?php
require_once __DIR__ . '/../vendor/autoload.php';
use Appwrite\Runtimes\Runtimes;
use Swoole\ConnectionPool;
use Swoole\Http\Request as SwooleRequest;
use Swoole\Http\Response as SwooleResponse;
use Swoole\Http\Server;
use Swoole\Process;
use Swoole\Runtime;
use Swoole\Timer;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Orchestration\Adapter\DockerCLI;
use Utopia\Orchestration\Orchestration;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Wasabi;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Storage;
use Utopia\Swoole\Request;
use Utopia\Swoole\Response;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Boolean;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL);
/** Constants */
const MAINTENANCE_INTERVAL = 3600; // 3600 seconds = 1 hour
/**
* Create a Swoole table to store runtime information
*/
$activeRuntimes = new Swoole\Table(1024);
$activeRuntimes->column('id', Swoole\Table::TYPE_STRING, 256);
$activeRuntimes->column('created', Swoole\Table::TYPE_INT, 8);
$activeRuntimes->column('updated', Swoole\Table::TYPE_INT, 8);
$activeRuntimes->column('name', Swoole\Table::TYPE_STRING, 128);
$activeRuntimes->column('status', Swoole\Table::TYPE_STRING, 128);
$activeRuntimes->column('key', Swoole\Table::TYPE_STRING, 256);
$activeRuntimes->create();
/**
* Create orchestration pool
*/
$orchestrationPool = new ConnectionPool(function () {
$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
$orchestration = new Orchestration(new DockerCLI($dockerUser, $dockerPass));
return $orchestration;
}, 10);
/**
* Create logger instance
*/
$providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
$logger = null;
if (!empty($providerName) && !empty($providerConfig) && Logger::hasProvider($providerName)) {
$classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
$adapter = new $classname($providerConfig);
$logger = new Logger($adapter);
}
function logError(Throwable $error, string $action, Utopia\Route $route = null)
{
global $logger;
if ($logger) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$log = new Log();
$log->setNamespace("executor");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
if ($route) {
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
}
$log->addTag('code', $error->getCode());
$log->addTag('verboseType', get_class($error));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Executor log pushed with status code: ' . $responseCode);
}
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $error->getMessage());
Console::error('[Error] File: ' . $error->getFile());
Console::error('[Error] Line: ' . $error->getLine());
}
function getStorageDevice($root): Device
{
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
App::post('/v1/runtimes')
->desc("Create a new runtime server")
->param('runtimeId', '', new Text(64), 'Unique runtime ID.')
->param('source', '', new Text(0), 'Path to source files.')
->param('destination', '', new Text(0), 'Destination folder to store build files into.', true)
->param('vars', [], new Assoc(), 'Environment Variables required for the build.')
->param('commands', [], new ArrayList(new Text(1024), 100), 'Commands required to build the container. Maximum of 100 commands are allowed, each 1024 characters long.')
->param('runtime', '', new Text(128), 'Runtime for the cloud function.')
->param('baseImage', '', new Text(128), 'Base image name of the runtime.')
->param('entrypoint', '', new Text(256), 'Entrypoint of the code file.', true)
->param('remove', false, new Boolean(), 'Remove a runtime after execution.')
->param('workdir', '', new Text(256), 'Working directory.', true)
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) {
if ($activeRuntimes->exists($runtimeId)) {
if ($activeRuntimes->get($runtimeId)['status'] == 'pending') {
throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500);
}
throw new Exception('Runtime already exists.', 409);
}
$container = [];
$containerId = '';
$stdout = '';
$stderr = '';
$startTime = DateTime::now();
$startTimeUnix = (new \DateTime($startTime))->getTimestamp();
$endTimeUnix = 0;
$orchestration = $orchestrationPool->get();
$secret = \bin2hex(\random_bytes(16));
if (!$remove) {
$activeRuntimes->set($runtimeId, [
'id' => $containerId,
'name' => $runtimeId,
'created' => $startTimeUnix,
'updated' => $endTimeUnix,
'status' => 'pending',
'key' => $secret,
]);
}
try {
Console::info('Building container : ' . $runtimeId);
/**
* Temporary file paths in the executor
*/
$tmpSource = "/tmp/$runtimeId/src/code.tar.gz";
$tmpBuild = "/tmp/$runtimeId/builds/code.tar.gz";
/**
* Copy code files from source to a temporary location on the executor
*/
$sourceDevice = getStorageDevice("/");
$localDevice = new Local();
$buffer = $sourceDevice->read($source);
if (!$localDevice->write($tmpSource, $buffer)) {
throw new Exception('Failed to copy source code to temporary directory', 500);
};
/**
* Create the mount folder
*/
if (!\file_exists(\dirname($tmpBuild))) {
if (!@\mkdir(\dirname($tmpBuild), 0755, true)) {
throw new Exception("Failed to create temporary directory", 500);
}
}
/**
* Create container
*/
$vars = \array_merge($vars, [
'INTERNAL_RUNTIME_KEY' => $secret,
'INTERNAL_RUNTIME_ENTRYPOINT' => $entrypoint,
]);
$vars = array_map(fn ($v) => strval($v), $vars);
$orchestration
->setCpus((int) App::getEnv('_APP_FUNCTIONS_CPUS', 0))
->setMemory((int) App::getEnv('_APP_FUNCTIONS_MEMORY', 0))
->setSwap((int) App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 0));
/** Keep the container alive if we have commands to be executed */
$entrypoint = !empty($commands) ? [
'tail',
'-f',
'/dev/null'
] : [];
$containerId = $orchestration->run(
image: $baseImage,
name: $runtimeId,
hostname: $runtimeId,
vars: $vars,
command: $entrypoint,
labels: [
'openruntimes-id' => $runtimeId,
'openruntimes-type' => 'runtime',
'openruntimes-created' => strval($startTimeUnix),
'openruntimes-runtime' => $runtime,
],
workdir: $workdir,
volumes: [
\dirname($tmpSource) . ':/tmp:rw',
\dirname($tmpBuild) . ':/usr/code:rw'
]
);
if (empty($containerId)) {
throw new Exception('Failed to create build container', 500);
}
$orchestration->networkConnect($runtimeId, App::getEnv('OPEN_RUNTIMES_NETWORK', 'appwrite_runtimes'));
/**
* Execute any commands if they were provided
*/
if (!empty($commands)) {
$status = $orchestration->execute(
name: $runtimeId,
command: $commands,
stdout: $stdout,
stderr: $stderr,
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
);
if (!$status) {
throw new Exception('Failed to build dependenices ' . $stderr, 500);
}
}
/**
* Move built code to expected build directory
*/
if (!empty($destination)) {
// Check if the build was successful by checking if file exists
if (!\file_exists($tmpBuild)) {
throw new Exception('Something went wrong during the build process', 500);
}
$destinationDevice = getStorageDevice($destination);
$outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$buffer = $localDevice->read($tmpBuild);
if (!$destinationDevice->write($outputPath, $buffer, $localDevice->getFileMimeType($tmpBuild))) {
throw new Exception('Failed to move built code to storage', 500);
};
$container['outputPath'] = $outputPath;
}
if (empty($stdout)) {
$stdout = 'Build Successful!';
}
$endTime = DateTime::now();
$endTimeUnix = (new \DateTime($endTime))->getTimestamp();
$duration = $endTimeUnix - $startTimeUnix;
$container = array_merge($container, [
'status' => 'ready',
'response' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB
'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB
'startTime' => $startTime,
'endTime' => $endTime,
'duration' => $duration,
]);
if (!$remove) {
$activeRuntimes->set($runtimeId, [
'id' => $containerId,
'name' => $runtimeId,
'created' => $startTimeUnix,
'updated' => $endTimeUnix,
'status' => 'Up ' . \round($duration, 2) . 's',
'key' => $secret,
]);
}
Console::success('Build Stage completed in ' . ($duration) . ' seconds');
} catch (Throwable $th) {
Console::error('Build failed: ' . $th->getMessage() . $stdout);
throw new Exception($th->getMessage() . $stdout, 500);
} finally {
// Container cleanup
if ($remove) {
if (!empty($containerId)) {
// If container properly created
$orchestration->remove($containerId, true);
$activeRuntimes->del($runtimeId);
} else {
// If whole creation failed, but container might have been initialized
try {
// Try to remove with contaier name instead of ID
$orchestration->remove($runtimeId, true);
$activeRuntimes->del($runtimeId);
} catch (Throwable $th) {
// If fails, means initialization also failed.
// Contianer is not there, no need to remove
}
}
}
// Release orchestration back to pool, we are done with it
$orchestrationPool->put($orchestration);
}
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->json($container);
});
App::get('/v1/runtimes')
->desc("List currently active runtimes")
->inject('activeRuntimes')
->inject('response')
->action(function ($activeRuntimes, Response $response) {
$runtimes = [];
foreach ($activeRuntimes as $runtime) {
$runtimes[] = $runtime;
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->json($runtimes);
});
App::get('/v1/runtimes/:runtimeId')
->desc("Get a runtime by its ID")
->param('runtimeId', '', new Text(64), 'Runtime unique ID.')
->inject('activeRuntimes')
->inject('response')
->action(function ($runtimeId, $activeRuntimes, Response $response) {
if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found', 404);
}
$runtime = $activeRuntimes->get($runtimeId);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->json($runtime);
});
App::delete('/v1/runtimes/:runtimeId')
->desc('Delete a runtime')
->param('runtimeId', '', new Text(64), 'Runtime unique ID.', false)
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) {
if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found', 404);
}
Console::info('Deleting runtime: ' . $runtimeId);
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($runtimeId, true);
$activeRuntimes->del($runtimeId);
Console::success('Removed runtime container: ' . $runtimeId);
} finally {
$orchestrationPool->put($orchestration);
}
// Remove all the build containers with that same ID
// TODO:: Delete build containers
// foreach ($buildIds as $buildId) {
// try {
// Console::info('Deleting build container : ' . $buildId);
// $status = $orchestration->remove('build-' . $buildId, true);
// } catch (Throwable $th) {
// Console::error($th->getMessage());
// }
// }
$response
->setStatusCode(Response::STATUS_CODE_OK)
->send();
});
App::post('/v1/execution')
->desc('Create an execution')
->param('runtimeId', '', new Text(64), 'The runtimeID to execute.')
->param('vars', [], new Assoc(), 'Environment variables required for the build.')
->param('data', '', new Text(8192), 'Data to be forwarded to the function, this is user specified.', true)
->param('timeout', 15, new Range(1, (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)), 'Function maximum execution time in seconds.')
->inject('activeRuntimes')
->inject('response')
->action(
function (string $runtimeId, array $vars, string $data, $timeout, $activeRuntimes, Response $response) {
if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found. Please create the runtime.', 404);
}
for ($i = 0; $i < 5; $i++) {
if ($activeRuntimes->get($runtimeId)['status'] === 'pending') {
Console::info('Waiting for runtime to be ready...');
sleep(1);
} else {
break;
}
if ($i === 4) {
throw new Exception('Runtime failed to launch in allocated time.', 500);
}
}
$runtime = $activeRuntimes->get($runtimeId);
$secret = $runtime['key'];
if (empty($secret)) {
throw new Exception('Runtime secret not found. Please re-create the runtime.', 500);
}
Console::info('Executing Runtime: ' . $runtimeId);
$execution = [];
$executionStart = \microtime(true);
$stdout = '';
$stderr = '';
$res = '';
$statusCode = 0;
$errNo = -1;
$executorResponse = '';
$timeout ??= (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900);
$ch = \curl_init();
$body = \json_encode([
'variables' => $vars,
'payload' => $data,
'timeout' => $timeout
]);
\curl_setopt($ch, CURLOPT_URL, "http://" . $runtimeId . ":3000/");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'Content-Length: ' . \strlen($body),
'x-internal-challenge: ' . $secret,
'host: null'
]);
$executorResponse = \curl_exec($ch);
$executorResponse = json_decode($executorResponse, true);
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
$error = \curl_error($ch);
$errNo = \curl_errno($ch);
\curl_close($ch);
switch (true) {
/** No Error. */
case $errNo === 0:
break;
/** Runtime not ready for requests yet. 111 is the swoole error code for Connection Refused - see https://openswoole.com/docs/swoole-error-code */
case $errNo === 111:
throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 406);
/** Any other CURL error */
default:
throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500);
}
switch (true) {
case $statusCode >= 500:
$stderr = ($executorResponse ?? [])['stderr'] ?? 'Internal Runtime error.';
$stdout = ($executorResponse ?? [])['stdout'] ?? 'Internal Runtime error.';
break;
case $statusCode >= 100:
$stdout = $executorResponse['stdout'];
$res = $executorResponse['response'];
if (is_array($res)) {
$res = json_encode($res, JSON_UNESCAPED_UNICODE);
}
break;
default:
$stderr = ($executorResponse ?? [])['stderr'] ?? 'Execution failed.';
$stdout = ($executorResponse ?? [])['stdout'] ?? '';
break;
}
$executionEnd = \microtime(true);
$executionTime = ($executionEnd - $executionStart);
$functionStatus = ($statusCode >= 500) ? 'failed' : 'completed';
Console::success('Function executed in ' . $executionTime . ' seconds, status: ' . $functionStatus);
$execution = [
'status' => $functionStatus,
'statusCode' => $statusCode,
'response' => \mb_strcut($res, 0, 1000000), // Limit to 1MB
'stdout' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB
'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB
'duration' => $executionTime,
];
/** Update swoole table */
$runtime['updated'] = \time();
$activeRuntimes->set($runtimeId, $runtime);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->json($execution);
}
);
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
$http = new Server("0.0.0.0", 80);
/** Set Resources */
App::setResource('orchestrationPool', fn() => $orchestrationPool);
App::setResource('activeRuntimes', fn() => $activeRuntimes);
/** Set callbacks */
App::error()
->inject('utopia')
->inject('error')
->inject('request')
->inject('response')
->action(function (App $utopia, throwable $error, Request $request, Response $response) {
$route = $utopia->match($request);
logError($error, "httpError", $route);
switch ($error->getCode()) {
case 400: // Error allowed publicly
case 401: // Error allowed publicly
case 402: // Error allowed publicly
case 403: // Error allowed publicly
case 404: // Error allowed publicly
case 406: // Error allowed publicly
case 409: // Error allowed publicly
case 412: // Error allowed publicly
case 425: // Error allowed publicly
case 429: // Error allowed publicly
case 501: // Error allowed publicly
case 503: // Error allowed publicly
$code = $error->getCode();
break;
default:
$code = 500; // All other errors get the generic 500 server error status code
}
$output = [
'message' => $error->getMessage(),
'code' => $error->getCode(),
'file' => $error->getFile(),
'line' => $error->getLine(),
'trace' => $error->getTrace(),
'version' => App::getEnv('_APP_VERSION', 'UNKNOWN')
];
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->setStatusCode($code);
$response->json($output);
});
App::init()
->inject('request')
->action(function (Request $request) {
$secretKey = $request->getHeader('x-appwrite-executor-key', '');
if (empty($secretKey)) {
throw new Exception('Missing executor key', 401);
}
if ($secretKey !== App::getEnv('_APP_EXECUTOR_SECRET', '')) {
throw new Exception('Missing executor key', 401);
}
});
$http->on('start', function ($http) {
global $orchestrationPool;
global $activeRuntimes;
/**
* Warmup: make sure images are ready to run fast 🚀
*/
$runtimes = new Runtimes('v2');
$allowList = empty(App::getEnv('_APP_FUNCTIONS_RUNTIMES')) ? [] : \explode(',', App::getEnv('_APP_FUNCTIONS_RUNTIMES'));
$runtimes = $runtimes->getAll(true, $allowList);
foreach ($runtimes as $runtime) {
go(function () use ($runtime, $orchestrationPool) {
try {
$orchestration = $orchestrationPool->get();
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
$response = $orchestration->pull($runtime['image']);
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
} else {
Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
}
} catch (\Throwable $th) {
} finally {
$orchestrationPool->put($orchestration);
}
});
}
/**
* Remove residual runtimes
*/
Console::info('Removing orphan runtimes...');
try {
$orchestration = $orchestrationPool->get();
$orphans = $orchestration->list(['label' => 'openruntimes-type=runtime']);
} finally {
$orchestrationPool->put($orchestration);
}
foreach ($orphans as $runtime) {
go(function () use ($runtime, $orchestrationPool) {
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($runtime->getName(), true);
Console::success("Successfully removed {$runtime->getName()}");
} catch (\Throwable $th) {
Console::error('Orphan runtime deletion failed: ' . $th->getMessage());
} finally {
$orchestrationPool->put($orchestration);
}
});
}
/**
* Register handlers for shutdown
*/
@Process::signal(SIGINT, function () use ($http) {
$http->shutdown();
});
@Process::signal(SIGQUIT, function () use ($http) {
$http->shutdown();
});
@Process::signal(SIGKILL, function () use ($http) {
$http->shutdown();
});
@Process::signal(SIGTERM, function () use ($http) {
$http->shutdown();
});
/**
* Run a maintenance worker every MAINTENANCE_INTERVAL seconds to remove inactive runtimes
*/
Timer::tick(MAINTENANCE_INTERVAL * 1000, function () use ($orchestrationPool, $activeRuntimes) {
Console::warning("Running maintenance task ...");
foreach ($activeRuntimes as $runtime) {
$inactiveThreshold = \time() - App::getEnv('_APP_FUNCTIONS_INACTIVE_THRESHOLD', 60);
if ($runtime['updated'] < $inactiveThreshold) {
go(function () use ($runtime, $orchestrationPool, $activeRuntimes) {
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($runtime['name'], true);
$activeRuntimes->del($runtime['name']);
Console::success("Successfully removed {$runtime['name']}");
} catch (\Throwable $th) {
Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());
} finally {
$orchestrationPool->put($orchestration);
}
});
}
}
});
});
$http->on('beforeShutdown', function () {
global $orchestrationPool;
Console::info('Cleaning up containers before shutdown...');
$orchestration = $orchestrationPool->get();
$functionsToRemove = $orchestration->list(['label' => 'openruntimes-type=runtime']);
$orchestrationPool->put($orchestration);
foreach ($functionsToRemove as $container) {
go(function () use ($orchestrationPool, $container) {
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($container->getId(), true);
Console::info('Removed container ' . $container->getName());
} catch (\Throwable $th) {
Console::error('Failed to remove container: ' . $container->getName());
} finally {
$orchestrationPool->put($orchestration);
}
});
}
});
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) {
$request = new Request($swooleRequest);
$response = new Response($swooleResponse);
$app = new App('UTC');
try {
$app->run($request, $response);
} catch (\Throwable $th) {
logError($th, "serverError");
$swooleResponse->setStatusCode(500);
$output = [
'message' => 'Error: ' . $th->getMessage(),
'code' => 500,
'file' => $th->getFile(),
'line' => $th->getLine(),
'trace' => $th->getTrace()
];
$swooleResponse->end(\json_encode($output));
}
});
$http->start();

View file

@ -1,3 +0,0 @@
#!/usr/bin/env sh
/usr/src/code/vendor/bin/phpbench run --config /usr/src/code/phpbench.json --report appwrite $@

View file

@ -1,3 +0,0 @@
#!/bin/sh
php -e /usr/src/code/app/executor.php -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php

View file

@ -99,14 +99,7 @@
"phpunit/phpunit": "9.5.20",
"squizlabs/php_codesniffer": "^3.6",
"swoole/ide-helper": "4.8.9",
"textalk/websocket": "1.5.7",
"phpbench/phpbench": "^1.2",
"symfony/console": "^5.0",
"symfony/filesystem": "^5.0",
"symfony/finder": "^5.0",
"symfony/options-resolver": "^5.0",
"symfony/process": "^5.0"
"textalk/websocket": "1.5.7"
},
"provide": {
"ext-phpiredis": "*"

6489
composer.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,17 +0,0 @@
{
"$schema": "./vendor/phpbench/phpbench/phpbench.schema.json",
"runner.path": "tests/benchmarks",
"runner.file_pattern": "*Bench.php",
"runner.bootstrap": "app/init.php",
"runner.revs": 1000,
"runner.iterations": 3,
"runner.retry_threshold": 5,
"runner.warmup": 1,
"report.generators": {
"appwrite": {
"extends": "aggregate",
"cols": ["benchmark", "subject", "set" ,"revs", "its", "worst", "best", "mean", "mode", "rstdev"],
"break": ["benchmark"]
}
}
}

View file

@ -1,14 +0,0 @@
<?php
namespace Tests\Benchmarks\Scopes;
use PhpBench\Attributes\AfterMethods;
use PhpBench\Attributes\BeforeMethods;
use Tests\E2E\Scopes\Scope as E2EScope;
#[BeforeMethods(['setUp'])]
#[AfterMethods(['tearDown'])]
abstract class Scope extends E2EScope
{
protected $endpoint = 'http://localhost/v1';
}

View file

@ -1,151 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Databases;
use PhpBench\Attributes\BeforeMethods;
use PhpBench\Attributes\ParamProviders;
use Tests\Benchmarks\Scopes\Scope;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
abstract class Base extends Scope
{
use ProjectCustom;
protected static string $databaseId;
protected static string $collectionId;
protected static string $documentId;
#[BeforeMethods(['createDatabase', 'createCollection'])]
public function benchDocumentCreate()
{
$this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'documentId' => ID::unique(),
'data' => [
'title' => 'The Matrix',
],
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
Permission::write(Role::user($this->getUser()['$id'])),
],
]);
}
#[ParamProviders(['provideCounts'])]
#[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
public function benchDocumentReadList(array $params)
{
$this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => ['limit(' . $params['documents'] . ')'],
]);
}
#[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
public function benchDocumentRead()
{
$this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
}
#[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
public function benchDocumentUpdate()
{
$this->client->call(Client::METHOD_PATCH, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'data' => [
'title' => 'The Matrix Reloaded',
],
]);
}
public function provideCounts(): array
{
return [
'1 Document' => ['documents' => 1],
'10 Documents' => ['documents' => 10],
'100 Documents' => ['documents' => 100],
];
}
public function createDatabase(array $params = [])
{
$database = $this->client->call(Client::METHOD_POST, '/databases', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'databaseId' => ID::unique(),
'name' => 'Test Database'
]);
static::$databaseId = $database['body']['$id'];
}
public function createCollection(array $params = [])
{
$collection = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'collectionId' => ID::unique(),
'name' => 'Movies',
'documentSecurity' => true,
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
Permission::write(Role::user($this->getUser()['$id'])),
],
]);
static::$collectionId = $collection['body']['$id'];
// Create attribute
$this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/attributes/string', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'key' => 'title',
'size' => 256,
'required' => true,
]);
// Wait for attribute to be ready
sleep(2);
}
public function createDocuments(array $params = [])
{
$count = $params['documents'] ?? 1;
for ($i = 0; $i < $count; $i++) {
$response = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'documentId' => ID::unique(),
'data' => [
'title' => 'Captain America' . $i,
],
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
Permission::write(Role::user($this->getUser()['$id'])),
]
]);
static::$documentId = $response['body']['$id'];
}
}
}

View file

@ -1,10 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Databases;
use Tests\E2E\Scopes\SideClient;
class DatabasesCustomClientBench extends Base
{
use SideClient;
}

View file

@ -1,10 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Databases;
use Tests\E2E\Scopes\SideServer;
class DatabasesCustomServerBench extends Base
{
use SideServer;
}

View file

@ -1,113 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Functions;
use CURLFile;
use PhpBench\Attributes\BeforeMethods;
use Tests\Benchmarks\Scopes\Scope;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Utopia\CLI\Console;
use Utopia\Database\ID;
use Utopia\Database\Role;
abstract class Base extends Scope
{
use ProjectCustom;
protected static string $functionId;
protected static string $deploymentId;
protected static string $executionId;
#[BeforeMethods(['createFunction', 'prepareDeployment', 'createDeployment', 'patchDeployment'])]
public function benchExecutionCreate()
{
$this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/executions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
}
public function createFunction()
{
$response = $this->client->call(Client::METHOD_POST, '/functions', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'functionId' => ID::unique(),
'name' => 'Test',
'runtime' => 'php-8.0',
'timeout' => 10,
'execute' => [Role::users()->toString()]
]);
static::$functionId = $response['body']['$id'];
}
public function prepareDeployment()
{
$stdout = '';
$stderr = '';
Console::execute(
'cd ' . realpath(__DIR__ . "/../../../resources/functions/php") . " && \
tar --exclude code.tar.gz -czf code.tar.gz .",
'',
$stdout,
$stderr
);
}
public function createDeployment()
{
$code = realpath(__DIR__ . '/../../../resources/functions/php/code.tar.gz');
$response = $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/deployments', [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'entrypoint' => 'index.php',
'code' => new CURLFile(
$code,
'application/x-gzip',
\basename($code)
),
]);
static::$deploymentId = $response['body']['$id'];
while (true) {
$response = $this->client->call(Client::METHOD_GET, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]);
$status = $response['body']['status'] ?? '';
switch ($status) {
case '':
case 'processing':
case 'building':
usleep(200);
break;
case 'ready':
break 2;
case 'failed':
throw new \Exception('Failed to build function');
}
}
sleep(1);
}
public function patchDeployment()
{
$this->client->call(Client::METHOD_PATCH, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], []);
}
}

View file

@ -1,10 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Functions;
use Tests\E2E\Scopes\SideClient;
class FunctionsCustomClientBench extends Base
{
use SideClient;
}

View file

@ -1,21 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Functions;
use PhpBench\Attributes\BeforeMethods;
use PhpBench\Attributes\Iterations;
use PhpBench\Attributes\Revs;
use Tests\E2E\Scopes\SideServer;
class FunctionsCustomServerBench extends Base
{
use SideServer;
#[Revs(1)]
#[Iterations(1)]
#[BeforeMethods(['createFunction', 'prepareDeployment'])]
public function benchDeploymentCreate()
{
$this->createDeployment();
}
}

View file

@ -1,117 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Storage;
use CURLFile;
use PhpBench\Attributes\BeforeMethods;
use PhpBench\Attributes\ParamProviders;
use Tests\Benchmarks\Scopes\Scope;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
abstract class Base extends Scope
{
use ProjectCustom;
protected static string $bucketId;
protected static string $fileId;
#[BeforeMethods(['createBucket'])]
public function benchFileCreate()
{
$this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => ID::unique(),
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
Permission::write(Role::user($this->getUser()['$id'])),
],
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'),
]);
}
#[ParamProviders(['provideCounts'])]
#[BeforeMethods(['createBucket', 'createFiles'])]
public function benchFileReadList(array $params)
{
$this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => ['limit(' . $params['files'] . ')'],
]);
}
#[BeforeMethods(['createBucket', 'createFiles'])]
public function benchFileRead()
{
$this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
}
#[BeforeMethods(['createBucket', 'createFiles'])]
public function benchFileUpdate()
{
$this->client->call(Client::METHOD_PUT, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Updated name',
'permissions' => [],
]);
}
public function provideCounts(): array
{
return [
'10 Files' => ['files' => 10],
'100 Files' => ['files' => 100],
];
}
public function createBucket(array $params = [])
{
// Create bucket
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket',
'permissions' => [
Permission::read(Role::user($this->getUser()['$id'])),
Permission::write(Role::user($this->getUser()['$id'])),
],
'fileSecurity' => true
]);
static::$bucketId = $bucket['body']['$id'];
}
public function createFiles(array $params = [])
{
$count = $params['files'] ?? 1;
// Create files
for ($i = 0; $i < $count; $i++) {
$response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'fileId' => ID::unique(),
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'),
]);
static::$fileId = $response['body']['$id'];
}
}
}

View file

@ -1,10 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Storage;
use Tests\E2E\Scopes\SideClient;
class StorageCustomClientBench extends Base
{
use SideClient;
}

View file

@ -1,10 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Storage;
use Tests\E2E\Scopes\SideServer;
class StorageCustomServerBench extends Base
{
use SideServer;
}

View file

@ -1,94 +0,0 @@
<?php
namespace Tests\Benchmarks\Services\Users;
use PhpBench\Attributes\BeforeMethods;
use PhpBench\Attributes\ParamProviders;
use Tests\Benchmarks\Scopes\Scope;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideServer;
use Utopia\Database\ID;
class UserCustomServerBench extends Scope
{
use ProjectCustom;
use SideServer;
protected static string $userId;
public function benchUserCreate()
{
$id = ID::unique();
$this->client->call(Client::METHOD_POST, '/users', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'userId' => $id,
'email' => 'test' . $id . '@example.com',
'password' => 'password',
]);
}
#[ParamProviders(['provideCounts'])]
#[BeforeMethods(['createUsers'])]
public function benchUserReadList(array $params)
{
$this->client->call(Client::METHOD_GET, '/users', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => ['limit(' . $params['users'] . ')'],
]);
}
#[BeforeMethods(['createUsers'])]
public function benchUserRead()
{
$this->client->call(Client::METHOD_GET, '/users/' . static::$userId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
}
#[BeforeMethods(['createUsers'])]
public function benchUserUpdate()
{
$this->client->call(Client::METHOD_PUT, '/users/' . static::$userId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'New Name',
]);
}
public function createUsers(array $params = [])
{
$count = $params['documents'] ?? 1;
for ($i = 0; $i < $count; $i++) {
$id = ID::unique();
$response = $this->client->call(Client::METHOD_POST, '/users', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'userId' => $id,
'email' => 'test' . $id . '@example.com',
'password' => 'password',
]);
static::$userId = $response['body']['$id'];
}
}
public function provideCounts(): array
{
return [
'1 User' => ['users' => 1],
'10 Users' => ['users' => 10],
'100 Users' => ['users' => 100],
];
}
}

34
tests/benchmarks/http.js Normal file
View file

@ -0,0 +1,34 @@
import http from 'k6/http';
import { check } from 'k6';
import { Counter } from 'k6/metrics';
// A simple counter for http requests
export const requests = new Counter('http_reqs');
// you can specify stages of your test (ramp up/down patterns) through the options object
// target is the number of VUs you are aiming for
export const options = {
stages: [
{ target: 50, duration: '1m' },
// { target: 15, duration: '1m' },
// { target: 0, duration: '1m' },
],
thresholds: {
requests: ['count < 100'],
},
};
export default function () {
const config = {
headers: {
'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695',
'X-Appwrite-Project': '60479fe35d95d'
}}
const resDb = http.get('http://localhost:9501/', config);
check(resDb, {
'status is 200': (r) => r.status === 200,
});
}

59
tests/benchmarks/ws.js Normal file
View file

@ -0,0 +1,59 @@
// k6 run tests/benchmarks/ws.js
import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js';
import ws from 'k6/ws';
import { check } from 'k6';
export let options = {
stages: [
{
duration: '10s',
target: 500
},
{
duration: '1m',
target: 500
},
],
}
export default function () {
// const url = new URL('wss://appwrite-realtime.monitor-api.com/v1/realtime');
// url.searchParams.append('project', '604249e6b1a9f');
const url = new URL('ws://localhost/v1/realtime');
url.searchParams.append('project', 'console');
url.searchParams.append('channels[]', 'files');
const res = ws.connect(url.toString(), function (socket) {
let connection = false;
let checked = false;
let payload = null;
socket.on('open', () => {
connection = true;
});
socket.on('message', (data) => {
payload = data;
checked = true;
});
socket.setTimeout(function () {
check(payload, {
'connection opened': (r) => connection,
'message received': (r) => checked,
'channels are right': (r) => r === JSON.stringify({
"type": "connected",
"data": {
"channels": [
"files"
],
"user": null
}
})
})
socket.close();
}, 5000);
});
check(res, { 'status is 101': (r) => r && r.status === 101 });
}