build && audit && usage workers
This commit is contained in:
parent
dcf0107a79
commit
a10246a9bd
4 changed files with 636 additions and 132 deletions
|
@ -218,8 +218,6 @@ if (isset($args[0])) {
|
|||
Console::error('Missing worker name');
|
||||
}
|
||||
|
||||
if (empty(App::getEnv('QUEUE'))) {
|
||||
throw new Exception('Please configure "QUEUE" environment variable.');
|
||||
try {
|
||||
$platform->init(Service::TYPE_WORKER, [
|
||||
'workersNum' => swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)),
|
||||
|
@ -251,9 +249,6 @@ $worker
|
|||
throw $error;
|
||||
}
|
||||
|
||||
if (($error->getCode() >= 500 || $error->getCode() === 0) && !empty($logger)) {
|
||||
$log = new Log();
|
||||
|
||||
if ($logger && ($error->getCode() >= 500 || $error->getCode() === 0)) {
|
||||
$log->setNamespace("appwrite-worker");
|
||||
$log->setServer(\gethostname());
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Exception;
|
||||
use Utopia\App;
|
||||
use Throwable;
|
||||
use Utopia\Audit\Audit;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
|
@ -32,6 +32,7 @@ class Audits extends Action
|
|||
|
||||
/**
|
||||
* @throws Exception
|
||||
* @throws Throwable
|
||||
*/
|
||||
public function action(Message $message, Database $dbForProject): void
|
||||
{
|
||||
|
@ -55,8 +56,7 @@ class Audits extends Action
|
|||
|
||||
$audit = new Audit($dbForProject);
|
||||
$audit->log(
|
||||
userInternalId: $user->getInternalId(),
|
||||
userId: $user->getId(),
|
||||
userId: $user->getInternalId(),
|
||||
// Pass first, most verbose event pattern
|
||||
event: $event,
|
||||
resource: $resource,
|
||||
|
|
|
@ -7,21 +7,26 @@ use Appwrite\Event\Func;
|
|||
use Appwrite\Event\Usage;
|
||||
use Appwrite\Messaging\Adapter\Realtime;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use Appwrite\Vcs\Comment;
|
||||
use Exception;
|
||||
use Swoole\Coroutine as Co;
|
||||
use Executor\Executor;
|
||||
use Utopia\App;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Authorization;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Exception\Structure;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\DSN\DSN;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Storage\Device\Local;
|
||||
use Utopia\Storage\Storage;
|
||||
use Utopia\VCS\Adapter\Git\GitHub;
|
||||
|
||||
class Builds extends Action
|
||||
{
|
||||
|
@ -43,13 +48,15 @@ class Builds extends Action
|
|||
->inject('queueForEvents')
|
||||
->inject('queueForFunctions')
|
||||
->inject('queueForUsage')
|
||||
->callback(fn($message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage) => $this->action($message, $dbForConsole, $dbForProject, $queueForEvents, $queueForFunctions, $queueForUsage));
|
||||
->inject('cache')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn($message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, callable $getProjectDB) => $this->action($message, $dbForConsole, $dbForProject, $queueForEvents, $queueForFunctions, $queueForUsage, $cache, $getProjectDB));
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Exception|\Throwable
|
||||
*/
|
||||
public function action(Message $message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage): void
|
||||
public function action(Message $message, Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, callable $getProjectDB): void
|
||||
{
|
||||
$payload = $message->getPayload() ?? [];
|
||||
|
||||
|
@ -61,21 +68,14 @@ class Builds extends Action
|
|||
$project = new Document($payload['project'] ?? []);
|
||||
$resource = new Document($payload['resource'] ?? []);
|
||||
$deployment = new Document($payload['deployment'] ?? []);
|
||||
$template = new Document($payload['template'] ?? []);
|
||||
|
||||
switch ($type) {
|
||||
case BUILD_TYPE_DEPLOYMENT:
|
||||
case BUILD_TYPE_RETRY:
|
||||
Console::info('Creating build for deployment: ' . $deployment->getId());
|
||||
$this->buildDeployment(
|
||||
dbForConsole: $dbForConsole,
|
||||
dbForProject: $dbForProject,
|
||||
queueForEvents: $queueForEvents,
|
||||
queueForFunctions: $queueForFunctions,
|
||||
queueForUsage: $queueForUsage,
|
||||
deployment: $deployment,
|
||||
project: $project,
|
||||
function: $resource
|
||||
);
|
||||
$github = new GitHub($cache);
|
||||
$this->buildDeployment($queueForEvents, $queueForUsage, $getProjectDB, $dbForConsole, $github, $project, $resource, $deployment, $template);
|
||||
break;
|
||||
|
||||
default:
|
||||
|
@ -88,11 +88,13 @@ class Builds extends Action
|
|||
* @throws \Throwable
|
||||
* @throws Structure
|
||||
*/
|
||||
private function buildDeployment(Database $dbForConsole, Database $dbForProject, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Document $deployment, Document $project, Document $function): void
|
||||
protected function buildDeployment(Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, callable $getProjectDB, GitHub $github, Document $project, Document $function, Document $deployment, Document $template)
|
||||
{
|
||||
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
|
||||
$function = $dbForProject->getDocument('functions', $function->getId());
|
||||
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
$function = $dbForProject->getDocument('functions', $function->getId());
|
||||
if ($function->isEmpty()) {
|
||||
throw new Exception('Function not found', 404);
|
||||
}
|
||||
|
@ -102,27 +104,33 @@ class Builds extends Action
|
|||
throw new Exception('Deployment not found', 404);
|
||||
}
|
||||
|
||||
$runtimes = Config::getParam('runtimes', []);
|
||||
if (empty($deployment->getAttribute('entrypoint', ''))) {
|
||||
throw new Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
|
||||
}
|
||||
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
|
||||
$key = $function->getAttribute('runtime');
|
||||
$runtime = $runtimes[$key] ?? null;
|
||||
if (\is_null($runtime)) {
|
||||
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
||||
}
|
||||
|
||||
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
|
||||
/** @TODO : move this to the registry or someplace else */
|
||||
$device = Storage::DEVICE_LOCAL;
|
||||
try {
|
||||
$dsn = new DSN($connection);
|
||||
$device = $dsn->getScheme();
|
||||
} catch (\Exception $e) {
|
||||
Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
|
||||
}
|
||||
|
||||
// Realtime preparation
|
||||
$allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
|
||||
'functionId' => $function->getId(),
|
||||
'deploymentId' => $deployment->getId()
|
||||
]);
|
||||
|
||||
$startTime = DateTime::now();
|
||||
$durationStart = \microtime(true);
|
||||
|
||||
$buildId = $deployment->getAttribute('buildId', '');
|
||||
$startTime = DateTime::now();
|
||||
|
||||
if (empty($buildId)) {
|
||||
$isNewBuild = empty($buildId);
|
||||
|
||||
if ($isNewBuild) {
|
||||
$buildId = ID::unique();
|
||||
$build = $dbForProject->createDocument('builds', new Document([
|
||||
'$id' => $buildId,
|
||||
|
@ -131,28 +139,172 @@ class Builds extends Action
|
|||
'deploymentInternalId' => $deployment->getInternalId(),
|
||||
'deploymentId' => $deployment->getId(),
|
||||
'status' => 'processing',
|
||||
'outputPath' => '',
|
||||
'path' => '',
|
||||
'runtime' => $function->getAttribute('runtime'),
|
||||
'source' => $deployment->getAttribute('path'),
|
||||
'sourceType' => $device,
|
||||
'stdout' => '',
|
||||
'stderr' => '',
|
||||
'duration' => 0
|
||||
'source' => $deployment->getAttribute('path', ''),
|
||||
'sourceType' => strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)),
|
||||
'logs' => '',
|
||||
'endTime' => null,
|
||||
'duration' => 0,
|
||||
'size' => 0
|
||||
]));
|
||||
|
||||
$deployment->setAttribute('buildId', $buildId);
|
||||
$deployment->setAttribute('buildId', $build->getId());
|
||||
$deployment->setAttribute('buildInternalId', $build->getInternalId());
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
||||
} else {
|
||||
$build = $dbForProject->getDocument('builds', $buildId);
|
||||
}
|
||||
|
||||
$source = $deployment->getAttribute('path', '');
|
||||
$installationId = $deployment->getAttribute('installationId', '');
|
||||
$providerRepositoryId = $deployment->getAttribute('providerRepositoryId', '');
|
||||
$providerCommitHash = $deployment->getAttribute('providerCommitHash', '');
|
||||
$isVcsEnabled = $providerRepositoryId ? true : false;
|
||||
$owner = '';
|
||||
$repositoryName = '';
|
||||
|
||||
if ($isVcsEnabled) {
|
||||
$installation = $dbForConsole->getDocument('installations', $installationId);
|
||||
$providerInstallationId = $installation->getAttribute('providerInstallationId');
|
||||
$privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
|
||||
$githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');
|
||||
|
||||
$github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
|
||||
}
|
||||
|
||||
try {
|
||||
if ($isNewBuild && $isVcsEnabled) {
|
||||
$tmpDirectory = '/tmp/builds/' . $buildId . '/code';
|
||||
$rootDirectory = $function->getAttribute('providerRootDirectory', '');
|
||||
$rootDirectory = \rtrim($rootDirectory, '/');
|
||||
$rootDirectory = \ltrim($rootDirectory, '.');
|
||||
$rootDirectory = \ltrim($rootDirectory, '/');
|
||||
|
||||
$owner = $github->getOwnerName($providerInstallationId);
|
||||
$repositoryName = $github->getRepositoryName($providerRepositoryId);
|
||||
|
||||
$cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner);
|
||||
$cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName);
|
||||
|
||||
$branchName = $deployment->getAttribute('providerBranch');
|
||||
$commitHash = $deployment->getAttribute('providerCommitHash', '');
|
||||
$gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $branchName, $tmpDirectory, $rootDirectory, $commitHash);
|
||||
$stdout = '';
|
||||
$stderr = '';
|
||||
Console::execute('mkdir -p /tmp/builds/' . \escapeshellcmd($buildId), '', $stdout, $stderr);
|
||||
$exit = Console::execute($gitCloneCommand, '', $stdout, $stderr);
|
||||
|
||||
if ($exit !== 0) {
|
||||
throw new \Exception('Unable to clone code repository: ' . $stderr);
|
||||
}
|
||||
|
||||
// Build from template
|
||||
$templateRepositoryName = $template->getAttribute('repositoryName', '');
|
||||
$templateOwnerName = $template->getAttribute('ownerName', '');
|
||||
$templateBranch = $template->getAttribute('branch', '');
|
||||
|
||||
$templateRootDirectory = $template->getAttribute('rootDirectory', '');
|
||||
$templateRootDirectory = \rtrim($templateRootDirectory, '/');
|
||||
$templateRootDirectory = \ltrim($templateRootDirectory, '.');
|
||||
$templateRootDirectory = \ltrim($templateRootDirectory, '/');
|
||||
|
||||
if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateBranch)) {
|
||||
// Clone template repo
|
||||
$tmpTemplateDirectory = '/tmp/builds/' . \escapeshellcmd($buildId) . '/template';
|
||||
$gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateBranch, $tmpTemplateDirectory, $templateRootDirectory);
|
||||
$exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
|
||||
|
||||
if ($exit !== 0) {
|
||||
throw new \Exception('Unable to clone code repository: ' . $stderr);
|
||||
}
|
||||
|
||||
// Ensure directories
|
||||
Console::execute('mkdir -p ' . $tmpTemplateDirectory . '/' . $templateRootDirectory, '', $stdout, $stderr);
|
||||
Console::execute('mkdir -p ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
|
||||
|
||||
// Merge template into user repo
|
||||
Console::execute('cp -rfn ' . $tmpTemplateDirectory . '/' . $templateRootDirectory . '/* ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
|
||||
|
||||
// Commit and push
|
||||
$exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr);
|
||||
|
||||
if ($exit !== 0) {
|
||||
throw new \Exception('Unable to push code repository: ' . $stderr);
|
||||
}
|
||||
|
||||
$exit = Console::execute('cd ' . $tmpDirectory . ' && git rev-parse HEAD', '', $stdout, $stderr);
|
||||
|
||||
if ($exit !== 0) {
|
||||
throw new \Exception('Unable to get vcs commit SHA: ' . $stderr);
|
||||
}
|
||||
|
||||
$providerCommitHash = \trim($stdout);
|
||||
$authorUrl = "https://github.com/$cloneOwner";
|
||||
|
||||
$deployment->setAttribute('providerCommitHash', $providerCommitHash ?? '');
|
||||
$deployment->setAttribute('providerCommitAuthorUrl', $authorUrl);
|
||||
$deployment->setAttribute('providerCommitAuthor', 'Appwrite');
|
||||
$deployment->setAttribute('providerCommitMessage', "Create '" . $function->getAttribute('name', '') . "' function");
|
||||
$deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash");
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
||||
|
||||
/**
|
||||
* Send realtime Event
|
||||
*/
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $allEvents[0],
|
||||
payload: $build,
|
||||
project: $project
|
||||
);
|
||||
Realtime::send(
|
||||
projectId: 'console',
|
||||
payload: $build->getArrayCopy(),
|
||||
events: $allEvents,
|
||||
channels: $target['channels'],
|
||||
roles: $target['roles']
|
||||
);
|
||||
}
|
||||
|
||||
$tmpPath = '/tmp/builds/' . \escapeshellcmd($buildId);
|
||||
$tmpPathFile = $tmpPath . '/code.tar.gz';
|
||||
|
||||
Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr);
|
||||
|
||||
$deviceFunctions = $this->getFunctionsDevice($project->getId());
|
||||
|
||||
$localDevice = new Local();
|
||||
$buffer = $localDevice->read($tmpPathFile);
|
||||
$mimeType = $localDevice->getFileMimeType($tmpPathFile);
|
||||
|
||||
$path = $deviceFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
||||
$result = $deviceFunctions->write($path, $buffer, $mimeType);
|
||||
|
||||
if (!$result) {
|
||||
throw new \Exception("Unable to move file");
|
||||
}
|
||||
|
||||
Console::execute('rm -rf ' . $tmpPath, '', $stdout, $stderr);
|
||||
|
||||
$source = $path;
|
||||
|
||||
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
|
||||
|
||||
$this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
|
||||
}
|
||||
|
||||
/** Request the executor to build the code... */
|
||||
$build->setAttribute('status', 'building');
|
||||
$build = $dbForProject->updateDocument('builds', $buildId, $build);
|
||||
|
||||
if ($isVcsEnabled) {
|
||||
$this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
|
||||
}
|
||||
|
||||
/** Trigger Webhook */
|
||||
$deploymentUpdate = $queueForEvents
|
||||
$deploymentUpdate =
|
||||
$queueForEvents
|
||||
->setQueue(Event::WEBHOOK_QUEUE_NAME)
|
||||
->setClass(Event::WEBHOOK_CLASS_NAME)
|
||||
->setProject($project)
|
||||
|
@ -172,12 +324,7 @@ class Builds extends Action
|
|||
->from($deploymentUpdate)
|
||||
->trigger();
|
||||
|
||||
|
||||
/** Trigger Realtime */
|
||||
$allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
|
||||
'functionId' => $function->getId(),
|
||||
'deploymentId' => $deployment->getId()
|
||||
]);
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $allEvents[0],
|
||||
|
@ -193,44 +340,122 @@ class Builds extends Action
|
|||
roles: $target['roles']
|
||||
);
|
||||
|
||||
$source = $deployment->getAttribute('path');
|
||||
$vars = [];
|
||||
|
||||
$vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
|
||||
$carry[$var->getAttribute('key')] = $var->getAttribute('value');
|
||||
return $carry;
|
||||
}, []);
|
||||
// Shared vars
|
||||
foreach ($function->getAttribute('varsProject', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
|
||||
// Function vars
|
||||
foreach ($function->getAttribute('vars', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
|
||||
// Appwrite vars
|
||||
$vars = \array_merge($vars, [
|
||||
'APPWRITE_FUNCTION_ID' => $function->getId(),
|
||||
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
|
||||
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
|
||||
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
|
||||
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
|
||||
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
|
||||
]);
|
||||
|
||||
$command = $deployment->getAttribute('commands', '');
|
||||
$command = \str_replace('"', '\\"', $command);
|
||||
|
||||
$response = null;
|
||||
|
||||
$err = null;
|
||||
|
||||
// TODO: Remove run() wrapper when switching to new utopia queue. That should be done on Swoole adapter in the libary
|
||||
Co\run(function () use ($executor, $project, $deployment, &$response, $source, $function, $runtime, $vars, $command, &$build, $dbForProject, $allEvents, &$err) {
|
||||
Co::join([
|
||||
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) {
|
||||
try {
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"';
|
||||
|
||||
$response = $executor->createRuntime(
|
||||
deploymentId: $deployment->getId(),
|
||||
projectId: $project->getId(),
|
||||
source: $source,
|
||||
image: $runtime['image'],
|
||||
version: $version,
|
||||
remove: true,
|
||||
entrypoint: $deployment->getAttribute('entrypoint'),
|
||||
workdir: '/usr/code',
|
||||
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
|
||||
variables: $vars,
|
||||
commands: [
|
||||
'sh', '-c',
|
||||
'tar -zxf /tmp/code.tar.gz -C /usr/code && \
|
||||
cd /usr/local/src/ && ./build.sh'
|
||||
]
|
||||
command: $command
|
||||
);
|
||||
} catch (Exception $error) {
|
||||
$err = $error;
|
||||
}
|
||||
}),
|
||||
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) {
|
||||
try {
|
||||
$executor->getLogs(
|
||||
deploymentId: $deployment->getId(),
|
||||
projectId: $project->getId(),
|
||||
callback: function ($logs) use (&$response, &$build, $dbForProject, $allEvents, $project) {
|
||||
if ($response === null) {
|
||||
$build = $dbForProject->getDocument('builds', $build->getId());
|
||||
|
||||
$endTime = new \DateTime();
|
||||
$endTime->setTimestamp($response['endTimeUnix']);
|
||||
if ($build->isEmpty()) {
|
||||
throw new Exception('Build not found', 404);
|
||||
}
|
||||
|
||||
$build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs);
|
||||
$build = $dbForProject->updateDocument('builds', $build->getId(), $build);
|
||||
|
||||
/**
|
||||
* Send realtime Event
|
||||
*/
|
||||
$target = Realtime::fromPayload(
|
||||
// Pass first, most verbose event pattern
|
||||
event: $allEvents[0],
|
||||
payload: $build,
|
||||
project: $project
|
||||
);
|
||||
Realtime::send(
|
||||
projectId: 'console',
|
||||
payload: $build->getArrayCopy(),
|
||||
events: $allEvents,
|
||||
channels: $target['channels'],
|
||||
roles: $target['roles']
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (Exception $error) {
|
||||
if (empty($err)) {
|
||||
$err = $error;
|
||||
}
|
||||
}
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
if ($err) {
|
||||
throw $err;
|
||||
}
|
||||
|
||||
$endTime = DateTime::now();
|
||||
$durationEnd = \microtime(true);
|
||||
|
||||
/** Update the build document */
|
||||
$build->setAttribute('endTime', DateTime::format($endTime));
|
||||
$build->setAttribute('duration', \intval($response['duration']));
|
||||
$build->setAttribute('status', $response['status']);
|
||||
$build->setAttribute('outputPath', $response['outputPath']);
|
||||
$build->setAttribute('stderr', $response['stderr']);
|
||||
$build->setAttribute('stdout', $response['stdout']);
|
||||
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
|
||||
$build->setAttribute('endTime', $endTime);
|
||||
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
|
||||
$build->setAttribute('status', 'ready');
|
||||
$build->setAttribute('path', $response['path']);
|
||||
$build->setAttribute('size', $response['size']);
|
||||
$build->setAttribute('logs', $response['output']);
|
||||
|
||||
/* Also update the deployment buildTime */
|
||||
$deployment->setAttribute('buildTime', $response['duration']);
|
||||
if ($isVcsEnabled) {
|
||||
$this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
|
||||
}
|
||||
|
||||
Console::success("Build id: $buildId created");
|
||||
|
||||
|
@ -238,27 +463,33 @@ class Builds extends Action
|
|||
if ($deployment->getAttribute('activate') === true) {
|
||||
$function->setAttribute('deploymentInternalId', $deployment->getInternalId());
|
||||
$function->setAttribute('deployment', $deployment->getId());
|
||||
$function->setAttribute('live', true);
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
|
||||
}
|
||||
|
||||
/** Update function schedule */
|
||||
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
|
||||
$schedule->setAttribute('resourceUpdatedAt', DateTime::now());
|
||||
|
||||
// Inform scheduler if function is still active
|
||||
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
|
||||
$schedule
|
||||
->setAttribute('resourceUpdatedAt', DateTime::now())
|
||||
->setAttribute('schedule', $function->getAttribute('schedule'))
|
||||
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
|
||||
|
||||
|
||||
\Utopia\Database\Validator\Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
|
||||
Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
|
||||
} catch (\Throwable $th) {
|
||||
$endTime = DateTime::now();
|
||||
$interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
|
||||
$durationEnd = \microtime(true);
|
||||
$build->setAttribute('endTime', $endTime);
|
||||
$build->setAttribute('duration', $interval->format('%s') + 0);
|
||||
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
|
||||
$build->setAttribute('status', 'failed');
|
||||
$build->setAttribute('stderr', $th->getMessage());
|
||||
$build->setAttribute('logs', $th->getMessage());
|
||||
Console::error($th->getMessage());
|
||||
Console::error($th->getFile() . ':' . $th->getLine());
|
||||
Console::error($th->getTraceAsString());
|
||||
|
||||
if ($isVcsEnabled) {
|
||||
$this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
|
||||
}
|
||||
} finally {
|
||||
$build = $dbForProject->updateDocument('builds', $buildId, $build);
|
||||
|
||||
|
@ -291,4 +522,288 @@ class Builds extends Action
|
|||
->trigger();
|
||||
}
|
||||
}
|
||||
|
||||
protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void
|
||||
{
|
||||
if ($function->getAttribute('providerSilentMode', false) === true) {
|
||||
return;
|
||||
}
|
||||
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$commentId = $deployment->getAttribute('providerCommentId', '');
|
||||
|
||||
if (!empty($providerCommitHash)) {
|
||||
$message = match ($status) {
|
||||
'ready' => 'Build succeeded.',
|
||||
'failed' => 'Build failed.',
|
||||
'processing' => 'Building...',
|
||||
default => $status
|
||||
};
|
||||
|
||||
$state = match ($status) {
|
||||
'ready' => 'success',
|
||||
'failed' => 'failure',
|
||||
'processing' => 'pending',
|
||||
default => $status
|
||||
};
|
||||
|
||||
$functionName = $function->getAttribute('name');
|
||||
$projectName = $project->getAttribute('name');
|
||||
|
||||
$name = "{$functionName} ({$projectName})";
|
||||
|
||||
$protocol = App::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
|
||||
$hostname = App::getEnv('_APP_DOMAIN');
|
||||
$functionId = $function->getId();
|
||||
$projectId = $project->getId();
|
||||
$providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$projectId/functions/function-$functionId";
|
||||
|
||||
$github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name);
|
||||
}
|
||||
|
||||
if (!empty($commentId)) {
|
||||
$retries = 0;
|
||||
|
||||
while (true) {
|
||||
$retries++;
|
||||
|
||||
try {
|
||||
$dbForConsole->createDocument('vcsCommentLocks', new Document([
|
||||
'$id' => $commentId
|
||||
]));
|
||||
break;
|
||||
} catch (Exception $err) {
|
||||
if ($retries >= 9) {
|
||||
throw $err;
|
||||
}
|
||||
|
||||
\sleep(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Wrap in try/finally to ensure lock file gets deleted
|
||||
try {
|
||||
$comment = new Comment();
|
||||
$comment->parseComment($github->getComment($owner, $repositoryName, $commentId));
|
||||
$comment->addBuild($project, $function, $status, $deployment->getId(), ['type' => 'logs']);
|
||||
$github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment());
|
||||
} finally {
|
||||
$dbForConsole->deleteDocument('vcsCommentLocks', $commentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// // Realtime preparation
|
||||
// $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
|
||||
// 'functionId' => $function->getId(),
|
||||
// 'deploymentId' => $deployment->getId()
|
||||
// ]);
|
||||
//
|
||||
// $startTime = DateTime::now();
|
||||
// $durationStart = \microtime(true);
|
||||
//
|
||||
//
|
||||
//
|
||||
// $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
|
||||
// $function = $dbForProject->getDocument('functions', $function->getId());
|
||||
//
|
||||
// if ($function->isEmpty()) {
|
||||
// throw new Exception('Function not found', 404);
|
||||
// }
|
||||
//
|
||||
// $deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
// if ($deployment->isEmpty()) {
|
||||
// throw new Exception('Deployment not found', 404);
|
||||
// }
|
||||
//
|
||||
// $runtimes = Config::getParam('runtimes', []);
|
||||
// $key = $function->getAttribute('runtime');
|
||||
// $runtime = $runtimes[$key] ?? null;
|
||||
// if (\is_null($runtime)) {
|
||||
// throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
||||
// }
|
||||
//
|
||||
// $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
|
||||
// /** @TODO : move this to the registry or someplace else */
|
||||
// $device = Storage::DEVICE_LOCAL;
|
||||
// try {
|
||||
// $dsn = new DSN($connection);
|
||||
// $device = $dsn->getScheme();
|
||||
// } catch (\Exception $e) {
|
||||
// Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
|
||||
// }
|
||||
//
|
||||
// $buildId = $deployment->getAttribute('buildId', '');
|
||||
// $startTime = DateTime::now();
|
||||
//
|
||||
// if (empty($buildId)) {
|
||||
// $buildId = ID::unique();
|
||||
// $build = $dbForProject->createDocument('builds', new Document([
|
||||
// '$id' => $buildId,
|
||||
// '$permissions' => [],
|
||||
// 'startTime' => $startTime,
|
||||
// 'deploymentInternalId' => $deployment->getInternalId(),
|
||||
// 'deploymentId' => $deployment->getId(),
|
||||
// 'status' => 'processing',
|
||||
// 'outputPath' => '',
|
||||
// 'runtime' => $function->getAttribute('runtime'),
|
||||
// 'source' => $deployment->getAttribute('path'),
|
||||
// 'sourceType' => $device,
|
||||
// 'stdout' => '',
|
||||
// 'stderr' => '',
|
||||
// 'duration' => 0
|
||||
// ]));
|
||||
//
|
||||
// $deployment->setAttribute('buildId', $buildId);
|
||||
// $deployment->setAttribute('buildInternalId', $build->getInternalId());
|
||||
// $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
||||
// } else {
|
||||
// $build = $dbForProject->getDocument('builds', $buildId);
|
||||
// }
|
||||
//
|
||||
// /** Request the executor to build the code... */
|
||||
// $build->setAttribute('status', 'building');
|
||||
// $build = $dbForProject->updateDocument('builds', $buildId, $build);
|
||||
//
|
||||
// /** Trigger Webhook */
|
||||
// $deploymentUpdate = $queueForEvents
|
||||
// ->setQueue(Event::WEBHOOK_QUEUE_NAME)
|
||||
// ->setClass(Event::WEBHOOK_CLASS_NAME)
|
||||
// ->setProject($project)
|
||||
// ->setEvent('functions.[functionId].deployments.[deploymentId].update')
|
||||
// ->setParam('functionId', $function->getId())
|
||||
// ->setParam('deploymentId', $deployment->getId())
|
||||
// ->setPayload($deployment->getArrayCopy(
|
||||
// array_keys(
|
||||
// (new Deployment())->getRules()
|
||||
// )
|
||||
// ));
|
||||
//
|
||||
// $deploymentUpdate->trigger();
|
||||
//
|
||||
// /** Trigger Functions */
|
||||
// $queueForFunctions
|
||||
// ->from($deploymentUpdate)
|
||||
// ->trigger();
|
||||
//
|
||||
//
|
||||
// /** Trigger Realtime */
|
||||
// $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
|
||||
// 'functionId' => $function->getId(),
|
||||
// 'deploymentId' => $deployment->getId()
|
||||
// ]);
|
||||
// $target = Realtime::fromPayload(
|
||||
// // Pass first, most verbose event pattern
|
||||
// event: $allEvents[0],
|
||||
// payload: $build,
|
||||
// project: $project
|
||||
// );
|
||||
//
|
||||
// Realtime::send(
|
||||
// projectId: 'console',
|
||||
// payload: $build->getArrayCopy(),
|
||||
// events: $allEvents,
|
||||
// channels: $target['channels'],
|
||||
// roles: $target['roles']
|
||||
// );
|
||||
//
|
||||
// $source = $deployment->getAttribute('path');
|
||||
//
|
||||
// $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
|
||||
// $carry[$var->getAttribute('key')] = $var->getAttribute('value');
|
||||
// return $carry;
|
||||
// }, []);
|
||||
//
|
||||
// try {
|
||||
// $response = $executor->createRuntime(
|
||||
// deploymentId: $deployment->getId(),
|
||||
// projectId: $project->getId(),
|
||||
// source: $source,
|
||||
// image: $runtime['image'],
|
||||
// remove: true,
|
||||
// entrypoint: $deployment->getAttribute('entrypoint'),
|
||||
// workdir: '/usr/code',
|
||||
// destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
|
||||
// variables: $vars,
|
||||
// commands: [
|
||||
// 'sh', '-c',
|
||||
// 'tar -zxf /tmp/code.tar.gz -C /usr/code && \
|
||||
// cd /usr/local/src/ && ./build.sh'
|
||||
// ]
|
||||
// );
|
||||
//
|
||||
// $endTime = new \DateTime();
|
||||
// $endTime->setTimestamp($response['endTimeUnix']);
|
||||
//
|
||||
// /** Update the build document */
|
||||
// $build->setAttribute('endTime', DateTime::format($endTime));
|
||||
// $build->setAttribute('duration', \intval($response['duration']));
|
||||
// $build->setAttribute('status', $response['status']);
|
||||
// $build->setAttribute('outputPath', $response['outputPath']);
|
||||
// $build->setAttribute('stderr', $response['stderr']);
|
||||
// $build->setAttribute('stdout', $response['stdout']);
|
||||
//
|
||||
// /* Also update the deployment buildTime */
|
||||
// $deployment->setAttribute('buildTime', $response['duration']);
|
||||
//
|
||||
// Console::success("Build id: $buildId created");
|
||||
//
|
||||
// /** Set auto deploy */
|
||||
// if ($deployment->getAttribute('activate') === true) {
|
||||
// $function->setAttribute('deploymentInternalId', $deployment->getInternalId());
|
||||
// $function->setAttribute('deployment', $deployment->getId());
|
||||
// $function = $dbForProject->updateDocument('functions', $function->getId(), $function);
|
||||
// }
|
||||
//
|
||||
// /** Update function schedule */
|
||||
// $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
|
||||
// $schedule->setAttribute('resourceUpdatedAt', DateTime::now());
|
||||
//
|
||||
// $schedule
|
||||
// ->setAttribute('schedule', $function->getAttribute('schedule'))
|
||||
// ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
|
||||
//
|
||||
//
|
||||
// \Utopia\Database\Validator\Authorization::skip(fn() => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
|
||||
// } catch (\Throwable $th) {
|
||||
// $endTime = DateTime::now();
|
||||
// $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
|
||||
// $build->setAttribute('endTime', $endTime);
|
||||
// $build->setAttribute('duration', $interval->format('%s') + 0);
|
||||
// $build->setAttribute('status', 'failed');
|
||||
// $build->setAttribute('stderr', $th->getMessage());
|
||||
// Console::error($th->getMessage());
|
||||
// } finally {
|
||||
// $build = $dbForProject->updateDocument('builds', $buildId, $build);
|
||||
//
|
||||
// /**
|
||||
// * Send realtime Event
|
||||
// */
|
||||
// $target = Realtime::fromPayload(
|
||||
// // Pass first, most verbose event pattern
|
||||
// event: $allEvents[0],
|
||||
// payload: $build,
|
||||
// project: $project
|
||||
// );
|
||||
// Realtime::send(
|
||||
// projectId: 'console',
|
||||
// payload: $build->getArrayCopy(),
|
||||
// events: $allEvents,
|
||||
// channels: $target['channels'],
|
||||
// roles: $target['roles']
|
||||
// );
|
||||
//
|
||||
// /** Trigger usage queue */
|
||||
// $queueForUsage
|
||||
// ->setProject($project)
|
||||
// ->addMetric(METRIC_BUILDS, 1) // per project
|
||||
// ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
|
||||
// ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
|
||||
// ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
|
||||
// ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
|
||||
// ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
|
||||
// ->trigger();
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
|
|
@ -89,12 +89,6 @@ class UsageHook extends Usage
|
|||
}
|
||||
}
|
||||
}
|
||||
if (!empty($project['keys'])) {
|
||||
$dbForProject->createDocument('statsLogger', new Document([
|
||||
'time' => DateTime::now(),
|
||||
'metrics' => $project['keys'],
|
||||
]));
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
console::error("[logger] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
|
||||
} finally {
|
||||
|
|
Loading…
Reference in a new issue