2022-01-24 11:25:46 +13:00
|
|
|
<?php
|
|
|
|
|
2022-04-19 04:21:45 +12:00
|
|
|
use Appwrite\Event\Event;
|
2022-11-16 07:13:17 +13:00
|
|
|
use Appwrite\Event\Func;
|
2022-03-01 00:50:27 +13:00
|
|
|
use Appwrite\Messaging\Adapter\Realtime;
|
2022-01-24 11:25:46 +13:00
|
|
|
use Appwrite\Resque\Worker;
|
2022-04-19 04:21:45 +12:00
|
|
|
use Appwrite\Utopia\Response\Model\Deployment;
|
2022-02-04 14:29:40 +13:00
|
|
|
use Executor\Executor;
|
2022-08-09 18:28:38 +12:00
|
|
|
use Appwrite\Usage\Stats;
|
2022-07-13 01:32:39 +12:00
|
|
|
use Utopia\Database\DateTime;
|
2022-01-24 11:25:46 +13:00
|
|
|
use Utopia\App;
|
|
|
|
use Utopia\CLI\Console;
|
2022-08-15 02:22:38 +12:00
|
|
|
use Utopia\Database\ID;
|
2022-11-18 01:50:17 +13:00
|
|
|
use Utopia\DSN\DSN;
|
2022-01-24 11:25:46 +13:00
|
|
|
use Utopia\Database\Document;
|
|
|
|
use Utopia\Config\Config;
|
2022-11-16 23:33:11 +13:00
|
|
|
use Utopia\Database\Validator\Authorization;
|
2022-11-18 06:43:59 +13:00
|
|
|
use Utopia\Storage\Storage;
|
2022-01-24 11:25:46 +13:00
|
|
|
|
2022-04-20 01:13:55 +12:00
|
|
|
require_once __DIR__ . '/../init.php';
|
2022-01-24 11:25:46 +13:00
|
|
|
|
|
|
|
Console::title('Builds V1 Worker');
|
2022-04-20 01:13:55 +12:00
|
|
|
Console::success(APP_NAME . ' build worker v1 has started');
|
2022-01-24 11:25:46 +13:00
|
|
|
|
2022-01-25 09:06:29 +13:00
|
|
|
// TODO: Executor should return appropriate response codes.
|
2022-01-24 11:25:46 +13:00
|
|
|
class BuildsV1 extends Worker
|
2022-04-14 00:39:31 +12:00
|
|
|
{
|
2022-04-20 01:13:55 +12:00
|
|
|
private ?Executor $executor = null;
|
2022-01-24 11:25:46 +13:00
|
|
|
|
2022-04-20 01:13:55 +12:00
|
|
|
public function getName(): string
|
2022-01-26 12:45:41 +13:00
|
|
|
{
|
2022-01-24 11:25:46 +13:00
|
|
|
return "builds";
|
|
|
|
}
|
|
|
|
|
2022-05-10 00:36:29 +12:00
|
|
|
public function init(): void
|
|
|
|
{
|
2022-04-14 04:15:25 +12:00
|
|
|
$this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
|
2022-02-04 14:29:40 +13:00
|
|
|
}
|
2022-01-24 11:25:46 +13:00
|
|
|
|
|
|
|
public function run(): void
|
|
|
|
{
|
|
|
|
$type = $this->args['type'] ?? '';
|
2022-04-20 01:13:55 +12:00
|
|
|
$project = new Document($this->args['project'] ?? []);
|
|
|
|
$resource = new Document($this->args['resource'] ?? []);
|
|
|
|
$deployment = new Document($this->args['deployment'] ?? []);
|
2022-04-14 00:39:31 +12:00
|
|
|
|
2022-01-24 11:25:46 +13:00
|
|
|
switch ($type) {
|
2022-01-27 02:09:32 +13:00
|
|
|
case BUILD_TYPE_DEPLOYMENT:
|
2022-02-17 09:06:22 +13:00
|
|
|
case BUILD_TYPE_RETRY:
|
2022-04-20 01:13:55 +12:00
|
|
|
Console::info('Creating build for deployment: ' . $deployment->getId());
|
|
|
|
$this->buildDeployment($project, $resource, $deployment);
|
2022-01-24 11:25:46 +13:00
|
|
|
break;
|
2022-01-24 12:01:42 +13:00
|
|
|
|
2022-01-24 11:25:46 +13:00
|
|
|
default:
|
2022-01-24 12:01:42 +13:00
|
|
|
throw new \Exception('Invalid build type');
|
2022-01-24 11:25:46 +13:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-20 01:13:55 +12:00
|
|
|
protected function buildDeployment(Document $project, Document $function, Document $deployment)
|
2022-01-24 11:25:46 +13:00
|
|
|
{
|
2022-11-17 01:19:29 +13:00
|
|
|
global $register;
|
|
|
|
|
2022-08-13 19:57:04 +12:00
|
|
|
$dbForProject = $this->getProjectDB($project);
|
2022-04-14 00:39:31 +12:00
|
|
|
|
2022-04-20 01:13:55 +12:00
|
|
|
$function = $dbForProject->getDocument('functions', $function->getId());
|
2022-01-24 11:25:46 +13:00
|
|
|
if ($function->isEmpty()) {
|
|
|
|
throw new Exception('Function not found', 404);
|
|
|
|
}
|
|
|
|
|
2022-04-20 01:13:55 +12:00
|
|
|
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
2022-01-27 02:09:32 +13:00
|
|
|
if ($deployment->isEmpty()) {
|
|
|
|
throw new Exception('Deployment not found', 404);
|
2022-01-24 11:25:46 +13:00
|
|
|
}
|
|
|
|
|
|
|
|
$runtimes = Config::getParam('runtimes', []);
|
|
|
|
$key = $function->getAttribute('runtime');
|
|
|
|
$runtime = isset($runtimes[$key]) ? $runtimes[$key] : null;
|
|
|
|
if (\is_null($runtime)) {
|
|
|
|
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
|
|
|
}
|
|
|
|
|
2022-11-17 21:04:05 +13:00
|
|
|
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */
|
2022-11-18 06:43:59 +13:00
|
|
|
$device = Storage::DEVICE_LOCAL;
|
2022-11-17 20:44:54 +13:00
|
|
|
try {
|
|
|
|
$dsn = new DSN($connection);
|
|
|
|
$device = $dsn->getScheme();
|
|
|
|
} catch (\Exception $e) {
|
2022-11-18 06:46:49 +13:00
|
|
|
Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
|
2022-11-17 20:44:54 +13:00
|
|
|
}
|
|
|
|
|
2022-01-27 02:09:32 +13:00
|
|
|
$buildId = $deployment->getAttribute('buildId', '');
|
2022-07-14 02:02:49 +12:00
|
|
|
$startTime = DateTime::now();
|
2023-03-29 02:21:42 +13:00
|
|
|
$durationStart = \microtime(true);
|
2022-01-24 11:25:46 +13:00
|
|
|
if (empty($buildId)) {
|
2022-08-15 02:22:38 +12:00
|
|
|
$buildId = ID::unique();
|
2022-02-03 07:58:03 +13:00
|
|
|
$build = $dbForProject->createDocument('builds', new Document([
|
|
|
|
'$id' => $buildId,
|
2022-08-02 21:21:53 +12:00
|
|
|
'$permissions' => [],
|
2022-02-14 10:26:36 +13:00
|
|
|
'startTime' => $startTime,
|
2022-12-26 00:06:25 +13:00
|
|
|
'deploymentInternalId' => $deployment->getInternalId(),
|
2022-04-20 01:13:55 +12:00
|
|
|
'deploymentId' => $deployment->getId(),
|
2022-02-03 07:58:03 +13:00
|
|
|
'status' => 'processing',
|
2023-03-15 21:20:25 +13:00
|
|
|
'path' => '',
|
2022-02-03 07:58:03 +13:00
|
|
|
'runtime' => $function->getAttribute('runtime'),
|
|
|
|
'source' => $deployment->getAttribute('path'),
|
2022-11-17 20:44:54 +13:00
|
|
|
'sourceType' => $device,
|
2022-02-03 07:58:03 +13:00
|
|
|
'stdout' => '',
|
|
|
|
'stderr' => '',
|
2022-12-21 01:56:56 +13:00
|
|
|
'endTime' => null,
|
2023-03-15 21:20:25 +13:00
|
|
|
'duration' => 0,
|
|
|
|
'size' => 0
|
2022-02-03 07:58:03 +13:00
|
|
|
]));
|
2022-12-26 00:06:25 +13:00
|
|
|
$deployment->setAttribute('buildId', $build->getId());
|
|
|
|
$deployment->setAttribute('buildInternalId', $build->getInternalId());
|
2022-04-20 01:13:55 +12:00
|
|
|
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
|
2022-02-03 07:58:03 +13:00
|
|
|
} else {
|
|
|
|
$build = $dbForProject->getDocument('builds', $buildId);
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Request the executor to build the code... */
|
|
|
|
$build->setAttribute('status', 'building');
|
|
|
|
$build = $dbForProject->updateDocument('builds', $buildId, $build);
|
|
|
|
|
2022-04-19 04:21:45 +12:00
|
|
|
/** Trigger Webhook */
|
|
|
|
$deploymentModel = new Deployment();
|
2022-09-16 21:54:59 +12:00
|
|
|
|
2022-04-19 04:21:45 +12:00
|
|
|
$deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME);
|
|
|
|
$deploymentUpdate
|
|
|
|
->setProject($project)
|
|
|
|
->setEvent('functions.[functionId].deployments.[deploymentId].update')
|
|
|
|
->setParam('functionId', $function->getId())
|
|
|
|
->setParam('deploymentId', $deployment->getId())
|
|
|
|
->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())))
|
|
|
|
->trigger();
|
|
|
|
|
|
|
|
/** Trigger Functions */
|
2022-11-16 07:13:17 +13:00
|
|
|
$pools = $register->get('pools');
|
|
|
|
$connection = $pools->get('queue')->pop();
|
2022-11-17 06:55:26 +13:00
|
|
|
|
2022-11-16 07:13:17 +13:00
|
|
|
$functions = new Func($connection->getResource());
|
|
|
|
$functions
|
2022-11-17 02:34:11 +13:00
|
|
|
->from($deploymentUpdate)
|
2022-04-19 04:21:45 +12:00
|
|
|
->trigger();
|
|
|
|
|
2022-11-16 07:13:17 +13:00
|
|
|
$connection->reclaim();
|
2022-04-19 04:21:45 +12:00
|
|
|
|
|
|
|
/** Trigger Realtime */
|
|
|
|
$allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
|
|
|
|
'functionId' => $function->getId(),
|
|
|
|
'deploymentId' => $deployment->getId()
|
|
|
|
]);
|
2022-05-10 00:36:29 +12:00
|
|
|
$target = Realtime::fromPayload(
|
|
|
|
// Pass first, most verbose event pattern
|
|
|
|
event: $allEvents[0],
|
|
|
|
payload: $build,
|
|
|
|
project: $project
|
|
|
|
);
|
|
|
|
|
2022-03-01 00:50:27 +13:00
|
|
|
Realtime::send(
|
|
|
|
projectId: 'console',
|
|
|
|
payload: $build->getArrayCopy(),
|
2022-04-19 04:21:45 +12:00
|
|
|
events: $allEvents,
|
2022-03-01 00:50:27 +13:00
|
|
|
channels: $target['channels'],
|
|
|
|
roles: $target['roles']
|
|
|
|
);
|
|
|
|
|
2022-02-14 10:26:36 +13:00
|
|
|
$source = $deployment->getAttribute('path');
|
2022-08-10 21:32:48 +12:00
|
|
|
|
2022-11-16 23:33:11 +13:00
|
|
|
$vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
|
2022-08-31 00:30:52 +12:00
|
|
|
$carry[$var->getAttribute('key')] = $var->getAttribute('value');
|
|
|
|
return $carry;
|
|
|
|
}, []);
|
2022-08-10 21:32:48 +12:00
|
|
|
|
2022-02-14 10:26:36 +13:00
|
|
|
try {
|
2023-03-29 02:21:42 +13:00
|
|
|
$command = '';
|
|
|
|
|
2023-05-12 08:52:48 +12:00
|
|
|
if (!empty($deployment->getAttribute('installCommand', ''))) {
|
2023-03-29 02:21:42 +13:00
|
|
|
$command .= $deployment->getAttribute('installCommand', '');
|
|
|
|
}
|
|
|
|
|
2023-05-12 08:52:48 +12:00
|
|
|
if (!empty($deployment->getAttribute('buildCommand', ''))) {
|
2023-03-29 02:21:42 +13:00
|
|
|
$separator = empty($command) ? '' : ' && ';
|
|
|
|
$command .= $separator . $deployment->getAttribute('buildCommand', '');
|
|
|
|
}
|
|
|
|
|
|
|
|
$command = \str_replace('"', '\\"', $command);
|
|
|
|
|
2022-02-14 10:26:36 +13:00
|
|
|
$response = $this->executor->createRuntime(
|
2022-04-20 01:13:55 +12:00
|
|
|
projectId: $project->getId(),
|
|
|
|
deploymentId: $deployment->getId(),
|
2022-02-14 10:26:36 +13:00
|
|
|
source: $source,
|
2023-02-03 08:21:00 +13:00
|
|
|
version: $function->getAttribute('version'),
|
2022-11-08 21:49:45 +13:00
|
|
|
image: $runtime['image'],
|
2022-02-18 13:43:04 +13:00
|
|
|
remove: true,
|
2022-11-08 21:49:45 +13:00
|
|
|
entrypoint: $deployment->getAttribute('entrypoint'),
|
|
|
|
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
|
|
|
|
variables: $vars,
|
2022-02-18 03:51:36 +13:00
|
|
|
commands: [
|
|
|
|
'sh', '-c',
|
2023-03-29 02:21:42 +13:00
|
|
|
'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'
|
2022-02-18 03:51:36 +13:00
|
|
|
]
|
2022-02-14 10:26:36 +13:00
|
|
|
);
|
|
|
|
|
2023-03-15 21:20:25 +13:00
|
|
|
$endTime = DateTime::now();
|
2022-12-21 01:56:56 +13:00
|
|
|
|
2022-02-14 10:26:36 +13:00
|
|
|
/** Update the build document */
|
2022-12-18 20:35:16 +13:00
|
|
|
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
|
2023-03-15 21:20:25 +13:00
|
|
|
$build->setAttribute('endTime', $endTime);
|
2023-03-11 01:20:24 +13:00
|
|
|
$build->setAttribute('duration', \intval(\ceil($response['duration'])));
|
2023-02-03 08:21:00 +13:00
|
|
|
$build->setAttribute('status', 'ready');
|
2023-03-15 21:20:25 +13:00
|
|
|
$build->setAttribute('path', $response['path']);
|
2022-12-18 20:20:50 +13:00
|
|
|
$build->setAttribute('size', $response['size']);
|
2022-02-14 10:26:36 +13:00
|
|
|
$build->setAttribute('stderr', $response['stderr']);
|
2022-11-08 23:02:22 +13:00
|
|
|
$build->setAttribute('stdout', $response['stdout']);
|
2022-02-17 00:43:21 +13:00
|
|
|
|
2022-10-19 14:43:40 +13:00
|
|
|
/* Also update the deployment buildTime */
|
2022-10-19 14:50:48 +13:00
|
|
|
$deployment->setAttribute('buildTime', $response['duration']);
|
2022-10-19 14:43:40 +13:00
|
|
|
|
2022-02-17 00:43:21 +13:00
|
|
|
Console::success("Build id: $buildId created");
|
|
|
|
|
|
|
|
/** Set auto deploy */
|
|
|
|
if ($deployment->getAttribute('activate') === true) {
|
2022-12-26 01:07:54 +13:00
|
|
|
$function->setAttribute('deploymentInternalId', $deployment->getInternalId());
|
2022-02-17 00:43:21 +13:00
|
|
|
$function->setAttribute('deployment', $deployment->getId());
|
2022-08-19 22:26:09 +12:00
|
|
|
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
|
2022-02-17 00:43:21 +13:00
|
|
|
}
|
|
|
|
|
|
|
|
/** Update function schedule */
|
2022-11-17 01:19:29 +13:00
|
|
|
$dbForConsole = $this->getConsoleDB();
|
2022-11-16 23:33:11 +13:00
|
|
|
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
|
2023-03-19 22:43:57 +13:00
|
|
|
$schedule->setAttribute('resourceUpdatedAt', DateTime::now());
|
2022-11-16 23:33:11 +13:00
|
|
|
|
|
|
|
$schedule
|
|
|
|
->setAttribute('schedule', $function->getAttribute('schedule'))
|
|
|
|
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
|
2022-11-16 23:54:21 +13:00
|
|
|
|
|
|
|
|
2022-11-16 23:33:11 +13:00
|
|
|
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
|
2022-02-14 10:26:36 +13:00
|
|
|
} catch (\Throwable $th) {
|
2022-09-03 02:19:36 +12:00
|
|
|
$endTime = DateTime::now();
|
2023-03-29 02:21:42 +13:00
|
|
|
$durationEnd = \microtime(true);
|
2022-12-21 01:56:56 +13:00
|
|
|
$build->setAttribute('endTime', $endTime);
|
2023-03-29 02:21:42 +13:00
|
|
|
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
|
2022-02-14 10:26:36 +13:00
|
|
|
$build->setAttribute('status', 'failed');
|
|
|
|
$build->setAttribute('stderr', $th->getMessage());
|
2022-02-14 13:20:12 +13:00
|
|
|
Console::error($th->getMessage());
|
2022-02-17 00:43:21 +13:00
|
|
|
} finally {
|
|
|
|
$build = $dbForProject->updateDocument('builds', $buildId, $build);
|
2022-03-01 00:50:27 +13:00
|
|
|
|
2022-05-24 02:54:50 +12:00
|
|
|
/**
|
2022-04-14 00:39:31 +12:00
|
|
|
* Send realtime Event
|
2022-03-01 00:50:27 +13:00
|
|
|
*/
|
2022-05-10 00:36:29 +12:00
|
|
|
$target = Realtime::fromPayload(
|
|
|
|
// Pass first, most verbose event pattern
|
|
|
|
event: $allEvents[0],
|
|
|
|
payload: $build,
|
|
|
|
project: $project
|
|
|
|
);
|
2022-03-01 00:50:27 +13:00
|
|
|
Realtime::send(
|
|
|
|
projectId: 'console',
|
|
|
|
payload: $build->getArrayCopy(),
|
2022-04-19 04:21:45 +12:00
|
|
|
events: $allEvents,
|
2022-03-01 00:50:27 +13:00
|
|
|
channels: $target['channels'],
|
|
|
|
roles: $target['roles']
|
|
|
|
);
|
2022-06-29 18:30:24 +12:00
|
|
|
|
|
|
|
/** Update usage stats */
|
|
|
|
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
|
|
|
|
$statsd = $register->get('statsd');
|
|
|
|
$usage = new Stats($statsd);
|
|
|
|
$usage
|
2022-11-23 04:50:41 +13:00
|
|
|
->setParam('projectInternalId', $project->getInternalId())
|
2022-06-29 18:30:24 +12:00
|
|
|
->setParam('projectId', $project->getId())
|
|
|
|
->setParam('functionId', $function->getId())
|
2022-08-11 17:25:39 +12:00
|
|
|
->setParam('builds.{scope}.compute', 1)
|
|
|
|
->setParam('buildStatus', $build->getAttribute('status', ''))
|
2022-08-11 18:49:20 +12:00
|
|
|
->setParam('buildTime', $build->getAttribute('duration'))
|
2022-06-29 18:30:24 +12:00
|
|
|
->setParam('networkRequestSize', 0)
|
|
|
|
->setParam('networkResponseSize', 0)
|
|
|
|
->submit();
|
|
|
|
}
|
2022-02-14 10:26:36 +13:00
|
|
|
}
|
2022-01-24 11:25:46 +13:00
|
|
|
}
|
|
|
|
|
2022-04-20 01:13:55 +12:00
|
|
|
public function shutdown(): void
|
|
|
|
{
|
|
|
|
}
|
2022-01-24 11:25:46 +13:00
|
|
|
}
|