1
0
Fork 0
mirror of synced 2024-09-18 18:40:24 +12:00

Merge remote-tracking branch 'origin/1.6.x' into feat-benchmark-headers

This commit is contained in:
Matej Bačo 2024-08-22 12:58:05 +00:00
commit 31e4a24a76
16 changed files with 229 additions and 58 deletions

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -1426,7 +1426,8 @@ App::get('/v1/functions/:functionId/deployments')
$result->setAttribute('status', $build->getAttribute('status', 'processing'));
$result->setAttribute('buildLogs', $build->getAttribute('logs', ''));
$result->setAttribute('buildTime', $build->getAttribute('duration', 0));
$result->setAttribute('size', $result->getAttribute('size', 0) + $build->getAttribute('size', 0));
$result->setAttribute('buildSize', $build->getAttribute('size', 0));
$result->setAttribute('size', $result->getAttribute('size', 0));
}
$response->dynamic(new Document([
@ -1472,7 +1473,8 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId')
$deployment->setAttribute('status', $build->getAttribute('status', 'waiting'));
$deployment->setAttribute('buildLogs', $build->getAttribute('logs', ''));
$deployment->setAttribute('buildTime', $build->getAttribute('duration', 0));
$deployment->setAttribute('size', $deployment->getAttribute('size', 0) + $build->getAttribute('size', 0));
$deployment->setAttribute('buildSize', $build->getAttribute('size', 0));
$deployment->setAttribute('size', $deployment->getAttribute('size', 0));
$response->dynamic($deployment, Response::MODEL_DEPLOYMENT);
});
@ -1680,8 +1682,17 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId/build')
]));
}
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
$deleteBuild = $executor->deleteRuntime($project->getId(), $deploymentId . "-build");
$dbForProject->purgeCachedDocument('deployments', $deployment->getId());
try {
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
$executor->deleteRuntime($project->getId(), $deploymentId . "-build");
} catch (\Throwable $th) {
// Don't throw if the deployment doesn't exist
if ($th->getCode() !== 404) {
throw $th;
}
}
$queueForEvents
->setParam('functionId', $function->getId())

View file

@ -789,7 +789,7 @@ $image = $this->getParam('image', '');
<<: *x-logging
restart: unless-stopped
stop_signal: SIGINT
image: openruntimes/executor:0.6.5
image: openruntimes/executor:0.6.7
networks:
- appwrite
- runtimes

46
composer.lock generated
View file

@ -4232,35 +4232,35 @@
},
{
"name": "phpunit/php-code-coverage",
"version": "9.2.31",
"version": "9.2.32",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "48c34b5d8d983006bd2adc2d0de92963b9155965"
"reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/48c34b5d8d983006bd2adc2d0de92963b9155965",
"reference": "48c34b5d8d983006bd2adc2d0de92963b9155965",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/85402a822d1ecf1db1096959413d35e1c37cf1a5",
"reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5",
"shasum": ""
},
"require": {
"ext-dom": "*",
"ext-libxml": "*",
"ext-xmlwriter": "*",
"nikic/php-parser": "^4.18 || ^5.0",
"nikic/php-parser": "^4.19.1 || ^5.1.0",
"php": ">=7.3",
"phpunit/php-file-iterator": "^3.0.3",
"phpunit/php-text-template": "^2.0.2",
"sebastian/code-unit-reverse-lookup": "^2.0.2",
"sebastian/complexity": "^2.0",
"sebastian/environment": "^5.1.2",
"sebastian/lines-of-code": "^1.0.3",
"sebastian/version": "^3.0.1",
"theseer/tokenizer": "^1.2.0"
"phpunit/php-file-iterator": "^3.0.6",
"phpunit/php-text-template": "^2.0.4",
"sebastian/code-unit-reverse-lookup": "^2.0.3",
"sebastian/complexity": "^2.0.3",
"sebastian/environment": "^5.1.5",
"sebastian/lines-of-code": "^1.0.4",
"sebastian/version": "^3.0.2",
"theseer/tokenizer": "^1.2.3"
},
"require-dev": {
"phpunit/phpunit": "^9.3"
"phpunit/phpunit": "^9.6"
},
"suggest": {
"ext-pcov": "PHP extension that provides line coverage",
@ -4269,7 +4269,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "9.2-dev"
"dev-main": "9.2.x-dev"
}
},
"autoload": {
@ -4298,7 +4298,7 @@
"support": {
"issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
"security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy",
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.31"
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.32"
},
"funding": [
{
@ -4306,7 +4306,7 @@
"type": "github"
}
],
"time": "2024-03-02T06:37:42+00:00"
"time": "2024-08-22T04:23:01+00:00"
},
{
"name": "phpunit/php-file-iterator",
@ -4756,16 +4756,16 @@
},
{
"name": "psr/log",
"version": "3.0.0",
"version": "3.0.1",
"source": {
"type": "git",
"url": "https://github.com/php-fig/log.git",
"reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001"
"reference": "79dff0b268932c640297f5208d6298f71855c03e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/log/zipball/fe5ea303b0887d5caefd3d431c3e61ad47037001",
"reference": "fe5ea303b0887d5caefd3d431c3e61ad47037001",
"url": "https://api.github.com/repos/php-fig/log/zipball/79dff0b268932c640297f5208d6298f71855c03e",
"reference": "79dff0b268932c640297f5208d6298f71855c03e",
"shasum": ""
},
"require": {
@ -4800,9 +4800,9 @@
"psr-3"
],
"support": {
"source": "https://github.com/php-fig/log/tree/3.0.0"
"source": "https://github.com/php-fig/log/tree/3.0.1"
},
"time": "2021-07-14T16:46:02+00:00"
"time": "2024-08-21T13:31:24+00:00"
},
{
"name": "sebastian/cli-parser",

View file

@ -873,7 +873,7 @@ services:
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.6.5
image: openruntimes/executor:0.6.7
restart: unless-stopped
networks:
- appwrite
@ -923,7 +923,7 @@ services:
hostname: proxy
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/proxy:0.5.4
image: openruntimes/proxy:0.5.5
networks:
- appwrite
- runtimes

View file

@ -244,13 +244,7 @@ class Builds extends Action
$tmpTemplateDirectory .= '/';
}
$directorySize = $localDevice->getDirectorySize($tmpTemplateDirectory);
$functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000');
if ($directorySize > $functionsSizeLimit) {
throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.');
}
$tarParamDirectory = \escapeshellarg('/tmp/builds/' . $buildId . '-template' . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory));
$tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory));
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
@ -262,8 +256,9 @@ class Builds extends Action
Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr);
$directorySize = $deviceForFunctions->getFileSize($source);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
}
} elseif ($isNewBuild && $isVcsEnabled) {
// VCS and VCS+Temaplte
@ -516,6 +511,8 @@ class Builds extends Action
return;
}
$isCanceled = false;
Co::join([
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $function, $runtime, $vars, $command, $cpus, $memory, &$err) {
try {
@ -540,12 +537,16 @@ class Builds extends Action
$err = $error;
}
}),
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) {
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) {
try {
$executor->getLogs(
deploymentId: $deployment->getId(),
projectId: $project->getId(),
callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project) {
callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) {
if($isCanceled) {
return;
}
// If we have response or error from concurrent coroutine, we already have latest logs
if ($response === null && $err === null) {
$build = $dbForProject->getDocument('builds', $build->getId());
@ -554,6 +555,12 @@ class Builds extends Action
throw new \Exception('Build not found', 404);
}
if ($build->getAttribute('status') === 'canceled') {
$isCanceled = true;
Console::info('Ignoring realtime logs because build has been canceled');
return;
}
$logs = \mb_substr($logs, 0, null, 'UTF-8'); // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors
$build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs);
@ -586,11 +593,12 @@ class Builds extends Action
}),
]);
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
if ($err) {
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
throw $err;
}
@ -611,6 +619,8 @@ class Builds extends Action
$build->setAttribute('size', $response['size']);
$build->setAttribute('logs', $response['output']);
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
}
@ -652,12 +662,12 @@ class Builds extends Action
$build->setAttribute('status', 'failed');
$build->setAttribute('logs', $th->getMessage());
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
}
} finally {
$build = $dbForProject->updateDocument('builds', $buildId, $build);
/**
* Send realtime Event
*/

View file

@ -33,9 +33,17 @@ class V16 extends Filter
protected function parseDeployment(array $content)
{
$content['buildStderr'] = '';
$content['buildStdout'] = $content['buildLogs'];
unset($content['buildLogs']);
if(isset($content['buildLogs'])) {
$content['buildStderr'] = '';
$content['buildStdout'] = $content['buildLogs'];
unset($content['buildLogs']);
}
if(isset($content['buildSize'])) {
$content['size'] += + $content['buildSize'] ?? 0;
unset($content['buildSize']);
}
return $content;
}

View file

@ -58,6 +58,12 @@ class Deployment extends Model
'default' => 0,
'example' => 128,
])
->addRule('buildSize', [
'type' => self::TYPE_INTEGER,
'description' => 'The build output size in bytes.',
'default' => 0,
'example' => 128,
])
->addRule('buildId', [
'type' => self::TYPE_STRING,
'description' => 'The current build ID.',

View file

@ -514,6 +514,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(200, $deployments['headers']['status-code']);
$this->assertEquals(1, $deployments['body']['total']);
$this->assertNotEmpty($deployments['body']['deployments'][0]['$id']);
$this->assertEquals(0, $deployments['body']['deployments'][0]['size']);
$deploymentId = $deployments['body']['deployments'][0]['$id'];
@ -521,6 +522,13 @@ class FunctionsCustomServerTest extends Scope
// Deployment is automatically activated
$this->awaitDeploymentIsBuilt($function['body']['$id'], $deploymentId);
$deployments = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], []);
$this->assertGreaterThan(0, $deployments['body']['size']);
$function = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@ -688,7 +696,12 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(200, $cancel['headers']['status-code']);
$this->assertEquals('canceled', $cancel['body']['status']);
// Confirm the deployment is canceled
/**
* Build worker still runs the build.
* 30s sleep gives worker enough time to finish build.
* After build finished, it should still be canceled, not ready.
*/
\sleep(30);
$deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'] . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@ -731,7 +744,8 @@ class FunctionsCustomServerTest extends Scope
$largeTag = $this->client->call(Client::METHOD_POST, '/functions/' . $data['functionId'] . '/deployments', array_merge($headers, $this->getHeaders()), [
'entrypoint' => 'index.php',
'code' => $curlFile,
'activate' => true
'activate' => true,
'commands' => 'cp blue.mp4 copy.mp4 && ls -al' // +7MB buildSize
]);
$counter++;
$id = $largeTag['body']['$id'];
@ -742,7 +756,23 @@ class FunctionsCustomServerTest extends Scope
$this->assertNotEmpty($largeTag['body']['$id']);
$this->assertEquals(true, (new DatetimeValidator())->isValid($largeTag['body']['$createdAt']));
$this->assertEquals('index.php', $largeTag['body']['entrypoint']);
$this->assertGreaterThan(10000, $largeTag['body']['size']);
$this->assertGreaterThan(1024 * 1024 * 5, $largeTag['body']['size']); // ~7MB video file
$this->assertLessThan(1024 * 1024 * 10, $largeTag['body']['size']); // ~7MB video file
$deploymentSize = $largeTag['body']['size'];
$deploymentId = $largeTag['body']['$id'];
$this->awaitDeploymentIsBuilt($data['functionId'], $deploymentId, true);
$response = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'] . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals($deploymentSize, $response['body']['size']);
$this->assertGreaterThan(1024 * 1024 * 10, $response['body']['buildSize']); // ~7MB video file + 10MB sample file
return $data;
}
@ -791,6 +821,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals($function['body']['total'], 3);
$this->assertIsArray($function['body']['deployments']);
$this->assertCount(3, $function['body']['deployments']);
$this->assertArrayHasKey('size', $function['body']['deployments'][0]);
$this->assertArrayHasKey('buildSize', $function['body']['deployments'][0]);
/**
* Test search queries
@ -996,6 +1028,58 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals($function['headers']['status-code'], 200);
$this->assertEquals(3, $function['body']['total']);
/**
* Ensure size output and size filters work exactly.
* Prevents buildSize being counted towards deployemtn size
*/
$response = $this->client->call(
Client::METHOD_GET,
'/functions/' . $data['functionId'] . '/deployments',
array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()),
[
Query::limit(1)->toString(),
]
);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertGreaterThanOrEqual(1, $response['body']['total']);
$this->assertNotEmpty($response['body']['deployments'][0]['$id']);
$this->assertNotEmpty($response['body']['deployments'][0]['size']);
$deploymentId = $function['body']['deployments'][0]['$id'];
$deploymentSize = $function['body']['deployments'][0]['size'];
$response = $this->client->call(
Client::METHOD_GET,
'/functions/' . $data['functionId'] . '/deployments',
array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()),
[
'queries' => [
Query::equal('size', [$deploymentSize])->toString(),
],
]
);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertGreaterThan(0, $response['body']['total']);
$found = false;
foreach ($response['body']['deployments'] as $deployment) {
if($deployment['$id'] === $deploymentId) {
$found = true;
$this->assertEquals($deploymentSize, $deployment['size']);
break;
}
}
$this->assertTrue($found);
return $data;
}
@ -1016,6 +1100,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertGreaterThan(0, $function['body']['buildTime']);
$this->assertNotEmpty($function['body']['status']);
$this->assertNotEmpty($function['body']['buildLogs']);
$this->assertArrayHasKey('size', $function['body']);
$this->assertArrayHasKey('buildSize', $function['body']);
/**
* Test for FAILURE

View file

@ -12,6 +12,31 @@ use Utopia\Database\Validator\Datetime as DatetimeValidator;
trait WebhooksBase
{
protected function awaitDeploymentIsBuilt($functionId, $deploymentId, $checkForSuccess = true): void
{
while (true) {
$deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
if (
$deployment['headers']['status-code'] >= 400
|| \in_array($deployment['body']['status'], ['ready', 'failed'])
) {
break;
}
\sleep(1);
}
if($checkForSuccess) {
$this->assertEquals(200, $deployment['headers']['status-code']);
$this->assertEquals('ready', $deployment['body']['status'], \json_encode($deployment['body']));
}
}
public static function getWebhookSignature(array $webhook, string $signatureKey): string
{
$payload = json_encode($webhook['data']);

View file

@ -501,7 +501,7 @@ class WebhooksCustomServerTest extends Scope
'activate' => true
]);
$id = $data['functionId'] ?? '';
$functionId = $data['functionId'] ?? '';
$deploymentId = $deployment['body']['$id'] ?? '';
$this->assertEquals($deployment['headers']['status-code'], 202);
@ -523,7 +523,7 @@ class WebhooksCustomServerTest extends Scope
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);
sleep(5);
$this->awaitDeploymentIsBuilt($functionId, $deploymentId);
return array_merge($data, ['deploymentId' => $deploymentId]);
}

View file

@ -36,6 +36,31 @@ class V16Test extends TestCase
'buildStderr' => '',
],
],
'size and buildSize' => [
[
'size' => 20,
'buildSize' => 40
],
[
'size' => 60
],
],
'size and buildSize missing' => [
[
'size' => 20
],
[
'size' => 20
],
],
'empty no errors' => [
[
],
[
],
],
];
}