CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-14 16:38:28 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8323 from appwrite/feat-opr-v4-support

Browse source 
OPR v4 support
This commit is contained in:
Damodar Lohani 2024-07-03 15:11:54 +05:45 committed by GitHub
commit 75cd558670
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 417 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/config/collections.php
View file

@ -3029,7 +3029,7 @@ $projectCollections = array_merge([
'size' => 8,
'signed' => true,
'required' => false,
'default' => 'v3',
'default' => 'v4',
'array' => false,
'filters' => [],
],

2
app/config/runtimes.php
View file

@ -6,4 +6,4 @@
use Appwrite\Runtimes\Runtimes;
return (new Runtimes('v3'))->getAll();
return (new Runtimes('v4'))->getAll();

13
app/controllers/api/functions.php
View file

@ -223,7 +223,7 @@ App::post('/v1/functions')
'commands' => $commands,
'scopes' => $scopes,
'search' => implode(' ', [$functionId, $name, $runtime]),
'version' => 'v3',
'version' => 'v4',
'installationId' => $installation->getId(),
'installationInternalId' => $installation->getInternalId(),
'providerRepositoryId' => $providerRepositoryId,
@ -1743,10 +1743,7 @@ App::post('/v1/functions/:functionId/executions')
->setContext('function', $function);
if ($async) {
if ($function->getAttribute('logging')) {
/** @var Document $execution */
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
}
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
if(is_null($scheduledAt)) {
$queueForFunctions
@ -1850,6 +1847,7 @@ App::post('/v1/functions/:functionId/executions')
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
logging: $function->getAttribute('logging', true),
requestTimeout: 30
);
@ -1889,10 +1887,7 @@ App::post('/v1/functions/:functionId/executions')
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
;
if ($function->getAttribute('logging')) {
/** @var Document $execution */
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
}
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
}
$roles = Authorization::getRoles();

8
app/controllers/general.php
View file

@ -273,6 +273,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
method: $method,
headers: $headers,
runtimeEntrypoint: $command,
logging: $function->getAttribute('logging', true),
requestTimeout: 30
);
@ -332,13 +333,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
$body = $execution['responseBody'] ?? '';
$encodingKey = \array_search('x-open-runtimes-encoding', \array_column($execution['responseHeaders'], 'name'));
if ($encodingKey !== false) {
if (($execution['responseHeaders'][$encodingKey]['value'] ?? '') === 'base64') {
$body = \base64_decode($body);
}
}
$contentType = 'text/plain';
foreach ($execution['responseHeaders'] as $header) {
if (\strtolower($header['name']) === 'content-type') {

16
composer.lock generated
View file

@ -3406,16 +3406,16 @@
},
{
"name": "nikic/php-parser",
"version": "v5.0.2",
"version": "v5.1.0",
"source": {
"type": "git",
"url": "https://github.com/nikic/PHP-Parser.git",
"reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13"
"reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/139676794dc1e9231bf7bcd123cfc0c99182cb13",
"reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13",
"url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/683130c2ff8c2739f4822ff7ac5c873ec529abd1",
"reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1",
"shasum": ""
},
"require": {
@ -3426,7 +3426,7 @@
},
"require-dev": {
"ircmaxell/php-yacc": "^0.0.7",
"phpunit/phpunit": "^7.0 || ^8.0 || ^9.0"
"phpunit/phpunit": "^9.0"
},
"bin": [
"bin/php-parse"
@ -3458,9 +3458,9 @@
],
"support": {
"issues": "https://github.com/nikic/PHP-Parser/issues",
"source": "https://github.com/nikic/PHP-Parser/tree/v5.0.2"
"source": "https://github.com/nikic/PHP-Parser/tree/v5.1.0"
},
"time": "2024-03-05T20:51:40+00:00"
"time": "2024-07-01T20:03:41+00:00"
},
{
"name": "phar-io/manifest",
@ -5615,5 +5615,5 @@
"platform-overrides": {
"php": "8.3"
},
"plugin-api-version": "2.6.0"
"plugin-api-version": "2.3.0"
}

4
docker-compose.yml
View file

@ -851,7 +851,7 @@ services:
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.5.5
image: openruntimes/executor:0.6.0
restart: unless-stopped
networks:
- appwrite
@ -872,7 +872,7 @@ services:
- OPR_EXECUTOR_ENV=$_APP_ENV
- OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES
- OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET
- OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v3
- OPR_EXECUTOR_RUNTIME_VERSIONS=v2,v4
- OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
- OPR_EXECUTOR_STORAGE_DEVICE=$_APP_STORAGE_DEVICE
- OPR_EXECUTOR_STORAGE_S3_ACCESS_KEY=$_APP_STORAGE_S3_ACCESS_KEY

17
src/Appwrite/Platform/Workers/Functions.php
View file

@ -405,9 +405,7 @@ class Functions extends Action
'search' => implode(' ', [$functionId, $executionId]),
]);
if ($function->getAttribute('logging')) {
$execution = $dbForProject->createDocument('executions', $execution);
}
$execution = $dbForProject->createDocument('executions', $execution);
// TODO: @Meldiron Trigger executions.create event here
@ -419,9 +417,7 @@ class Functions extends Action
if ($execution->getAttribute('status') !== 'processing') {
$execution->setAttribute('status', 'processing');
if ($function->getAttribute('logging')) {
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
}
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
}
$durationStart = \microtime(true);
@ -490,7 +486,8 @@ class Functions extends Action
path: $path,
method: $method,
headers: $headers,
runtimeEntrypoint: $command
runtimeEntrypoint: $command,
logging: $function->getAttribute('logging', true),
);
$status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed';
@ -532,9 +529,9 @@ class Functions extends Action
;
}
if ($function->getAttribute('logging')) {
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
}
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
/** Trigger Webhook */
$executionModel = new Execution();
$queueForEvents

147
src/Appwrite/Utopia/Fetch/BodyMultipart.php Normal file
View file

@ -0,0 +1,147 @@
<?php
namespace Appwrite\Utopia\Fetch;
class BodyMultipart
{
/**
* @var array<string, mixed> $parts
*/
private array $parts = [];
private string $boundary = "";
public function __construct(string $boundary = null)
{
if (is_null($boundary)) {
$this->boundary = self::generateBoundary();
} else {
$this->boundary = $boundary;
}
}
public static function generateBoundary(): string
{
return '-----------------------------' . \uniqid();
}
public function load(string $body): self
{
$eol = "\r\n";
$sections = \explode('--' . $this->boundary, $body);
foreach ($sections as $section) {
if (empty($section)) {
continue;
}
if (strpos($section, $eol) === 0) {
$section = substr($section, \strlen($eol));
}
if (substr($section, -2) === $eol) {
$section = substr($section, 0, -1 * \strlen($eol));
}
if ($section == '--') {
continue;
}
$partChunks = \explode($eol . $eol, $section, 2);
if (\count($partChunks) < 2) {
continue; // Broken part
}
[ $partHeaders, $partBody ] = $partChunks;
$partHeaders = \explode($eol, $partHeaders);
$partName = "";
foreach ($partHeaders as $partHeader) {
if (!empty($partName)) {
break;
}
$partHeaderArray = \explode(':', $partHeader, 2);
$partHeaderName = \strtolower($partHeaderArray[0] ?? '');
$partHeaderValue = $partHeaderArray[1] ?? '';
if ($partHeaderName == "content-disposition") {
$dispositionChunks = \explode("; ", $partHeaderValue);
foreach ($dispositionChunks as $dispositionChunk) {
$dispositionChunkValues = \explode("=", $dispositionChunk, 2);
if (\count($dispositionChunkValues) >= 2) {
if ($dispositionChunkValues[0] === "name") {
$partName = \trim($dispositionChunkValues[1], "\"");
break;
}
}
}
}
}
if (!empty($partName)) {
$this->parts[$partName] = $partBody;
}
}
return $this;
}
/**
* @return array<string, mixed>
*/
public function getParts(): array
{
return $this->parts ?? [];
}
public function getPart(string $key, mixed $default = ''): mixed
{
return $this->parts[$key] ?? $default;
}
public function setPart(string $key, mixed $value): self
{
$this->parts[$key] = $value;
return $this;
}
public function getBoundary(): string
{
return $this->boundary;
}
public function setBoundary(string $boundary): self
{
$this->boundary = $boundary;
return $this;
}
public function exportHeader(): string
{
return 'multipart/form-data; boundary=' . $this->boundary;
}
public function exportBody(): string
{
$eol = "\r\n";
$query = '--' . $this->boundary;
foreach ($this->parts as $key => $value) {
$query .= $eol . 'Content-Disposition: form-data; name="' . $key . '"';
if (\is_array($value)) {
$query .= $eol . 'Content-Type: application/json';
$value = \json_encode($value);
}
$query .= $eol . $eol;
$query .= $value . $eol;
$query .= '--' . $this->boundary;
}
$query .= "--" . $eol;
return $query;
}
}

2
src/Appwrite/Utopia/Response/Model/Func.php
View file

@ -119,7 +119,7 @@ class Func extends Model
->addRule('version', [
'type' => self::TYPE_STRING,
'description' => 'Version of Open Runtimes used for the function.',
'default' => 'v3',
'default' => 'v4',
'example' => 'v2',
])
->addRule('installationId', [

34
src/Executor/Executor.php
View file

@ -3,6 +3,7 @@
namespace Executor;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Utopia\Fetch\BodyMultipart;
use Exception;
use Utopia\System\System;
@ -178,6 +179,7 @@ class Executor
string $method,
array $headers,
string $runtimeEntrypoint = null,
bool $logging,
int $requestTimeout = null
) {
if (empty($headers['host'])) {
@ -189,7 +191,6 @@ class Executor
$params = [
'runtimeId' => $runtimeId,
'variables' => $variables,
'body' => $body,
'timeout' => $timeout,
'path' => $path,
'method' => $method,
@ -201,15 +202,20 @@ class Executor
'memory' => $this->memory,
'version' => $version,
'runtimeEntrypoint' => $runtimeEntrypoint,
'logging' => $logging,
];
if(!empty($body)) {
$params['body'] = $body;
}
// Safety timeout. Executor has timeout, and open runtime has soft timeout.
// This one shouldn't really happen, but prevents from unexpected networking behaviours.
if ($requestTimeout == null) {
$requestTimeout = $timeout + 15;
}
$response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $requestTimeout);
$response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId, 'content-type' => 'multipart/form-data', 'accept' => 'multipart/form-data' ], $params, true, $requestTimeout);
$status = $response['headers']['status-code'];
if ($status >= 400) {
@ -217,6 +223,11 @@ class Executor
throw new \Exception($message, $status);
}
$response['body']['headers'] = \json_decode($response['body']['headers'] ?? '{}', true);
$response['body']['statusCode'] = \intval($response['body']['statusCode'] ?? 500);
$response['body']['duration'] = \intval($response['body']['duration'] ?? 0);
$response['body']['startTime'] = \intval($response['body']['startTime'] ?? \microtime(true));
return $response['body'];
}
@ -248,7 +259,13 @@ class Executor
break;
case 'multipart/form-data':
$query = $this->flatten($params);
$multipart = new BodyMultipart();
foreach ($params as $key => $value) {
$multipart->setPart($key, $value);
}
$headers['content-type'] = $multipart->exportHeader();
$query = $multipart->exportBody();
break;
default:
@ -315,7 +332,16 @@ class Executor
$curlErrorMessage = curl_error($ch);
if ($decode) {
switch (substr($responseType, 0, strpos($responseType, ';'))) {
$strpos = strpos($responseType, ';');
$strpos = \is_bool($strpos) ? \strlen($responseType) : $strpos;
switch (substr($responseType, 0, $strpos)) {
case 'multipart/form-data':
$boundary = \explode('boundary=', $responseHeaders['content-type'] ?? '')[1] ?? '';
$multipartResponse = new BodyMultipart($boundary);
$multipartResponse->load(\is_bool($responseBody) ? '' : $responseBody);
$responseBody = $multipartResponse->getParts();
break;
case 'application/json':
$json = json_decode($responseBody, true);

3
tests/e2e/Client.php
View file

@ -163,7 +163,7 @@ class Client
* @return array
* @throws Exception
*/
public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true): array
public function call(string $method, string $path = '', array $headers = [], mixed $params = [], bool $decode = true): array
{
$headers = array_merge($this->headers, $headers);
$ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : ''));
@ -174,6 +174,7 @@ class Client
'application/json' => json_encode($params),
'multipart/form-data' => $this->flatten($params),
'application/graphql' => $params[0],
'text/plain' => $params,
default => http_build_query($params),
};

1
tests/e2e/General/HTTPTest.php
View file

@ -108,7 +108,6 @@ class HTTPTest extends Scope
'0.14.x',
];
// var_dump($files);
foreach ($files as $file) {
if (in_array($file, ['.', '..'])) {
continue;

201
tests/e2e/Services/Functions/FunctionsCustomServerTest.php
View file

@ -1804,4 +1804,205 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testFunctionsDomainBianryResponse()
{
$timeout = 15;
$code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-response/code.tar.gz";
$this->packageCode('php-binary-response');
$function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'functionId' => ID::unique(),
'name' => 'Test PHP Binary executions',
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'timeout' => $timeout,
'execute' => ['any']
]);
$functionId = $function['body']['$id'] ?? '';
$this->assertEquals(201, $function['headers']['status-code']);
$rules = $this->client->call(Client::METHOD_GET, '/proxy/rules', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => [
Query::equal('resourceId', [$functionId])->toString(),
Query::equal('resourceType', ['function'])->toString(),
],
]);
$this->assertEquals(200, $rules['headers']['status-code']);
$this->assertEquals(1, $rules['body']['total']);
$this->assertCount(1, $rules['body']['rules']);
$this->assertNotEmpty($rules['body']['rules'][0]['domain']);
$domain = $rules['body']['rules'][0]['domain'];
$deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'entrypoint' => 'index.php',
'code' => new CURLFile($code, 'application/x-gzip', basename($code)),
'activate' => true
]);
$deploymentId = $deployment['body']['$id'] ?? '';
$this->assertEquals(202, $deployment['headers']['status-code']);
// Poll until deployment is built
while (true) {
$deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
if (
$deployment['headers']['status-code'] >= 400
|| \in_array($deployment['body']['status'], ['ready', 'failed'])
) {
break;
}
\sleep(1);
}
$deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $deployment['headers']['status-code']);
// Wait a little for activation to finish
sleep(5);
$proxyClient = new Client();
$proxyClient->setEndpoint('http://' . $domain);
$response = $proxyClient->call(Client::METHOD_GET, '/', [], [], false);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']);
$bytes = unpack('C*byte', $response['body']);
$this->assertCount(3, $bytes);
$this->assertEquals(0, $bytes['byte1']);
$this->assertEquals(10, $bytes['byte2']);
$this->assertEquals(255, $bytes['byte3']);
// Cleanup : Delete function
$response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], []);
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testFunctionsDomainBianryRequest()
{
$timeout = 15;
$code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-request/code.tar.gz";
$this->packageCode('php-binary-request');
$function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'functionId' => ID::unique(),
'name' => 'Test PHP Binary executions',
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'timeout' => $timeout,
'execute' => ['any']
]);
$functionId = $function['body']['$id'] ?? '';
$this->assertEquals(201, $function['headers']['status-code']);
$rules = $this->client->call(Client::METHOD_GET, '/proxy/rules', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => [
Query::equal('resourceId', [$functionId])->toString(),
Query::equal('resourceType', ['function'])->toString(),
],
]);
$this->assertEquals(200, $rules['headers']['status-code']);
$this->assertEquals(1, $rules['body']['total']);
$this->assertCount(1, $rules['body']['rules']);
$this->assertNotEmpty($rules['body']['rules'][0]['domain']);
$domain = $rules['body']['rules'][0]['domain'];
$deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'entrypoint' => 'index.php',
'code' => new CURLFile($code, 'application/x-gzip', basename($code)),
'activate' => true
]);
$deploymentId = $deployment['body']['$id'] ?? '';
$this->assertEquals(202, $deployment['headers']['status-code']);
// Poll until deployment is built
while (true) {
$deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
if (
$deployment['headers']['status-code'] >= 400
|| \in_array($deployment['body']['status'], ['ready', 'failed'])
) {
break;
}
\sleep(1);
}
$deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $deployment['headers']['status-code']);
// Wait a little for activation to finish
sleep(5);
$proxyClient = new Client();
$proxyClient->setEndpoint('http://' . $domain);
$bytes = pack('C*', ...[0,20,255]);
$response = $proxyClient->call(Client::METHOD_POST, '/', [ 'content-type' => 'text/plain' ], $bytes, false);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(\md5($bytes), $response['body']);
// Cleanup : Delete function
$response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], []);
$this->assertEquals(204, $response['headers']['status-code']);
}
}

6
tests/resources/functions/php-binary-request/index.php Normal file
View file

@ -0,0 +1,6 @@
<?php
return function ($context) {
$hash = md5($context->req->bodyBinary);
return $context->res->send($hash);
};

6
tests/resources/functions/php-binary-response/index.php Normal file
View file

@ -0,0 +1,6 @@
<?php
return function ($context) {
$bytes = pack('C*', ...[0, 10, 255]);
return $context->res->binary($bytes);
};