diff --git a/.env b/.env index 5eceb3a819..cfc76aad6f 100644 --- a/.env +++ b/.env @@ -15,7 +15,7 @@ _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled _APP_OPENSSL_KEY_V1=your-secret-key -_APP_DOMAIN=localhost +_APP_DOMAIN=traefik _APP_DOMAIN_FUNCTIONS=functions.localhost _APP_DOMAIN_TARGET=localhost _APP_REDIS_HOST=redis diff --git a/app/config/collections.php b/app/config/collections.php index 2509ccc4ba..3e0dd9b7a7 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3054,7 +3054,18 @@ $projectCollections = array_merge([ 'required' => false, 'default' => null, 'filters' => [], - ] + ], + [ + '$id' => ID::custom('scopes'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/config/errors.php b/app/config/errors.php index 287e639f6b..05409cf8cb 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -332,6 +332,11 @@ return [ 'description' => 'API key and session used in the same request. Use either `setSession` or `setKey`. Learn about which authentication method to use in the SSR docs: https://appwrite.io/docs/products/auth/server-side-rendering', 'code' => 403, ], + Exception::API_KEY_EXPIRED => [ + 'name' => Exception::API_KEY_EXPIRED, + 'description' => 'The dynamic API key has expired. Please don\'t use dynamic API keys for more than duration of the execution.', + 'code' => 401, + ], /** Teams */ Exception::TEAM_NOT_FOUND => [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 4ff2b40241..670ca2f99e 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -150,6 +150,7 @@ App::post('/v1/functions') ->param('logging', true, new Boolean(), 'Whether executions will be logged. When set to false, executions will not be logged, but will reduce resource used by your Appwrite project.', true) ->param('entrypoint', '', new Text(1028, 0), 'Entrypoint File. This path is relative to the "providerRootDirectory".', true) ->param('commands', '', new Text(8192, 0), 'Build Commands.', true) + ->param('scopes', [], new ArrayList(new WhiteList(array_keys(Config::getParam('scopes')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of scopes allowed for API key auto-generated for every execution. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed.', true) ->param('installationId', '', new Text(128, 0), 'Appwrite Installation ID for VCS (Version Control System) deployment.', true) ->param('providerRepositoryId', '', new Text(128, 0), 'Repository ID of the repo linked to the function.', true) ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function.', true) @@ -168,7 +169,7 @@ App::post('/v1/functions') ->inject('queueForBuilds') ->inject('dbForConsole') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateBranch, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateBranch, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; $allowList = \array_filter(\explode(',', System::getEnv('_APP_FUNCTIONS_RUNTIMES', ''))); @@ -218,6 +219,7 @@ App::post('/v1/functions') 'timeout' => $timeout, 'entrypoint' => $entrypoint, 'commands' => $commands, + 'scopes' => $scopes, 'search' => implode(' ', [$functionId, $name, $runtime]), 'version' => 'v3', 'installationId' => $installation->getId(), @@ -681,6 +683,7 @@ App::put('/v1/functions/:functionId') ->param('logging', true, new Boolean(), 'Whether executions will be logged. When set to false, executions will not be logged, but will reduce resource used by your Appwrite project.', true) ->param('entrypoint', '', new Text(1028, 0), 'Entrypoint File. This path is relative to the "providerRootDirectory".', true) ->param('commands', '', new Text(8192, 0), 'Build Commands.', true) + ->param('scopes', [], new ArrayList(new WhiteList(array_keys(Config::getParam('scopes')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of scopes allowed for API Key auto-generated for every execution. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed.', true) ->param('installationId', '', new Text(128, 0), 'Appwrite Installation ID for VCS (Version Controle System) deployment.', true) ->param('providerRepositoryId', '', new Text(128, 0), 'Repository ID of the repo linked to the function', true) ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function', true) @@ -694,7 +697,7 @@ App::put('/v1/functions/:functionId') ->inject('queueForBuilds') ->inject('dbForConsole') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) use ($redeployVcs) { // TODO: If only branch changes, re-deploy $function = $dbForProject->getDocument('functions', $functionId); @@ -806,6 +809,7 @@ App::put('/v1/functions/:functionId') 'logging' => $logging, 'entrypoint' => $entrypoint, 'commands' => $commands, + 'scopes' => $scopes, 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), 'providerRepositoryId' => $providerRepositoryId, @@ -1589,6 +1593,14 @@ App::post('/v1/functions/:functionId/executions') } } + $jwtExpiry = $function->getAttribute('timeout', 900); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 10); + $apiKey = $jwtObj->encode([ + 'projectId' => $project->getId(), + 'scopes' => $function->getAttribute('scopes', []) + ]); + + $headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey; $headers['x-appwrite-trigger'] = 'http'; $headers['x-appwrite-user-id'] = $user->getId() ?? ''; $headers['x-appwrite-user-jwt'] = $jwt ?? ''; @@ -1693,8 +1705,13 @@ App::post('/v1/functions/:functionId/executions') $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; + $hostname = System::getEnv('_APP_DOMAIN'); + $endpoint = $protocol . '://' . $hostname . "/v1"; + // Appwrite vars $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, 'APPWRITE_FUNCTION_ID' => $functionId, 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index cb75b76da5..d81c37a61c 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -1148,7 +1148,7 @@ App::post('/v1/projects/:projectId/keys') 'expire' => $expire, 'sdks' => [], 'accessedAt' => null, - 'secret' => \bin2hex(\random_bytes(128)), + 'secret' => API_KEY_STANDARD . '_' . \bin2hex(\random_bytes(128)), ]); $key = $dbForConsole->createDocument('keys', $key); diff --git a/app/controllers/mock.php b/app/controllers/mock.php index 6679fa14f5..fdb1d80dcc 100644 --- a/app/controllers/mock.php +++ b/app/controllers/mock.php @@ -6,6 +6,7 @@ use Appwrite\Extend\Exception; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -154,6 +155,55 @@ App::patch('/v1/mock/functions-v2') $response->noContent(); }); +App::post('/v1/mock/api-key-unprefixed') + ->desc('Create API Key (without standard prefix)') + ->groups(['mock', 'api', 'projects']) + ->label('scope', 'projects.write') + ->label('docs', false) + ->param('projectId', '', new UID(), 'Project ID.') + ->inject('response') + ->inject('dbForConsole') + ->action(function (string $projectId, Response $response, Database $dbForConsole) { + $isDevelopment = System::getEnv('_APP_ENV', 'development') === 'development'; + + if (!$isDevelopment) { + throw new Exception(Exception::GENERAL_NOT_IMPLEMENTED); + } + + $project = $dbForConsole->getDocument('projects', $projectId); + + if ($project->isEmpty()) { + throw new Exception(Exception::PROJECT_NOT_FOUND); + } + + $scopes = array_keys(Config::getParam('scopes')); + + $key = new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'projectInternalId' => $project->getInternalId(), + 'projectId' => $project->getId(), + 'name' => 'Outdated key', + 'scopes' => $scopes, + 'expire' => null, + 'sdks' => [], + 'accessedAt' => null, + 'secret' => \bin2hex(\random_bytes(128)), + ]); + + $key = $dbForConsole->createDocument('keys', $key); + + $dbForConsole->purgeCachedDocument('projects', $project->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($key, Response::MODEL_KEY); + }); + App::get('/v1/mock/github/callback') ->desc('Create installation document using GitHub installation id') ->groups(['mock', 'api', 'vcs']) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 1afd6b652e..fc6792a913 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -1,5 +1,7 @@ getLabel('scope', 'none'); // Allowed scope for chosen route $scopes = $roles[$role]['scopes']; // Allowed scopes for user role - $authKey = $request->getHeader('x-appwrite-key', ''); + $apiKey = $request->getHeader('x-appwrite-key', ''); - if (!empty($authKey)) { // API Key authentication + // API Key authentication + if (!empty($apiKey)) { // Do not allow API key and session to be set at the same time if (!$user->isEmpty()) { throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } - // Check if given key match project API keys - $key = $project->find('secret', $authKey, 'keys'); - if ($key) { - $user = new Document([ - '$id' => '', - 'status' => true, - 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), - 'password' => '', - 'name' => $project->getAttribute('name', 'Untitled'), - ]); + if(!\str_contains($apiKey, '_')) { + $keyType = API_KEY_STANDARD; + $authKey = $apiKey; + } else { + [ $keyType, $authKey ] = \explode('_', $apiKey, 2); + } - $role = Auth::USER_ROLE_APPS; - $scopes = \array_merge($roles[$role]['scopes'], $key->getAttribute('scopes', [])); + if($keyType === API_KEY_DYNAMIC) { + // Dynamic key - $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) { - throw new Exception(Exception::PROJECT_KEY_EXPIRED); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 10); + + try { + $payload = $jwtObj->decode($authKey); + } catch (JWTException $error) { + throw new Exception(Exception::API_KEY_EXPIRED); } - Authorization::setRole(Auth::USER_ROLE_APPS); - Authorization::setDefaultStatus(false); // Cancel security segmentation for API keys. + $projectId = $payload['projectId'] ?? ''; + $tokenScopes = $payload['scopes'] ?? []; - $accessedAt = $key->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCCESS)) > $accessedAt) { - $key->setAttribute('accessedAt', DateTime::now()); - $dbForConsole->updateDocument('keys', $key->getId(), $key); - $dbForConsole->purgeCachedDocument('projects', $project->getId()); + // JWT includes project ID for better security + if ($projectId === $project->getId()) { + $user = new Document([ + '$id' => '', + 'status' => true, + 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), + 'password' => '', + 'name' => $project->getAttribute('name', 'Untitled'), + ]); + + $role = Auth::USER_ROLE_APPS; + $scopes = \array_merge($roles[$role]['scopes'], $tokenScopes); + + Authorization::setRole(Auth::USER_ROLE_APPS); + Authorization::setDefaultStatus(false); // Cancel security segmentation for API keys. } + } elseif($keyType === API_KEY_STANDARD) { + // No underline means no prefix. Backwards compatibility. + // Regular key - $sdkValidator = new WhiteList($servers, true); - $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); - if ($sdkValidator->isValid($sdk)) { - $sdks = $key->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { - array_push($sdks, $sdk); - $key->setAttribute('sdks', $sdks); + // Check if given key match project API keys + $key = $project->find('secret', $apiKey, 'keys'); + if ($key) { + $user = new Document([ + '$id' => '', + 'status' => true, + 'email' => 'app.' . $project->getId() . '@service.' . $request->getHostname(), + 'password' => '', + 'name' => $project->getAttribute('name', 'Untitled'), + ]); - /** Update access time as well */ - $key->setAttribute('accessedAt', Datetime::now()); + $role = Auth::USER_ROLE_APPS; + $scopes = \array_merge($roles[$role]['scopes'], $key->getAttribute('scopes', [])); + + $expire = $key->getAttribute('expire'); + if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) { + throw new Exception(Exception::PROJECT_KEY_EXPIRED); + } + + Authorization::setRole(Auth::USER_ROLE_APPS); + Authorization::setDefaultStatus(false); // Cancel security segmentation for API keys. + + $accessedAt = $key->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCCESS)) > $accessedAt) { + $key->setAttribute('accessedAt', DateTime::now()); $dbForConsole->updateDocument('keys', $key->getId(), $key); $dbForConsole->purgeCachedDocument('projects', $project->getId()); } + + $sdkValidator = new WhiteList($servers, true); + $sdk = $request->getHeader('x-sdk-name', 'UNKNOWN'); + if ($sdkValidator->isValid($sdk)) { + $sdks = $key->getAttribute('sdks', []); + if (!in_array($sdk, $sdks)) { + array_push($sdks, $sdk); + $key->setAttribute('sdks', $sdks); + + /** Update access time as well */ + $key->setAttribute('accessedAt', Datetime::now()); + $dbForConsole->updateDocument('keys', $key->getId(), $key); + $dbForConsole->purgeCachedDocument('projects', $project->getId()); + } + } } } } diff --git a/app/init.php b/app/init.php index f94f7e380b..3c3bc445b8 100644 --- a/app/init.php +++ b/app/init.php @@ -208,6 +208,9 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length']; const MESSAGE_TYPE_EMAIL = 'email'; const MESSAGE_TYPE_SMS = 'sms'; const MESSAGE_TYPE_PUSH = 'push'; +// API key types +const API_KEY_STANDARD = 'standard'; +const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; @@ -228,11 +231,19 @@ const METRIC_FUNCTIONS = 'functions'; const METRIC_DEPLOYMENTS = 'deployments'; const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage'; const METRIC_BUILDS = 'builds'; +const METRIC_BUILDS_SUCCESS = 'builds.success'; +const METRIC_BUILDS_FAILED = 'builds.failed'; const METRIC_BUILDS_STORAGE = 'builds.storage'; const METRIC_BUILDS_COMPUTE = 'builds.compute'; +const METRIC_BUILDS_COMPUTE_SUCCESS = 'builds.compute.success'; +const METRIC_BUILDS_COMPUTE_FAILED = 'builds.compute.failed'; const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds'; +const METRIC_FUNCTION_ID_BUILDS_SUCCESS = '{functionInternalId}.builds.success'; +const METRIC_FUNCTION_ID_BUILDS_FAILED = '{functionInternalId}.builds.failed'; const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute'; +const METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS = '{functionInternalId}.builds.compute.success'; +const METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED = '{functionInternalId}.builds.compute.failed'; const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; const METRIC_EXECUTIONS = 'executions'; @@ -340,8 +351,7 @@ Database::addFilter( if (isset($formatOptions['min']) || isset($formatOptions['max'])) { $attribute ->setAttribute('min', $formatOptions['min']) - ->setAttribute('max', $formatOptions['max']) - ; + ->setAttribute('max', $formatOptions['max']); } return $value; diff --git a/composer.lock b/composer.lock index 34a3cc143c..7c660bf176 100644 --- a/composer.lock +++ b/composer.lock @@ -1966,16 +1966,16 @@ }, { "name": "utopia-php/migration", - "version": "0.4.0", + "version": "0.4.1", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a72f27bd3dde68752fb185d306c4820e1b8d9657" + "reference": "ae3cfe93f6d313105d226aeb68806660c806a925" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a72f27bd3dde68752fb185d306c4820e1b8d9657", - "reference": "a72f27bd3dde68752fb185d306c4820e1b8d9657", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/ae3cfe93f6d313105d226aeb68806660c806a925", + "reference": "ae3cfe93f6d313105d226aeb68806660c806a925", "shasum": "" }, "require": { @@ -2008,9 +2008,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.4.0" + "source": "https://github.com/utopia-php/migration/tree/0.4.1" }, - "time": "2024-02-25T12:35:21+00:00" + "time": "2024-05-01T13:19:18+00:00" }, { "name": "utopia-php/mongo", @@ -2899,16 +2899,16 @@ }, { "name": "laravel/pint", - "version": "v1.15.2", + "version": "v1.15.3", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "2c9f8004899815f3f0ee3cb28ef7281e2b589134" + "reference": "3600b5d17aff52f6100ea4921849deacbbeb8656" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/2c9f8004899815f3f0ee3cb28ef7281e2b589134", - "reference": "2c9f8004899815f3f0ee3cb28ef7281e2b589134", + "url": "https://api.github.com/repos/laravel/pint/zipball/3600b5d17aff52f6100ea4921849deacbbeb8656", + "reference": "3600b5d17aff52f6100ea4921849deacbbeb8656", "shasum": "" }, "require": { @@ -2961,7 +2961,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-04-23T15:42:34+00:00" + "time": "2024-04-30T15:02:26+00:00" }, { "name": "matthiasmullie/minify", diff --git a/docker-compose.yml b/docker-compose.yml index bf69f7caf4..250eb8b7aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,6 +42,7 @@ services: networks: - gateway - appwrite + - runtimes appwrite: container_name: appwrite @@ -497,6 +498,8 @@ services: - _APP_ENV - _APP_WORKER_PER_CORE - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_OPTIONS_FORCE_HTTPS - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 1c5692cd9d..08fb899992 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -107,6 +107,8 @@ class Exception extends \Exception public const USER_TARGET_ALREADY_EXISTS = 'user_target_already_exists'; public const USER_API_KEY_AND_SESSION_SET = 'user_key_and_session_set'; + public const API_KEY_EXPIRED = 'api_key_expired'; + /** Teams */ public const TEAM_NOT_FOUND = 'team_not_found'; public const TEAM_INVITE_ALREADY_EXISTS = 'team_invite_already_exists'; diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index ff1c439aa2..7f43d776ce 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -339,14 +339,13 @@ class Builds extends Action $deploymentModel = new Deployment(); $deploymentUpdate = $queueForEvents - ->setQueue(Event::WEBHOOK_QUEUE_NAME) - ->setClass(Event::WEBHOOK_CLASS_NAME) - ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $function->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))) - ; + ->setQueue(Event::WEBHOOK_QUEUE_NAME) + ->setClass(Event::WEBHOOK_CLASS_NAME) + ->setProject($project) + ->setEvent('functions.[functionId].deployments.[deploymentId].update') + ->setParam('functionId', $function->getId()) + ->setParam('deploymentId', $deployment->getId()) + ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); $deploymentUpdate->trigger(); @@ -437,8 +436,8 @@ class Builds extends Action $build = $dbForProject->updateDocument('builds', $build->getId(), $build); /** - * Send realtime Event - */ + * Send realtime Event + */ $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], @@ -534,6 +533,20 @@ class Builds extends Action ); /** Trigger usage queue */ + if ($build->getAttribute('status') === 'ready') { + $queueForUsage + ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); + } elseif ($build->getAttribute('status') === 'failed') { + $queueForUsage + ->addMetric(METRIC_BUILDS_FAILED, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000); + } + $queueForUsage ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index e527456c7e..fc9a1242ac 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Workers; +use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Usage; @@ -282,6 +283,14 @@ class Functions extends Action $runtime = $runtimes[$function->getAttribute('runtime')]; + $jwtExpiry = $function->getAttribute('timeout', 900); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 10); + $apiKey = $jwtObj->encode([ + 'projectId' => $project->getId(), + 'scopes' => $function->getAttribute('scopes', []) + ]); + + $headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey; $headers['x-appwrite-trigger'] = $trigger; $headers['x-appwrite-event'] = $event ?? ''; $headers['x-appwrite-user-id'] = $user->getId() ?? ''; @@ -369,8 +378,13 @@ class Functions extends Action $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; + $hostname = System::getEnv('_APP_DOMAIN'); + $endpoint = $protocol . '://' . $hostname . "/v1"; + // Appwrite vars $vars = \array_merge($vars, [ + 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, 'APPWRITE_FUNCTION_ID' => $functionId, 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId, diff --git a/src/Appwrite/Platform/Workers/Usage.php b/src/Appwrite/Platform/Workers/Usage.php index 48724c0d0d..034e558d5d 100644 --- a/src/Appwrite/Platform/Workers/Usage.php +++ b/src/Appwrite/Platform/Workers/Usage.php @@ -184,8 +184,12 @@ class Usage extends Action $deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS))); $deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE))); $builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS))); + $buildsSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS))); + $buildsFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED))); $buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE))); $buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE))); + $buildsComputeSuccess = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS))); + $buildsComputeFailed = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED))); $executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS))); $executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE))); @@ -210,6 +214,20 @@ class Usage extends Action ]; } + if (!empty($buildsSuccess['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_SUCCESS, + 'value' => ($buildsSuccess['value'] * -1), + ]; + } + + if (!empty($buildsFailed['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_FAILED, + 'value' => ($buildsFailed['value'] * -1), + ]; + } + if (!empty($buildsStorage['value'])) { $metrics[] = [ 'key' => METRIC_BUILDS_STORAGE, @@ -224,6 +242,20 @@ class Usage extends Action ]; } + if (!empty($buildsComputeSuccess['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_COMPUTE_SUCCESS, + 'value' => ($buildsComputeSuccess['value'] * -1), + ]; + } + + if (!empty($buildsComputeFailed['value'])) { + $metrics[] = [ + 'key' => METRIC_BUILDS_COMPUTE_FAILED, + 'value' => ($buildsComputeFailed['value'] * -1), + ]; + } + if (!empty($executions['value'])) { $metrics[] = [ 'key' => METRIC_EXECUTIONS, diff --git a/src/Appwrite/Utopia/Response/Model/Func.php b/src/Appwrite/Utopia/Response/Model/Func.php index 0c8d4d42d2..66c356e2b1 100644 --- a/src/Appwrite/Utopia/Response/Model/Func.php +++ b/src/Appwrite/Utopia/Response/Model/Func.php @@ -71,6 +71,13 @@ class Func extends Model 'default' => '', 'example' => '5e5ea5c16897e', ]) + ->addRule('scopes', [ + 'type' => self::TYPE_STRING, + 'description' => 'Allowed permission scopes.', + 'default' => [], + 'example' => 'users.read', + 'array' => true, + ]) ->addRule('vars', [ 'type' => Response::MODEL_VARIABLE, 'description' => 'Function variables.', diff --git a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php index 70bbedcb38..63fc3ca75f 100644 --- a/tests/e2e/Services/Databases/DatabasesCustomServerTest.php +++ b/tests/e2e/Services/Databases/DatabasesCustomServerTest.php @@ -1296,8 +1296,6 @@ class DatabasesCustomServerTest extends Scope 'x-appwrite-key' => $this->getProject()['apiKey'], ], $this->getHeaders())); - \var_dump($attributes['body']); - $this->assertEquals(0, $attributes['body']['total']); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 1b9edc4f02..6a8dc322e3 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1023,7 +1023,7 @@ class FunctionsCustomServerTest extends Scope */ public function testCreateCustomExecution(string $folder, string $name, string $entrypoint, string $runtimeName, string $runtimeVersion) { - $timeout = 5; + $timeout = 15; $code = realpath(__DIR__ . '/../../../resources/functions') . "/$folder/code.tar.gz"; $this->packageCode($folder); @@ -1144,7 +1144,7 @@ class FunctionsCustomServerTest extends Scope public function testv2Function() { - $timeout = 5; + $timeout = 15; $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-v2/code.tar.gz"; $this->packageCode('php-v2'); @@ -1264,7 +1264,7 @@ class FunctionsCustomServerTest extends Scope public function testEventTrigger() { - $timeout = 5; + $timeout = 15; $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-event/code.tar.gz"; $this->packageCode('php-event'); @@ -1374,9 +1374,94 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); } + public function testScopes() + { + $timeout = 15; + $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-scopes/code.tar.gz"; + $this->packageCode('php-scopes'); + + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'functionId' => ID::unique(), + 'name' => 'Test PHP Scopes executions', + 'commands' => 'composer update --no-interaction --ignore-platform-reqs --optimize-autoloader --prefer-dist --no-dev', + 'runtime' => 'php-8.0', + 'entrypoint' => 'index.php', + 'scopes' => ['users.read'], + 'timeout' => $timeout, + ]); + + $functionId = $function['body']['$id'] ?? ''; + + $this->assertEquals(201, $function['headers']['status-code']); + + $deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile($code, 'application/x-gzip', basename($code)), + 'activate' => true + ]); + + $deploymentId = $deployment['body']['$id'] ?? ''; + $this->assertEquals(202, $deployment['headers']['status-code']); + + // Poll until deployment is built + while (true) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $function['body']['$id'] . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + if ( + $deployment['headers']['status-code'] >= 400 + || \in_array($deployment['body']['status'], ['ready', 'failed']) + ) { + break; + } + + \sleep(1); + } + + $deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $deployment['headers']['status-code']); + + // Wait a little for activation to finish + sleep(5); + + $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'async' => false + ]); + + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status']); + $this->assertEquals(200, $execution['body']['responseStatusCode']); + $this->assertNotEmpty($execution['body']['responseBody']); + + // Cleanup : Delete function + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + + $this->assertEquals(204, $response['headers']['status-code']); + } + public function testCookieExecution() { - $timeout = 5; + $timeout = 15; $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-cookie/code.tar.gz"; $this->packageCode('php-cookie'); @@ -1464,7 +1549,7 @@ class FunctionsCustomServerTest extends Scope public function testFunctionsDomain() { - $timeout = 5; + $timeout = 15; $code = realpath(__DIR__ . '/../../../resources/functions') . "/php-cookie/code.tar.gz"; $this->packageCode('php-cookie'); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 6f60f01c73..b07ca1c8f4 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -2704,6 +2704,45 @@ class ProjectsConsoleClientTest extends Scope return $data; } + /** + * @depends testCreateProject + */ + public function testCreateProjectKeyOutdated($data): void + { + $id = $data['projectId'] ?? ''; + + $response = $this->client->call(Client::METHOD_POST, '/mock/api-key-unprefixed', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'projectId' => $id + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertContains('users.read', $response['body']['scopes']); + $this->assertNotEmpty($response['body']['secret']); + $this->assertStringStartsNotWith(API_KEY_STANDARD . '_', $response['body']['secret']); + + $keyId = $response['body']['$id']; + $secret = $response['body']['secret']; + + $response = $this->client->call(Client::METHOD_GET, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $id, + 'x-appwrite-key' => $secret + ], []); + + $this->assertEquals(200, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_DELETE, '/projects/' . $id . '/keys/' . $keyId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(204, $response['headers']['status-code']); + $this->assertEmpty($response['body']); + } + // Platforms /** diff --git a/tests/resources/functions/php-scopes/composer.json b/tests/resources/functions/php-scopes/composer.json new file mode 100644 index 0000000000..8eacdab3ec --- /dev/null +++ b/tests/resources/functions/php-scopes/composer.json @@ -0,0 +1,18 @@ +{ + "name": "appwrite/php-scopes", + "description": "PHP scopes test script", + "type": "library", + "license": "BSD-3-Clause", + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "require": { + "php": ">=7.4.0", + "ext-curl": "*", + "ext-json": "*", + "appwrite/appwrite": "11.0.*" + } +} diff --git a/tests/resources/functions/php-scopes/index.php b/tests/resources/functions/php-scopes/index.php new file mode 100644 index 0000000000..e05653b709 --- /dev/null +++ b/tests/resources/functions/php-scopes/index.php @@ -0,0 +1,16 @@ +setEndpoint(getenv('APPWRITE_FUNCTION_API_ENDPOINT')) + ->setProject(getenv('APPWRITE_FUNCTION_PROJECT_ID')) + ->setKey($context->req->headers['x-appwrite-key']); + $users = new Users($client); + return $context->res->json($users->list()); +};