CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-02 19:04:49 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2810 from appwrite/feat-func-storage-sync

Browse source 
Function Stroage Sync
This commit is contained in:
Christy Jacob 2022-02-22 15:29:56 +04:00 committed by GitHub
parent 46183c6654 5d9a9f53d8
commit 953f769ed6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 198 additions and 2530 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.env
View file

@ -34,6 +34,7 @@ _APP_SMTP_SECURE=
_APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_STORAGE_LIMIT=30000000
_APP_FUNCTIONS_DEPLOYMENT_LIMIT=30000000
_APP_FUNCTIONS_TIMEOUT=900
_APP_FUNCTIONS_BUILD_TIMEOUT=900
_APP_FUNCTIONS_CONTAINERS=10

2
CHANGES.md
View file

@ -1,4 +1,5 @@
# Unreleased Version 0.13.0
- Added ability to create syncronous function executions
- Introduced new execution model for functions
- Improved functions execution times
@ -10,6 +11,7 @@
- Updated endpoints to reflect the new terminology
- Updated UI with these changes
- Updated event names from `function.tags.*` to `function.deployments.*`
# Version 0.12.2
## Bugs

1616
app/config/collections.old.php
View file

File diff suppressed because it is too large Load diff

17
app/config/collections.php
View file

@ -1751,9 +1751,9 @@ $collections = [
'$id' => '_key_search',
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [2048],
'orders' => [Database::ORDER_ASC],
]
'lengths' => [],
'orders' => [],
],
],
],
@ -1873,6 +1873,17 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'metadata',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['json'],
],
[
'$id' => 'search',
'type' => Database::VAR_STRING,

6
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

6
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

10
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

10
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

10
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

10
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

9
app/config/variables.php
View file

@ -498,6 +498,15 @@ return [
'category' => 'Functions',
'description' => '',
'variables' => [
[
'name' => '_APP_FUNCTIONS_DEPLOYMENT_LIMIT',
'description' => 'The maximum size deployment in bytes. The default value is 30MB.',
'introduction' => '0.13.0',
'default' => '30000000',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_FUNCTIONS_TIMEOUT',
'description' => 'The maximum number of seconds allowed as a timeout value when creating a new function. The default value is 900 seconds.',

25
app/controllers/api/functions.php
View file

@ -3,6 +3,7 @@
use Ahc\Jwt\JWT;
use Appwrite\Auth\Auth;
use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Validator\CustomId;
use Utopia\Database\Validator\UID;
use Utopia\Storage\Validator\File;
@ -12,7 +13,6 @@ use Utopia\Storage\Validator\Upload;
use Appwrite\Utopia\Response;
use Appwrite\Task\Validator\Cron;
use Utopia\App;
use Utopia\Exception;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
@ -375,7 +375,7 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId')
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
throw new Exception('Deployment not found', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
if ($build->isEmpty()) {
@ -489,7 +489,7 @@ App::post('/v1/functions/:functionId/deployments')
$file = $request->getFiles('code');
$fileExt = new FileExt([FileExt::TYPE_GZIP]);
$fileSizeValidator = new FileSize(App::getEnv('_APP_STORAGE_LIMIT', 0));
$fileSizeValidator = new FileSize(App::getEnv('_APP_FUNCTIONS_DEPLOYMENT_LIMIT', 0));
$upload = new Upload();
if (empty($file)) {
@ -583,6 +583,7 @@ App::post('/v1/functions/:functionId/deployments')
'$read' => ['role:all'],
'$write' => ['role:all'],
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'dateCreated' => time(),
'entrypoint' => $entrypoint,
'path' => $path,
@ -612,6 +613,7 @@ App::post('/v1/functions/:functionId/deployments')
'$read' => ['role:all'],
'$write' => ['role:all'],
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'dateCreated' => time(),
'entrypoint' => $entrypoint,
'path' => $path,
@ -667,7 +669,8 @@ App::get('/v1/functions/:functionId/deployments')
$cursorDeployment = $dbForProject->getDocument('deployments', $cursor);
if ($cursorDeployment->isEmpty()) {
throw new Exception("Deployment '{$cursor}' for the 'cursor' value not found.", 400);
// TODO: Shouldn't this be a 404 error ?
throw new Exception("Tag '{$cursor}' for the 'cursor' value not found.", 400, Exception::GENERAL_CURSOR_NOT_FOUND);
}
}
@ -724,11 +727,11 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId')
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if ($deployment->getAttribute('resourceId') !== $function->getId()) {
throw new Exception('Deployment not found', 404);
throw new Exception('Deployment not found', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
throw new Exception('Deployment not found', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
$response->dynamic($deployment, Response::MODEL_DEPLOYMENT);
@ -766,16 +769,16 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId')
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found', 404);
throw new Exception('Deployment not found', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
if ($deployment->getAttribute('resourceId') !== $function->getId()) {
throw new Exception('Deployment not found', 404);
throw new Exception('Deployment not found', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
if ($deviceFunctions->delete($deployment->getAttribute('path', ''))) {
if (!$dbForProject->deleteDocument('deployments', $deployment->getId())) {
throw new Exception('Failed to remove deployment from DB', 500);
throw new Exception('Failed to remove deployment from DB', 500, Exception::GENERAL_SERVER_ERROR);
}
}
@ -841,11 +844,11 @@ App::post('/v1/functions/:functionId/executions')
$deployment = Authorization::skip(fn() => $dbForProject->getDocument('deployments', $function->getAttribute('deployment', '')));
if ($deployment->getAttribute('resourceId') !== $function->getId()) {
throw new Exception('Deployment not found. Deploy deployment before trying to execute a function', 404);
throw new Exception('Deployment not found. Deploy deployment before trying to execute a function', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found. Deploy deployment before trying to execute a function', 404);
throw new Exception('Deployment not found. Deploy deployment before trying to execute a function', 404, Exception::DEPLOYMENT_NOT_FOUND);
}
/** Check if build has completed */

39
app/executor.php
View file

@ -15,7 +15,10 @@ use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Orchestration\Adapter\DockerCLI;
use Utopia\Orchestration\Orchestration;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Storage;
use Utopia\Swoole\Request;
use Utopia\Swoole\Response;
@ -108,6 +111,27 @@ function logError(Throwable $error, string $action, Utopia\Route $route = null)
Console::error('[Error] Line: ' . $error->getLine());
};
function getStorageDevice($root): Device {
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
}
}
App::post('/v1/runtimes')
->desc("Create a new runtime server")
->param('runtimeId', '', new Text(64), 'Unique runtime ID.')
@ -148,9 +172,10 @@ App::post('/v1/runtimes')
/**
* Copy code files from source to a temporary location on the executor
*/
$device = new Local();
$buffer = $device->read($source);
if(!$device->write($tmpSource, $buffer)) {
$sourceDevice = getStorageDevice("/");
$localDevice = new Local();
$buffer = $sourceDevice->read($source);
if(!$localDevice->write($tmpSource, $buffer)) {
throw new Exception('Failed to copy source code to temporary directory', 500);
};
@ -238,11 +263,11 @@ App::post('/v1/runtimes')
throw new Exception('Something went wrong during the build process', 500);
}
$device = new Local($destination);
$outputPath = $device->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$destinationDevice = getStorageDevice($destination);
$outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$buffer = $device->read($tmpBuild);
if(!$device->write($outputPath, $buffer)) {
$buffer = $localDevice->read($tmpBuild);
if(!$destinationDevice->write($outputPath, $buffer, $localDevice->getFileMimeType($tmpBuild))) {
throw new Exception('Failed to move built code to storage', 500);
};

36
app/init.php
View file

@ -50,6 +50,7 @@ use Swoole\Database\PDOPool;
use Swoole\Database\RedisConfig;
use Swoole\Database\RedisPool;
use Utopia\Database\Query;
use Utopia\Storage\Device;
use Utopia\Storage\Storage;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
@ -806,46 +807,37 @@ App::setResource('deviceLocal', function() {
});
App::setResource('deviceFiles', function($project) {
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
}
return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
}, ['project']);
App::setResource('deviceFunctions', function($project) {
return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId());
}, ['project']);
App::setResource('deviceBuilds', function($project) {
return getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId());
}, ['project']);
function getDevice($root): Device {
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
return new Local(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId());
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId(), $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
}
}, ['project']);
}
App::setResource('mode', function($request) {
/** @var Appwrite\Utopia\Request $request */

1
app/views/install/compose.phtml
View file

@ -100,6 +100,7 @@ services:
- _APP_STORAGE_ANTIVIRUS
- _APP_STORAGE_ANTIVIRUS_HOST
- _APP_STORAGE_ANTIVIRUS_PORT
- _APP_FUNCTIONS_DEPLOYMENT_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS

4
app/workers/builds.php
View file

@ -39,7 +39,7 @@ class BuildsV1 extends Worker
{
$type = $this->args['type'] ?? '';
$projectId = $this->args['projectId'] ?? '';
$functionId = $this->args['functionId'] ?? '';
$functionId = $this->args['resourceId'] ?? '';
$deploymentId = $this->args['deploymentId'] ?? '';
switch ($type) {
@ -91,7 +91,7 @@ class BuildsV1 extends Worker
'outputPath' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path'),
'sourceType' => Storage::DEVICE_LOCAL,
'sourceType' => App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL),
'stdout' => '',
'stderr' => '',
'endTime' => 0,

23
app/workers/deletes.php
View file

@ -545,28 +545,7 @@ class DeletesV1 extends Worker
$bucketId = $document->getId();
$dbForProject = $this->getProjectDB($projectId);
$dbForProject->deleteCollection('bucket_' . $bucketId);
$device = new Local(APP_STORAGE_UPLOADS.'/app-'.$projectId);
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
$device = new S3(APP_STORAGE_UPLOADS . '/app-' . $projectId, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
break;
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
$device = new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
break;
}
$device = $this->getFilesDevice($projectId);
$device->deletePath($bucketId);
}
}

1
docker-compose.yml
View file

@ -127,6 +127,7 @@ services:
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_STORAGE_LIMIT
- _APP_FUNCTIONS_DEPLOYMENT_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS

628
src/Appwrite/Database/Database.php
View file

@ -1,628 +0,0 @@
<?php
namespace Appwrite\Database;
use Exception;
use Appwrite\Database\Validator\Authorization;
use Appwrite\Database\Validator\Structure;
use Appwrite\Database\Exception\Authorization as AuthorizationException;
use Appwrite\Database\Exception\Structure as StructureException;
class Database
{
// System Core
const SYSTEM_COLLECTION_COLLECTIONS = 0;
const SYSTEM_COLLECTION_RULES = 'rules';
// Project
const SYSTEM_COLLECTION_PROJECTS = 'projects';
const SYSTEM_COLLECTION_WEBHOOKS = 'webhooks';
const SYSTEM_COLLECTION_KEYS = 'keys';
const SYSTEM_COLLECTION_TASKS = 'tasks';
const SYSTEM_COLLECTION_PLATFORMS = 'platforms';
const SYSTEM_COLLECTION_USAGES = 'usages'; // TODO add structure
const SYSTEM_COLLECTION_DOMAINS = 'domains';
const SYSTEM_COLLECTION_CERTIFICATES = 'certificates';
const SYSTEM_COLLECTION_RESERVED = 'reserved';
// Auth, Account and Users (private to user)
const SYSTEM_COLLECTION_USERS = 'users';
const SYSTEM_COLLECTION_SESSIONS = 'sessions';
const SYSTEM_COLLECTION_TOKENS = 'tokens';
// Teams (shared among team members)
const SYSTEM_COLLECTION_MEMBERSHIPS = 'memberships';
const SYSTEM_COLLECTION_TEAMS = 'teams';
// Storage
const SYSTEM_COLLECTION_FILES = 'files';
// Functions
const SYSTEM_COLLECTION_FUNCTIONS = 'functions';
const SYSTEM_COLLECTION_DEPLOYMENTS = 'deployments';
const SYSTEM_COLLECTION_EXECUTIONS = 'executions';
// Realtime
const SYSTEM_COLLECTION_CONNECTIONS = 'connections';
// Var Types
const SYSTEM_VAR_TYPE_TEXT = 'text';
const SYSTEM_VAR_TYPE_NUMERIC = 'numeric';
const SYSTEM_VAR_TYPE_BOOLEAN = 'boolean';
const SYSTEM_VAR_TYPE_DOCUMENT = 'document';
const SYSTEM_VAR_TYPE_WILDCARD = 'wildcard';
const SYSTEM_VAR_TYPE_EMAIL = 'email';
const SYSTEM_VAR_TYPE_IP = 'ip';
const SYSTEM_VAR_TYPE_URL = 'url';
const SYSTEM_VAR_TYPE_KEY = 'key';
/**
* @var array
*/
protected static $filters = [];
/**
* @var bool
*/
protected static $statusFilters = true;
/**
* @var array
*/
protected $mocks = [];
/**
* @var Adapter
*/
protected $adapter;
/**
* Set Adapter.
*
* @param Adapter $adapter
*
* @return $this
*/
public function setAdapter(Adapter $adapter)
{
$this->adapter = $adapter;
return $this;
}
/**
* Set Namespace.
*
* Set namespace to divide different scope of data sets
*
* @param $namespace
*
* @return $this
*
* @throws Exception
*/
public function setNamespace($namespace)
{
$this->adapter->setNamespace($namespace);
return $this;
}
/**
* Get Namespace.
*
* Get namespace of current set scope
*
* @return string
*
* @throws Exception
*/
public function getNamespace()
{
return $this->adapter->getNamespace();
}
/**
* Create Namespace.
*
* @param string $namespace
*
* @return bool
*/
public function createNamespace($namespace)
{
return $this->adapter->createNamespace($namespace);
}
/**
* Delete Namespace.
*
* @param string $namespace
*
* @return bool
*/
public function deleteNamespace($namespace)
{
return $this->adapter->deleteNamespace($namespace);
}
/**
* @param array $options
* @param array $filterTypes
*
* @return Document[]
*/
public function getCollection(array $options, array $filterTypes = [])
{
$options = \array_merge([
'offset' => 0,
'limit' => 15,
'search' => '',
'relations' => true,
'orderField' => '',
'orderType' => 'ASC',
'orderCast' => 'int',
'filters' => [],
], $options);
$results = $this->adapter->getCollection($options, $filterTypes);
foreach ($results as &$node) {
$node = $this->decode(new Document($node));
}
return $results;
}
/**
* @param array $options
*
* @return Document
*/
public function getCollectionFirst(array $options)
{
$results = $this->getCollection($options);
return \reset($results);
}
/**
* @param array $options
*
* @return Document
*/
public function getCollectionLast(array $options)
{
$results = $this->getCollection($options);
return \end($results);
}
/**
* @param string $id
* @param bool $mock is mocked data allowed?
* @param bool $decode enable decoding?
*
* @return Document
*/
public function getDocument($id, bool $mock = true, bool $decode = true)
{
if (\is_null($id)) {
return new Document();
}
$document = new Document((isset($this->mocks[$id]) && $mock) ? $this->mocks[$id] : $this->adapter->getDocument($id));
$validator = new Authorization($document, 'read');
if (!$validator->isValid($document->getPermissions())) { // Check if user has read access to this document
return new Document();
}
$document = ($decode) ? $this->decode($document) : $document;
return $document;
}
/**
* @param array $data
*
* @return Document
*
* @throws AuthorizationException
* @throws StructureException
*/
public function createDocument(array $data, array $unique = [])
{
$document = new Document($data);
$validator = new Authorization($document, 'write');
if (!$validator->isValid($document->getPermissions())) { // Check if user has write access to this document
throw new AuthorizationException($validator->getDescription());
}
$validator = new Structure($this);
$document = $this->encode($document);
if (!$validator->isValid($document)) {
throw new StructureException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$document = new Document($this->adapter->createDocument($document->getArrayCopy(), $unique));
$document = $this->decode($document);
return $document;
}
/**
* @param array $data
*
* @return Document|false
*
* @throws Exception
*/
public function updateDocument(array $data)
{
if (!isset($data['$id'])) {
throw new Exception('Must define $id attribute');
}
$document = $this->getDocument($data['$id']); // TODO make sure user don\'t need read permission for write operations
// Make sure reserved keys stay constant
$data['$id'] = $document->getId();
$data['$collection'] = $document->getCollection();
$validator = new Authorization($document, 'write');
if (!$validator->isValid($document->getPermissions())) { // Check if user has write access to this document
throw new AuthorizationException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$new = new Document($data);
if (!$validator->isValid($new->getPermissions())) { // Check if user has write access to this document
throw new AuthorizationException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$new = $this->encode($new);
$validator = new Structure($this);
if (!$validator->isValid($new)) { // Make sure updated structure still apply collection rules (if any)
throw new StructureException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$new = new Document($this->adapter->updateDocument($new->getArrayCopy()));
$new = $this->decode($new);
return $new;
}
/**
* @param array $data
*
* @return Document|false
*
* @throws Exception
*/
public function overwriteDocument(array $data)
{
if (!isset($data['$id'])) {
throw new Exception('Must define $id attribute');
}
$document = $this->getDocument($data['$id']); // TODO make sure user don\'t need read permission for write operations
$validator = new Authorization($document, 'write');
if (!$validator->isValid($document->getPermissions())) { // Check if user has write access to this document
throw new AuthorizationException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$new = new Document($data);
if (!$validator->isValid($new->getPermissions())) { // Check if user has write access to this document
throw new AuthorizationException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$new = $this->encode($new);
$validator = new Structure($this);
if (!$validator->isValid($new)) { // Make sure updated structure still apply collection rules (if any)
throw new StructureException($validator->getDescription()); // var_dump($validator->getDescription()); return false;
}
$new = new Document($this->adapter->updateDocument($new->getArrayCopy()));
$new = $this->decode($new);
return $new;
}
/**
* @param string $id
*
* @return Document|false
*
* @throws AuthorizationException
*/
public function deleteDocument(string $id)
{
$document = $this->getDocument($id);
$validator = new Authorization($document, 'write');
if (!$validator->isValid($document->getPermissions())) { // Check if user has write access to this document
throw new AuthorizationException($validator->getDescription());
}
return new Document($this->adapter->deleteDocument($id));
}
/**
* @param int $key
*
* @return Document|false
*
* @throws AuthorizationException
*/
public function deleteUniqueKey($key)
{
return new Document($this->adapter->deleteUniqueKey($key));
}
/**
* @param int $key
*
* @return Document|false
*
* @throws AuthorizationException
*/
public function addUniqueKey($key)
{
return new Document($this->adapter->addUniqueKey($key));
}
/**
* @return array
*/
public function getDebug()
{
return $this->adapter->getDebug();
}
/**
* @return int
*/
public function getSum()
{
$debug = $this->getDebug();
return (isset($debug['sum'])) ? $debug['sum'] : 0;
}
/**
* @param array $options
*
* @return int
*/
public function getCount(array $options)
{
$options = \array_merge([
'filters' => [],
], $options);
$results = $this->adapter->getCount($options);
return $results;
}
/**
* @param string $key
* @param string $value
*
* @return self
*/
public function setMock($key, $value): self
{
$this->mocks[$key] = $value;
return $this;
}
/**
* @param array $mocks
*
* @return self
*/
public function setMocks(array $mocks): self
{
$this->mocks = $mocks;
return $this;
}
/**
* @return array
*/
public function getMocks()
{
return $this->mocks;
}
/**
* Add Attribute Filter
*
* @param string $name
* @param callable $encode
* @param callable $decode
*
* @return void
*/
public static function addFilter(string $name, callable $encode, callable $decode): void
{
self::$filters[$name] = [
'encode' => $encode,
'decode' => $decode,
];
}
/**
* Disable Attribute decoding
*
* @return void
*/
public static function disableFilters(): void
{
self::$statusFilters = false;
}
/**
* Enable Attribute decoding
*
* @return void
*/
public static function enableFilters(): void
{
self::$statusFilters = true;
}
public function encode(Document $document):Document
{
if (!self::$statusFilters) {
return $document;
}
$collection = $this->getDocument($document->getCollection(), true, false);
$rules = $collection->getAttribute('rules', []);
foreach ($rules as $key => $rule) {
$key = $rule->getAttribute('key', null);
$type = $rule->getAttribute('type', null);
$array = $rule->getAttribute('array', false);
$filters = $rule->getAttribute('filter', []);
$value = $document->getAttribute($key, null);
if (($value !== null)) {
if ($type === self::SYSTEM_VAR_TYPE_DOCUMENT) {
if ($array) {
$list = [];
foreach ($value as $child) {
$list[] = $this->encode($child);
}
$document->setAttribute($key, $list);
} else {
$document->setAttribute($key, $this->encode($value));
}
} else {
foreach ($filters as $filter) {
$value = $this->encodeAttribute($filter, $value);
$document->setAttribute($key, $value);
}
}
}
}
return $document;
}
public function decode(Document $document):Document
{
if (!self::$statusFilters) {
return $document;
}
$collection = $this->getDocument($document->getCollection(), true, false);
$rules = $collection->getAttribute('rules', []);
foreach ($rules as $key => $rule) {
$key = $rule->getAttribute('key', null);
$type = $rule->getAttribute('type', null);
$array = $rule->getAttribute('array', false);
$filters = $rule->getAttribute('filter', []);
$value = $document->getAttribute($key, null);
if (($value !== null)) {
if ($type === self::SYSTEM_VAR_TYPE_DOCUMENT) {
if ($array) {
$list = [];
foreach ($value as $child) {
$list[] = $this->decode($child);
}
$document->setAttribute($key, $list);
} else {
$document->setAttribute($key, $this->decode($value));
}
} else {
foreach (array_reverse($filters) as $filter) {
$value = $this->decodeAttribute($filter, $value);
$document->setAttribute($key, $value);
}
}
}
}
return $document;
}
/**
* Encode Attribute
*
* @param string $name
* @param mixed $value
*/
protected static function encodeAttribute(string $name, $value)
{
if (!isset(self::$filters[$name])) {
return $value;
throw new Exception("Filter '{$name}' not found");
}
try {
$value = self::$filters[$name]['encode']($value);
} catch (\Throwable $th) {
$value = null;
}
return $value;
}
/**
* Decode Attribute
*
* @param string $name
* @param mixed $value
*/
protected static function decodeAttribute(string $name, $value)
{
if (!isset(self::$filters[$name])) {
return $value;
throw new Exception("Filter '{$name}' not found");
}
try {
$value = self::$filters[$name]['decode']($value);
} catch (\Throwable $th) {
$value = null;
}
return $value;
}
/**
* Get Last Modified.
*
* Return Unix timestamp of last time a node queried in current session has been changed
*
* @return int
*/
public function lastModified()
{
return $this->adapter->lastModified();
}
}

61
src/Appwrite/Resque/Worker.php
View file

@ -8,8 +8,14 @@ use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Storage\Device;
use Utopia\Storage\Storage;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\S3;
use Exception;
abstract class Worker
{
/**
@ -219,4 +225,59 @@ abstract class Worker
return $database;
}
/**
* Get Functions Storage Device
* @param string $projectId of the project
* @return Device
*/
protected function getFunctionsDevice($projectId): Device {
return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
}
/**
* Get Files Storage Device
* @param string $projectId of the project
* @return Device
*/
protected function getFilesDevice($projectId): Device {
return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
}
/**
* Get Builds Storage Device
* @param string $projectId of the project
* @return Device
*/
protected function getBuildsDevice($projectId): Device {
return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
}
/**
* Get Device based on selected storage environment
* @param string $root path of the device
* @return Device
*/
private function getDevice($root): Device
{
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_LOCAL:default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
}
}
}

110
src/Appwrite/Utopia/Response/Filters/V07.php
View file

@ -1,110 +0,0 @@
<?php
namespace Appwrite\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filter;
use Exception;
class V07 extends Filter
{
// Convert 0.8 Data format to 0.7 format
public function parse(array $content, string $model): array
{
$parsedResponse = [];
switch ($model) {
case Response::MODEL_DOCUMENT_LIST: /** ANY was replaced by DOCUMENT in 0.8.x but this is backward compatible with 0.7.x */
// no break
case Response::MODEL_DOCUMENT: /** ANY was replaced by DOCUMENT in 0.8.x but this is backward compatible with 0.7.x */
// no break
case Response::MODEL_USER_LIST: /** [FIELDS ADDED in 0.8.x] passwordUpdate */
// no break
case Response::MODEL_USER: /** [FIELDS ADDED in 0.8.x] passwordUpdate */
// no break
case Response::MODEL_COLLECTION_LIST:
case Response::MODEL_COLLECTION:
case Response::MODEL_FILE_LIST:
case Response::MODEL_FILE:
case Response::MODEL_DEPLOYMENT_LIST:
case Response::MODEL_DEPLOYMENT:
case Response::MODEL_EXECUTION_LIST:
case Response::MODEL_EXECUTION:
case Response::MODEL_TEAM_LIST:
case Response::MODEL_TEAM:
case Response::MODEL_MEMBERSHIP_LIST:
case Response::MODEL_MEMBERSHIP:
case Response::MODEL_SESSION_LIST: /** [FIELDS ADDED in 0.8.x] provider, providerUid, providerToken */
// no break
case Response::MODEL_SESSION: /** [FIELDS ADDED in 0.8.x] provider, providerUid, providerToken */
// no break
case Response::MODEL_JWT:
case Response::MODEL_LOG:
case Response::MODEL_LOG_LIST:
case Response::MODEL_TOKEN:
case Response::MODEL_LOCALE:
case Response::MODEL_COUNTRY:
case Response::MODEL_COUNTRY_LIST:
case Response::MODEL_PHONE:
case Response::MODEL_PHONE_LIST:
case Response::MODEL_CONTINENT:
case Response::MODEL_CONTINENT_LIST:
case Response::MODEL_CURRENCY:
case Response::MODEL_CURRENCY_LIST:
case Response::MODEL_LANGUAGE:
case Response::MODEL_LANGUAGE_LIST:
case Response::MODEL_PROJECT:
case Response::MODEL_PROJECT_LIST:
case Response::MODEL_PLATFORM:
case Response::MODEL_PLATFORM_LIST:
case Response::MODEL_DOMAIN:
case Response::MODEL_DOMAIN_LIST:
case Response::MODEL_KEY:
case Response::MODEL_KEY_LIST:
case Response::MODEL_PERMISSIONS:
case Response::MODEL_RULE:
case Response::MODEL_TASK:
case Response::MODEL_WEBHOOK:
case Response::MODEL_WEBHOOK_LIST:
case Response::MODEL_MOCK:
case Response::MODEL_ANY:
case Response::MODEL_PREFERENCES: /** ANY was replaced by PREFERENCES in 0.8.x but this is backward compatible with 0.7.x */
// no break
case Response::MODEL_NONE:
case Response::MODEL_ERROR:
case Response::MODEL_ERROR_DEV:
$parsedResponse = $content;
break;
case Response::MODEL_FUNCTION_LIST: /** Function property env was renamed to runtime in 0.9.x */
$parsedResponse = $this->parseFunctionList($content);
break;
case Response::MODEL_FUNCTION:
$parsedResponse = $this->parseFunctionList($content); /** Function property env was renamed to runtime in 0.9.x */
break;
default:
throw new Exception('Received invalid model : '.$model);
}
return $parsedResponse;
}
protected function parseFunction(array $content)
{
$content['env'] = $content['runtime'];
unset($content['runtime']);
return $content;
}
protected function parseFunctionList(array $content)
{
$functions = $content['functions'];
$parsedResponse = [];
foreach ($functions as $function) {
$parsedResponse[] = $this->parseFunction($function);
}
$content['functions'] = $parsedResponse;
return $content;
}
}

103
src/Appwrite/Utopia/Response/Filters/V08.php
View file

@ -1,103 +0,0 @@
<?php
namespace Appwrite\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filter;
use Exception;
class V08 extends Filter
{
// Convert 0.9 Data format to 0.8 format
public function parse(array $content, string $model): array
{
$parsedResponse = [];
switch ($model) {
case Response::MODEL_DOCUMENT_LIST:
case Response::MODEL_DOCUMENT:
case Response::MODEL_USER_LIST:
case Response::MODEL_USER:
case Response::MODEL_COLLECTION_LIST:
case Response::MODEL_COLLECTION:
case Response::MODEL_FILE_LIST:
case Response::MODEL_FILE:
case Response::MODEL_DEPLOYMENT_LIST:
case Response::MODEL_DEPLOYMENT:
case Response::MODEL_EXECUTION_LIST:
case Response::MODEL_EXECUTION:
case Response::MODEL_TEAM_LIST:
case Response::MODEL_TEAM:
case Response::MODEL_MEMBERSHIP_LIST:
case Response::MODEL_MEMBERSHIP:
case Response::MODEL_SESSION_LIST:
case Response::MODEL_SESSION:
case Response::MODEL_JWT:
case Response::MODEL_LOG:
case Response::MODEL_LOG_LIST:
case Response::MODEL_TOKEN:
case Response::MODEL_LOCALE:
case Response::MODEL_COUNTRY:
case Response::MODEL_COUNTRY_LIST:
case Response::MODEL_PHONE:
case Response::MODEL_PHONE_LIST:
case Response::MODEL_CONTINENT:
case Response::MODEL_CONTINENT_LIST:
case Response::MODEL_CURRENCY:
case Response::MODEL_CURRENCY_LIST:
case Response::MODEL_LANGUAGE:
case Response::MODEL_LANGUAGE_LIST:
case Response::MODEL_PROJECT:
case Response::MODEL_PROJECT_LIST:
case Response::MODEL_PLATFORM:
case Response::MODEL_PLATFORM_LIST:
case Response::MODEL_DOMAIN:
case Response::MODEL_DOMAIN_LIST:
case Response::MODEL_KEY:
case Response::MODEL_KEY_LIST:
case Response::MODEL_PERMISSIONS:
case Response::MODEL_RULE:
case Response::MODEL_TASK:
case Response::MODEL_WEBHOOK:
case Response::MODEL_WEBHOOK_LIST:
case Response::MODEL_MOCK:
case Response::MODEL_ANY:
case Response::MODEL_PREFERENCES:
case Response::MODEL_NONE:
case Response::MODEL_ERROR:
case Response::MODEL_ERROR_DEV:
$parsedResponse = $content;
break;
case Response::MODEL_FUNCTION_LIST: /** Function property env was renamed to runtime in 0.9.x */
$parsedResponse = $this->parseFunctionList($content);
break;
case Response::MODEL_FUNCTION: /** Function property env was renamed to runtime in 0.9.x */
$parsedResponse = $this->parseFunctionList($content);
break;
default:
throw new Exception('Received invalid model : '.$model);
}
return $parsedResponse;
}
protected function parseFunction(array $content)
{
$content['env'] = $content['runtime'];
unset($content['runtime']);
return $content;
}
protected function parseFunctionList(array $content)
{
$functions = $content['functions'];
$parsedResponse = [];
foreach ($functions as $function) {
$parsedResponse[] = $this->parseFunction($function);
}
$content['functions'] = $parsedResponse;
return $content;
}
}