1
0
Fork 0
mirror of synced 2024-07-01 20:50:49 +12:00
appwrite/app/controllers/api/storage.php

1653 lines
77 KiB
PHP
Raw Normal View History

2019-05-09 18:54:39 +12:00
<?php
use Appwrite\Auth\Auth;
2022-01-19 00:05:04 +13:00
use Appwrite\ClamAV\Network;
use Appwrite\Event\Delete;
2022-05-05 00:23:34 +12:00
use Appwrite\Event\Event;
2022-01-19 00:05:04 +13:00
use Appwrite\Utopia\Database\Validator\CustomId;
use Appwrite\OpenSSL\OpenSSL;
2022-01-19 00:05:04 +13:00
use Appwrite\Utopia\Response;
2020-06-29 05:31:21 +12:00
use Utopia\App;
2022-01-19 00:05:04 +13:00
use Utopia\Config\Config;
use Utopia\Database\Database;
2021-05-03 20:28:31 +12:00
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Database\Exception\Structure as StructureException;
2022-08-14 22:33:36 +12:00
use Utopia\Database\ID;
2022-08-16 00:56:19 +12:00
use Utopia\Database\Permission;
2022-01-19 00:05:04 +13:00
use Utopia\Database\Query;
2022-08-16 23:26:38 +12:00
use Utopia\Database\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Permissions;
2021-07-26 02:47:18 +12:00
use Utopia\Database\Validator\UID;
use Appwrite\Extend\Exception;
2022-08-23 20:37:55 +12:00
use Appwrite\Utopia\Database\Validator\Queries\Buckets;
2022-08-23 20:49:39 +12:00
use Appwrite\Utopia\Database\Validator\Queries\Files;
2022-01-19 00:05:04 +13:00
use Utopia\Image\Image;
use Utopia\Storage\Compression\Algorithms\GZIP;
2022-08-31 01:46:55 +12:00
use Utopia\Storage\Compression\Algorithms\Zstd;
2022-05-05 00:23:34 +12:00
use Utopia\Storage\Device;
2021-01-22 21:28:33 +13:00
use Utopia\Storage\Storage;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
2021-01-22 21:28:33 +13:00
use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
2022-01-19 00:05:04 +13:00
use Utopia\Validator\ArrayList;
2021-07-07 22:07:11 +12:00
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
2022-01-19 00:05:04 +13:00
use Utopia\Validator\Range;
use Utopia\Validator\Text;
2021-07-07 22:07:11 +12:00
use Utopia\Validator\WhiteList;
2022-05-24 02:54:50 +12:00
use Utopia\Swoole\Request;
2019-05-09 18:54:39 +12:00
2021-06-14 23:32:09 +12:00
App::post('/v1/storage/buckets')
2022-01-31 19:22:28 +13:00
->desc('Create bucket')
2021-06-14 23:32:09 +12:00
->groups(['api', 'storage'])
->label('scope', 'buckets.write')
->label('event', 'buckets.[bucketId].create')
2022-09-05 20:00:08 +12:00
->label('audits.event', 'bucket.create')
2022-09-09 00:16:54 +12:00
->label('audits.resource', 'bucket/{response.$id}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'buckets.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
2021-06-14 23:32:09 +12:00
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createBucket')
->label('sdk.description', '/docs/references/storage/create-bucket.md')
->label('sdk.response.code', Response::STATUS_CODE_CREATED)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_BUCKET)
2022-10-04 09:22:28 +13:00
->param('bucketId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('name', '', new Text(128), 'Bucket name')
2022-08-27 15:19:00 +12:00
->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default no user is granted with any permissions. [Learn more about permissions](/docs/permissions).', true)
->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](/docs/permissions).', true)
2022-02-23 18:50:55 +13:00
->param('enabled', true, new Boolean(true), 'Is bucket enabled?', true)
2022-05-10 22:28:16 +12:00
->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '. For self-hosted setups you can change the max limit by changing the `_APP_STORAGE_LIMIT` environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true)
2022-05-01 19:54:58 +12:00
->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
->param('compression', 'none', new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
2022-02-23 18:50:55 +13:00
->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
2021-06-14 23:32:09 +12:00
->inject('response')
->inject('dbForProject')
->inject('events')
->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $events) {
2021-06-14 23:59:47 +12:00
2022-08-15 02:22:38 +12:00
$bucketId = $bucketId === 'unique()' ? ID::unique() : $bucketId;
2022-08-23 13:42:25 +12:00
// Map aggregate permissions into the multiple permissions they represent.
$permissions = Permission::aggregate($permissions);
2022-08-16 20:33:06 +12:00
2021-09-02 22:49:07 +12:00
try {
$files = Config::getParam('collections', [])['files'] ?? [];
if (empty($files)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.');
2021-12-13 22:17:53 +13:00
}
$attributes = [];
$indexes = [];
foreach ($files['attributes'] as $attribute) {
$attributes[] = new Document([
2022-08-15 23:24:31 +12:00
'$id' => $attribute['$id'],
2021-12-13 22:17:53 +13:00
'type' => $attribute['type'],
'size' => $attribute['size'],
'required' => $attribute['required'],
'signed' => $attribute['signed'],
'array' => $attribute['array'],
'filters' => $attribute['filters'],
2022-03-27 21:01:50 +13:00
'default' => $attribute['default'] ?? null,
'format' => $attribute['format'] ?? ''
2021-12-13 22:17:53 +13:00
]);
}
foreach ($files['indexes'] as $index) {
$indexes[] = new Document([
2022-08-15 23:24:31 +12:00
'$id' => $index['$id'],
2021-12-13 22:17:53 +13:00
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]);
}
2022-08-08 22:58:36 +12:00
$dbForProject->createDocument('buckets', new Document([
2022-08-15 02:22:38 +12:00
'$id' => $bucketId,
2022-08-15 23:24:31 +12:00
'$collection' => 'buckets',
'$permissions' => $permissions,
2021-11-11 19:20:52 +13:00
'name' => $name,
2021-09-02 22:49:07 +12:00
'maximumFileSize' => $maximumFileSize,
'allowedFileExtensions' => $allowedFileExtensions,
2022-08-27 15:19:51 +12:00
'fileSecurity' => $fileSecurity,
'enabled' => $enabled,
'compression' => $compression,
2022-08-27 15:19:51 +12:00
'encryption' => $encryption,
'antivirus' => $antivirus,
2021-10-17 19:06:13 +13:00
'search' => implode(' ', [$bucketId, $name]),
2021-09-02 22:49:07 +12:00
]));
2022-02-18 15:14:57 +13:00
$bucket = $dbForProject->getDocument('buckets', $bucketId);
$dbForProject->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
2022-08-08 22:58:36 +12:00
} catch (Duplicate) {
throw new Exception(Exception::STORAGE_BUCKET_ALREADY_EXISTS);
2021-09-02 22:49:07 +12:00
}
2021-06-14 23:59:47 +12:00
$events
->setParam('bucketId', $bucket->getId())
2021-06-14 23:59:47 +12:00
;
2022-09-07 23:11:10 +12:00
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($bucket, Response::MODEL_BUCKET);
2021-06-14 23:32:09 +12:00
});
2021-06-15 19:23:22 +12:00
App::get('/v1/storage/buckets')
->desc('List buckets')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'buckets.{scope}.requests.read')
2021-06-16 17:43:59 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
2021-06-15 19:23:22 +12:00
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listBuckets')
->label('sdk.description', '/docs/references/storage/list-buckets.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_BUCKET_LIST)
2022-08-23 20:37:55 +12:00
->param('queries', [], new Buckets(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Buckets::ALLOWED_ATTRIBUTES), true)
2021-06-15 19:23:22 +12:00
->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true)
->inject('response')
->inject('dbForProject')
2022-08-25 21:59:28 +12:00
->action(function (array $queries, string $search, Response $response, Database $dbForProject) {
2021-06-15 19:23:22 +12:00
2022-08-23 20:37:55 +12:00
$queries = Query::parseQueries($queries);
2022-08-12 11:53:52 +12:00
if (!empty($search)) {
2022-08-23 20:37:55 +12:00
$queries[] = Query::search('search', $search);
2022-08-12 11:53:52 +12:00
}
2022-08-23 20:37:55 +12:00
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE);
$cursor = reset($cursor);
2022-08-30 23:55:23 +12:00
if ($cursor) {
2022-08-23 20:37:55 +12:00
/** @var Query $cursor */
$bucketId = $cursor->getValue();
$cursorDocument = $dbForProject->getDocument('buckets', $bucketId);
2021-09-02 20:30:10 +12:00
2022-08-12 11:53:52 +12:00
if ($cursorDocument->isEmpty()) {
2022-08-23 20:37:55 +12:00
throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Bucket '{$bucketId}' for the 'cursor' value not found.");
2021-09-02 20:30:10 +12:00
}
2022-08-12 11:53:52 +12:00
2022-08-23 20:37:55 +12:00
$cursor->setValue($cursorDocument);
2021-09-02 20:30:10 +12:00
}
2021-06-15 19:23:22 +12:00
2022-08-23 20:37:55 +12:00
$filterQueries = Query::groupByType($queries)['filters'];
2021-07-27 22:19:39 +12:00
$response->dynamic(new Document([
2022-08-23 20:37:55 +12:00
'buckets' => $dbForProject->find('buckets', $queries),
2022-08-12 11:53:52 +12:00
'total' => $dbForProject->count('buckets', $filterQueries, APP_LIMIT_COUNT),
2021-06-15 19:23:22 +12:00
]), Response::MODEL_BUCKET_LIST);
});
2021-06-15 19:48:59 +12:00
App::get('/v1/storage/buckets/:bucketId')
->desc('Get Bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'buckets.{scope}.requests.read')
2021-06-16 17:43:59 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
2021-06-15 19:48:59 +12:00
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucket')
->label('sdk.description', '/docs/references/storage/get-bucket.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_BUCKET)
->param('bucketId', '', new UID(), 'Bucket unique ID.')
->inject('response')
->inject('dbForProject')
2022-08-11 14:18:22 +12:00
->action(function (string $bucketId, Response $response, Database $dbForProject) {
2021-06-15 19:48:59 +12:00
$bucket = $dbForProject->getDocument('buckets', $bucketId);
2021-06-15 19:48:59 +12:00
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2021-06-15 19:48:59 +12:00
}
2021-07-27 22:19:39 +12:00
$response->dynamic($bucket, Response::MODEL_BUCKET);
2021-06-15 19:48:59 +12:00
});
2021-06-16 21:59:47 +12:00
App::put('/v1/storage/buckets/:bucketId')
2021-06-16 22:03:44 +12:00
->desc('Update Bucket')
2021-06-16 21:59:47 +12:00
->groups(['api', 'storage'])
2021-06-16 22:03:44 +12:00
->label('scope', 'buckets.write')
->label('event', 'buckets.[bucketId].update')
2022-09-05 20:00:08 +12:00
->label('audits.event', 'bucket.update')
2022-09-09 00:16:54 +12:00
->label('audits.resource', 'bucket/{response.$id}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'buckets.{scope}.requests.update')
2021-06-16 21:59:47 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
2021-06-16 22:03:44 +12:00
->label('sdk.method', 'updateBucket')
->label('sdk.description', '/docs/references/storage/update-bucket.md')
2021-06-16 21:59:47 +12:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_BUCKET)
->param('bucketId', '', new UID(), 'Bucket unique ID.')
->param('name', null, new Text(128), 'Bucket name', false)
2022-08-27 15:19:00 +12:00
->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default the current permissions are inherited. [Learn more about permissions](/docs/permissions).', true)
2022-09-03 12:44:33 +12:00
->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](/docs/permissions).', true)
2022-02-23 18:50:55 +13:00
->param('enabled', true, new Boolean(true), 'Is bucket enabled?', true)
2022-05-10 22:28:16 +12:00
->param('maximumFileSize', null, new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human((int)App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '. For self hosted version you can change the limit by changing _APP_STORAGE_LIMIT environment variable. [Learn more about storage environment variables](docs/environment-variables#storage)', true)
2022-05-01 19:54:58 +12:00
->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
->param('compression', 'none', new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
2022-02-23 18:50:55 +13:00
->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
2021-06-16 21:59:47 +12:00
->inject('response')
->inject('dbForProject')
->inject('events')
2022-08-31 01:50:38 +12:00
->action(function (string $bucketId, string $name, ?array $permissions, bool $fileSecurity, bool $enabled, ?int $maximumFileSize, array $allowedFileExtensions, string $compression, bool $encryption, bool $antivirus, Response $response, Database $dbForProject, Event $events) {
$bucket = $dbForProject->getDocument('buckets', $bucketId);
2021-06-16 21:59:47 +12:00
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2021-06-16 21:59:47 +12:00
}
$permissions ??= $bucket->getPermissions();
$maximumFileSize ??= $bucket->getAttribute('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0));
$allowedFileExtensions ??= $bucket->getAttribute('allowedFileExtensions', []);
$enabled ??= $bucket->getAttribute('enabled', true);
$encryption ??= $bucket->getAttribute('encryption', true);
$antivirus ??= $bucket->getAttribute('antivirus', true);
2021-06-16 21:59:47 +12:00
/**
* Map aggregate permissions into the multiple permissions they represent,
* accounting for the resource type given that some types not allowed specific permissions.
*/
2022-08-23 13:42:25 +12:00
// Map aggregate permissions into the multiple permissions they represent.
$permissions = Permission::aggregate($permissions);
2022-08-16 20:33:06 +12:00
$bucket = $dbForProject->updateDocument('buckets', $bucket->getId(), $bucket
->setAttribute('name', $name)
->setAttribute('$permissions', $permissions)
->setAttribute('maximumFileSize', $maximumFileSize)
->setAttribute('allowedFileExtensions', $allowedFileExtensions)
2022-08-27 15:19:51 +12:00
->setAttribute('fileSecurity', $fileSecurity)
->setAttribute('enabled', $enabled)
->setAttribute('encryption', $encryption)
2022-08-31 01:50:38 +12:00
->setAttribute('compression', $compression)
2022-08-27 15:19:51 +12:00
->setAttribute('antivirus', $antivirus));
2021-06-16 21:59:47 +12:00
$events
->setParam('bucketId', $bucket->getId())
2021-06-16 21:59:47 +12:00
;
2021-07-27 22:19:39 +12:00
$response->dynamic($bucket, Response::MODEL_BUCKET);
2021-06-16 21:59:47 +12:00
});
2021-06-16 22:17:14 +12:00
App::delete('/v1/storage/buckets/:bucketId')
->desc('Delete Bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.write')
2022-09-05 20:00:08 +12:00
->label('audits.event', 'bucket.delete')
->label('event', 'buckets.[bucketId].delete')
2022-09-09 00:16:54 +12:00
->label('audits.resource', 'bucket/{request.bucketId}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'buckets.{scope}.requests.delete')
2021-06-16 22:17:14 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'deleteBucket')
->label('sdk.description', '/docs/references/storage/delete-bucket.md')
->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
->label('sdk.response.model', Response::MODEL_NONE)
->param('bucketId', '', new UID(), 'Bucket unique ID.')
->inject('response')
->inject('dbForProject')
2021-06-16 22:17:14 +12:00
->inject('deletes')
2021-06-17 20:23:21 +12:00
->inject('events')
->action(function (string $bucketId, Response $response, Database $dbForProject, Delete $deletes, Event $events) {
$bucket = $dbForProject->getDocument('buckets', $bucketId);
2019-05-09 18:54:39 +12:00
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2021-06-16 22:17:14 +12:00
}
if (!$dbForProject->deleteDocument('buckets', $bucketId)) {
2022-08-14 18:56:12 +12:00
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove bucket from DB');
2021-09-15 16:54:49 +12:00
}
2021-06-16 22:17:14 +12:00
$deletes
2022-04-18 08:34:32 +12:00
->setType(DELETE_TYPE_DOCUMENT)
->setDocument($bucket);
2021-06-16 22:17:14 +12:00
2021-06-17 20:23:21 +12:00
$events
->setParam('bucketId', $bucket->getId())
->setPayload($response->output($bucket, Response::MODEL_BUCKET))
2021-06-17 20:23:21 +12:00
;
2021-06-16 22:17:14 +12:00
$response->noContent();
});
2021-06-17 22:10:58 +12:00
App::post('/v1/storage/buckets/:bucketId/files')
2021-07-07 22:07:11 +12:00
->alias('/v1/storage/files', ['bucketId' => 'default'])
2020-02-01 11:34:07 +13:00
->desc('Create File')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
2020-02-01 11:34:07 +13:00
->label('scope', 'files.write')
2022-09-05 20:00:08 +12:00
->label('audits.event', 'file.create')
->label('event', 'buckets.[bucketId].files.[fileId].create')
2022-09-09 00:16:54 +12:00
->label('audits.resource', 'file/{response.$id}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2020-02-01 11:34:07 +13:00
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createFile')
->label('sdk.description', '/docs/references/storage/create-file.md')
2020-11-12 10:02:24 +13:00
->label('sdk.request.type', 'multipart/form-data')
2020-04-11 06:59:14 +12:00
->label('sdk.methodType', 'upload')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_CREATED)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_FILE)
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
2022-10-04 09:22:28 +13:00
->param('fileId', '', new CustomId(), 'File ID. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
2020-09-11 02:40:14 +12:00
->param('file', [], new File(), 'Binary file.', false)
2022-09-06 21:17:25 +12:00
->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permission strings. By default the current user is granted with all permissions. [Learn more about permissions](/docs/permissions).', true)
2020-12-27 05:48:36 +13:00
->inject('request')
->inject('response')
->inject('dbForProject')
2021-03-22 11:17:20 +13:00
->inject('user')
2022-02-16 14:30:27 +13:00
->inject('events')
->inject('mode')
2021-12-13 21:08:27 +13:00
->inject('deviceFiles')
->inject('deviceLocal')
2022-08-16 00:16:32 +12:00
->inject('deletes')
->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal, Delete $deletes) {
2022-08-08 22:58:36 +12:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2020-06-30 09:43:34 +12:00
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2021-06-17 22:10:58 +12:00
}
2022-08-08 22:58:36 +12:00
2022-08-25 01:24:54 +12:00
$validator = new Authorization(Database::PERMISSION_CREATE);
if (!$validator->isValid($bucket->getCreate())) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
2022-08-23 13:42:25 +12:00
$allowedPermissions = [
Database::PERMISSION_READ,
Database::PERMISSION_UPDATE,
Database::PERMISSION_DELETE,
];
2022-08-23 13:42:25 +12:00
// Map aggregate permissions to into the set of individual permissions they represent.
$permissions = Permission::aggregate($permissions, $allowedPermissions);
// Add permissions for current the user if none were provided.
2022-08-16 00:56:19 +12:00
if (\is_null($permissions)) {
$permissions = [];
2022-08-16 01:16:20 +12:00
if (!empty($user->getId())) {
2022-08-16 00:56:19 +12:00
foreach ($allowedPermissions as $permission) {
2022-08-16 01:16:20 +12:00
$permissions[] = (new Permission($permission, 'user', $user->getId()))->toString();
2022-08-16 00:56:19 +12:00
}
}
}
2022-08-16 00:56:19 +12:00
// Users can only manage their own roles, API keys and Admin users can manage any
$roles = Authorization::getRoles();
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles)) {
foreach (Database::PERMISSIONS as $type) {
foreach ($permissions as $permission) {
2022-08-16 23:26:38 +12:00
$permission = Permission::parse($permission);
if ($permission->getPermission() != $type) {
2022-08-16 00:56:19 +12:00
continue;
}
2022-08-16 23:26:38 +12:00
$role = (new Role(
$permission->getRole(),
$permission->getIdentifier(),
$permission->getDimension()
))->toString();
2022-08-16 00:56:19 +12:00
if (!Authorization::isRole($role)) {
2022-08-25 15:51:21 +12:00
throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')');
2022-08-16 00:56:19 +12:00
}
}
}
}
2020-06-30 09:43:34 +12:00
$maximumFileSize = $bucket->getAttribute('maximumFileSize', 0);
2021-07-07 22:07:11 +12:00
if ($maximumFileSize > (int) App::getEnv('_APP_STORAGE_LIMIT', 0)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Maximum bucket file size is larger than _APP_STORAGE_LIMIT');
}
2021-12-14 00:46:59 +13:00
$file = $request->getFiles('file');
2020-06-30 09:43:34 +12:00
if (empty($file)) {
throw new Exception(Exception::STORAGE_FILE_EMPTY);
2020-06-30 09:43:34 +12:00
}
2020-02-01 11:34:07 +13:00
2020-06-30 09:43:34 +12:00
// Make sure we handle a single file and multiple files the same way
2021-07-07 22:07:11 +12:00
$fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name'];
$fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name'];
2021-11-30 21:04:19 +13:00
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
2021-07-07 22:07:11 +12:00
$contentRange = $request->getHeader('content-range');
2022-08-15 02:22:38 +12:00
$fileId = $fileId === 'unique()' ? ID::unique() : $fileId;
2021-07-07 22:07:11 +12:00
$chunk = 1;
$chunks = 1;
if (!empty($contentRange)) {
2021-07-15 21:40:41 +12:00
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
2021-11-30 21:04:19 +13:00
$fileSize = $request->getContentRangeSize();
2021-09-27 19:04:17 +13:00
$fileId = $request->getHeader('x-appwrite-id', $fileId);
if (is_null($start) || is_null($end) || is_null($fileSize)) {
throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE);
2021-07-07 22:07:11 +12:00
}
2020-02-01 11:34:07 +13:00
2021-12-07 21:15:42 +13:00
if ($end === $fileSize) {
2021-12-14 00:46:59 +13:00
//if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to -1 notify it's last chunk
2021-07-07 22:07:11 +12:00
$chunks = $chunk = -1;
} else {
2021-07-13 19:58:16 +12:00
// Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart)
2021-11-30 21:04:19 +13:00
$chunks = (int) ceil($fileSize / ($end + 1 - $start));
2021-11-25 17:47:40 +13:00
$chunk = (int) ($start / ($end + 1 - $start)) + 1;
2021-07-07 22:07:11 +12:00
}
}
2020-02-01 11:34:07 +13:00
/**
2022-08-15 16:29:44 +12:00
* Validators
*/
2021-12-14 00:46:59 +13:00
// Check if file type is allowed
$allowedFileExtensions = $bucket->getAttribute('allowedFileExtensions', []);
$fileExt = new FileExt($allowedFileExtensions);
2021-07-07 22:07:11 +12:00
if (!empty($allowedFileExtensions) && !$fileExt->isValid($fileName)) {
2022-08-14 19:02:41 +12:00
throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, 'File extension not allowed');
2020-06-30 09:43:34 +12:00
}
2020-02-01 11:34:07 +13:00
2021-12-14 00:46:59 +13:00
// Check if file size is exceeding allowed limit
$fileSizeValidator = new FileSize($maximumFileSize);
if (!$fileSizeValidator->isValid($fileSize)) {
2022-08-14 18:56:12 +12:00
throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE, 'File size not allowed');
2021-06-17 22:10:58 +12:00
}
2020-02-01 11:34:07 +13:00
2021-12-14 00:46:59 +13:00
$upload = new Upload();
2021-07-07 22:07:11 +12:00
if (!$upload->isValid($fileTmpName)) {
throw new Exception(Exception::STORAGE_INVALID_FILE);
2020-06-30 09:43:34 +12:00
}
2020-02-01 11:34:07 +13:00
2020-06-30 09:43:34 +12:00
// Save to storage
2022-05-24 02:54:50 +12:00
$fileSize ??= $deviceLocal->getFileSize($fileTmpName);
2021-12-13 21:08:27 +13:00
$path = $deviceFiles->getPath($fileId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = str_ireplace($deviceFiles->getRoot(), $deviceFiles->getRoot() . DIRECTORY_SEPARATOR . $bucket->getId(), $path); // Add bucket id to path after root
2021-10-08 21:39:37 +13:00
2022-08-15 22:11:17 +12:00
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
2020-02-01 11:34:07 +13:00
2021-12-13 21:08:27 +13:00
$metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)];
2021-07-07 22:07:11 +12:00
if (!$file->isEmpty()) {
2021-07-15 23:34:05 +12:00
$chunks = $file->getAttribute('chunksTotal', 1);
2021-11-25 17:47:40 +13:00
$metadata = $file->getAttribute('metadata', []);
2021-12-07 20:30:39 +13:00
if ($chunk === -1) {
2021-11-25 17:47:40 +13:00
$chunk = $chunks;
2021-06-17 22:10:58 +12:00
}
}
2020-02-01 11:34:07 +13:00
2021-12-13 21:08:27 +13:00
$chunksUploaded = $deviceFiles->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
2021-07-13 19:58:16 +12:00
if (empty($chunksUploaded)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed uploading file');
2021-06-17 22:10:58 +12:00
}
2020-02-01 11:34:07 +13:00
2021-12-07 20:30:39 +13:00
if ($chunksUploaded === $chunks) {
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) === Storage::DEVICE_LOCAL) {
2022-05-24 02:54:50 +12:00
$antivirus = new Network(
App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
);
if (!$antivirus->fileScan($path)) {
2021-12-13 21:08:27 +13:00
$deviceFiles->delete($path);
throw new Exception(Exception::STORAGE_INVALID_FILE);
2021-07-07 22:07:11 +12:00
}
2020-02-01 11:34:07 +13:00
}
2021-12-13 21:08:27 +13:00
$mimeType = $deviceFiles->getFileMimeType($path); // Get mime-type before compression and encryption
2021-07-13 20:16:24 +12:00
$data = '';
2021-07-07 22:07:11 +12:00
// Compression
2022-08-31 01:46:55 +12:00
$algorithm = $bucket->getAttribute('compression', 'none');
if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != 'none') {
2021-12-13 21:08:27 +13:00
$data = $deviceFiles->read($path);
2022-08-31 14:56:44 +12:00
switch ($algorithm) {
2022-08-31 01:46:55 +12:00
case 'zstd':
$compressor = new Zstd();
2022-08-31 14:59:51 +12:00
break;
2022-08-31 01:46:55 +12:00
case 'gzip':
default:
$compressor = new GZIP();
break;
}
2021-07-13 20:16:24 +12:00
$data = $compressor->compress($data);
2021-06-17 22:10:58 +12:00
}
2020-02-01 11:34:07 +13:00
2021-11-30 21:04:19 +13:00
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
if (empty($data)) {
2021-12-13 21:08:27 +13:00
$data = $deviceFiles->read($path);
2021-07-08 23:26:11 +12:00
}
2021-07-13 20:16:24 +12:00
$key = App::getEnv('_APP_OPENSSL_KEY_V1');
$iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM));
$data = OpenSSL::encrypt($data, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag);
}
2020-06-30 09:43:34 +12:00
2021-12-09 23:44:44 +13:00
if (!empty($data)) {
2021-12-13 21:08:27 +13:00
if (!$deviceFiles->write($path, $data, $mimeType)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to save file');
2021-07-08 23:26:11 +12:00
}
2021-07-07 22:07:11 +12:00
}
2021-06-17 22:10:58 +12:00
2021-12-13 21:08:27 +13:00
$sizeActual = $deviceFiles->getFileSize($path);
$fileHash = $deviceFiles->getFileHash($path);
2021-07-07 22:07:11 +12:00
2021-11-30 21:04:19 +13:00
if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
2021-07-07 22:07:11 +12:00
$openSSLVersion = '1';
$openSSLCipher = OpenSSL::CIPHER_AES_128_GCM;
$openSSLTag = \bin2hex($tag);
$openSSLIV = \bin2hex($iv);
2021-11-07 19:35:49 +13:00
}
try {
if ($file->isEmpty()) {
$doc = new Document([
2022-08-15 02:22:38 +12:00
'$id' => $fileId,
'$permissions' => $permissions,
2022-08-15 23:24:31 +12:00
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
'mimeType' => $mimeType,
2021-11-30 21:04:19 +13:00
'sizeOriginal' => $fileSize,
'sizeActual' => $sizeActual,
'algorithm' => $algorithm,
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'openSSLVersion' => $openSSLVersion,
'openSSLCipher' => $openSSLCipher,
'openSSLTag' => $openSSLTag,
'openSSLIV' => $openSSLIV,
'search' => implode(' ', [$fileId, $fileName]),
2021-11-25 17:47:40 +13:00
'metadata' => $metadata,
]);
2022-08-08 22:58:36 +12:00
2022-08-15 19:20:10 +12:00
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
} else {
$file = $file
->setAttribute('$permissions', $permissions)
2021-11-15 01:23:45 +13:00
->setAttribute('signature', $fileHash)
->setAttribute('mimeType', $mimeType)
->setAttribute('sizeActual', $sizeActual)
->setAttribute('algorithm', $algorithm)
->setAttribute('openSSLVersion', $openSSLVersion)
->setAttribute('openSSLCipher', $openSSLCipher)
->setAttribute('openSSLTag', $openSSLTag)
2021-11-25 17:47:40 +13:00
->setAttribute('openSSLIV', $openSSLIV)
->setAttribute('metadata', $metadata)
->setAttribute('chunksUploaded', $chunksUploaded);
2022-08-15 19:20:10 +12:00
$file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
}
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (StructureException $exception) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage());
2022-08-08 22:58:36 +12:00
} catch (DuplicateException) {
throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS);
2021-07-07 22:07:11 +12:00
}
} else {
try {
if ($file->isEmpty()) {
$doc = new Document([
2022-08-14 22:33:36 +12:00
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
'mimeType' => '',
2021-11-30 21:04:19 +13:00
'sizeOriginal' => $fileSize,
'sizeActual' => 0,
'algorithm' => '',
'comment' => '',
'chunksTotal' => $chunks,
'chunksUploaded' => $chunksUploaded,
'search' => implode(' ', [$fileId, $fileName]),
2021-11-25 17:47:40 +13:00
'metadata' => $metadata,
]);
2022-08-08 22:58:36 +12:00
2022-08-15 19:20:10 +12:00
$file = $dbForProject->createDocument('bucket_' . $bucket->getInternalId(), $doc);
} else {
$file = $file
2021-11-25 17:47:40 +13:00
->setAttribute('chunksUploaded', $chunksUploaded)
->setAttribute('metadata', $metadata);
2022-08-15 19:20:10 +12:00
$file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
}
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
} catch (StructureException $exception) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $exception->getMessage());
2022-08-08 22:58:36 +12:00
} catch (DuplicateException) {
throw new Exception(Exception::DOCUMENT_ALREADY_EXISTS);
2021-07-07 22:07:11 +12:00
}
2021-11-07 19:35:49 +13:00
}
2021-06-17 22:10:58 +12:00
$events
->setParam('bucketId', $bucket->getId())
->setParam('fileId', $file->getId())
Database layer (#3338) * database response model * database collection config * new database scopes * database service update * database execption codes * remove read write permission from database model * updating tests and fixing some bugs * server side tests are now passing * databases api * tests for database endpoint * composer update * fix error * formatting * formatting fixes * get database test * more updates to events and usage * more usage updates * fix delete type * fix test * delete database * more fixes * databaseId in attributes and indexes * more fixes * fix issues * fix index subquery * fix console scope and index query * updating tests as required * fix phpcs errors and warnings * updates to review suggestions * UI progress * ui updates and cleaning up * fix type * rework database events * update tests * update types * event generation fixed * events config updated * updating context to support multiple * realtime updates * fix ids * update context * validator updates * fix naming conflict * fix tests * fix lint errors * fix wprler and realtime tests * fix webhooks test * fix event validator and other tests * formatting fixes * removing leftover var_dumps * remove leftover comment * update usage params * usage metrics updates * update database usage * fix usage * specs update * updates to usage * fix UI and usage * fix lints * internal id fixes * fixes for internal Id * renaming services and related files * rename tests * rename doc link * rename readme * fix test name * tests: fixes for 0.15.x sync Co-authored-by: Torsten Dittmann <torsten.dittmann@googlemail.com>
2022-06-22 22:51:49 +12:00
->setContext('bucket', $bucket)
;
2022-02-16 14:30:27 +13:00
2022-08-16 00:16:32 +12:00
$deletes
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
->setResource('file/' . $file->getId())
;
2021-11-25 17:47:40 +13:00
$metadata = null; // was causing leaks as it was passed by reference
2020-06-30 09:43:34 +12:00
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($file, Response::MODEL_FILE);
2020-12-27 05:48:36 +13:00
});
2020-02-01 11:34:07 +13:00
2021-06-18 21:24:16 +12:00
App::get('/v1/storage/buckets/:bucketId/files')
->alias('/v1/storage/files', ['bucketId' => 'default'])
2019-05-09 18:54:39 +12:00
->desc('List Files')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
->label('scope', 'files.read')
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
2019-05-09 18:54:39 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'listFiles')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/list-files.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_FILE_LIST)
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
2022-08-23 20:49:39 +12:00
->param('queries', [], new Files(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Files::ALLOWED_ATTRIBUTES), true)
2020-09-11 02:40:14 +12:00
->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true)
2020-12-27 05:48:36 +13:00
->inject('response')
->inject('dbForProject')
2021-11-19 22:45:42 +13:00
->inject('mode')
2022-08-25 21:59:28 +12:00
->action(function (string $bucketId, array $queries, string $search, Response $response, Database $dbForProject, string $mode) {
2021-06-18 21:24:16 +12:00
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2021-06-20 22:55:24 +12:00
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2021-06-20 22:55:24 +12:00
}
2022-08-08 22:58:36 +12:00
2022-08-25 01:24:54 +12:00
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
$validator = new Authorization(Database::PERMISSION_READ);
$valid = $validator->isValid($bucket->getRead());
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2021-11-07 19:35:49 +13:00
}
2022-08-23 20:49:39 +12:00
$queries = Query::parseQueries($queries);
2021-06-18 21:24:16 +12:00
2022-08-12 11:53:52 +12:00
if (!empty($search)) {
2022-08-23 20:49:39 +12:00
$queries[] = Query::search('search', $search);
2021-06-18 21:24:16 +12:00
}
2021-05-03 20:28:31 +12:00
2022-08-23 20:49:39 +12:00
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE);
$cursor = reset($cursor);
2022-08-30 23:55:23 +12:00
if ($cursor) {
2022-08-23 20:49:39 +12:00
/** @var Query $cursor */
$fileId = $cursor->getValue();
if ($fileSecurity && !$valid) {
$cursorDocument = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
} else {
$cursorDocument = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
2021-08-07 00:36:27 +12:00
2022-08-12 11:53:52 +12:00
if ($cursorDocument->isEmpty()) {
2022-08-23 20:49:39 +12:00
throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "File '{$fileId}' for the 'cursor' value not found.");
2021-08-07 00:36:27 +12:00
}
2021-08-19 01:42:03 +12:00
2022-08-23 20:49:39 +12:00
$cursor->setValue($cursorDocument);
2021-08-19 01:42:03 +12:00
}
2022-08-08 22:58:36 +12:00
2022-08-23 20:49:39 +12:00
$filterQueries = Query::groupByType($queries)['filters'];
2022-08-25 01:24:54 +12:00
if ($fileSecurity && !$valid) {
2022-08-23 20:49:39 +12:00
$files = $dbForProject->find('bucket_' . $bucket->getInternalId(), $queries);
2022-08-12 11:53:52 +12:00
$total = $dbForProject->count('bucket_' . $bucket->getInternalId(), $filterQueries, APP_LIMIT_COUNT);
2022-08-08 22:58:36 +12:00
} else {
2022-08-23 20:49:39 +12:00
$files = Authorization::skip(fn () => $dbForProject->find('bucket_' . $bucket->getInternalId(), $queries));
$total = Authorization::skip(fn () => $dbForProject->count('bucket_' . $bucket->getInternalId(), $filterQueries, APP_LIMIT_COUNT));
2022-08-08 22:58:36 +12:00
}
2021-11-07 19:35:49 +13:00
2020-10-31 08:53:27 +13:00
$response->dynamic(new Document([
2021-11-07 19:35:49 +13:00
'files' => $files,
2022-08-12 11:53:52 +12:00
'total' => $total,
2020-10-31 08:53:27 +13:00
]), Response::MODEL_FILE_LIST);
2020-12-27 05:48:36 +13:00
});
2019-05-09 18:54:39 +12:00
2021-06-18 21:33:00 +12:00
App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->alias('/v1/storage/files/:fileId', ['bucketId' => 'default'])
2019-05-09 18:54:39 +12:00
->desc('Get File')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
->label('scope', 'files.read')
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
2019-05-09 18:54:39 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'getFile')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/get-file.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_FILE)
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
2020-12-27 05:48:36 +13:00
->inject('response')
->inject('dbForProject')
2021-11-19 22:45:42 +13:00
->inject('mode')
2022-08-11 14:18:22 +12:00
->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, string $mode) {
2019-05-09 18:54:39 +12:00
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2019-05-09 18:54:39 +12:00
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2021-06-20 22:55:24 +12:00
}
2022-08-08 22:58:36 +12:00
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
2022-08-25 01:24:54 +12:00
$validator = new Authorization(Database::PERMISSION_READ);
2022-08-08 22:58:36 +12:00
$valid = $validator->isValid($bucket->getRead());
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2021-11-07 19:35:49 +13:00
}
2022-08-08 22:58:36 +12:00
2022-08-25 01:24:54 +12:00
if ($fileSecurity && !$valid) {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
2022-08-25 01:24:54 +12:00
} else {
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
2021-06-18 21:33:00 +12:00
2022-08-26 15:01:16 +12:00
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2019-05-09 18:54:39 +12:00
}
2022-03-15 22:51:51 +13:00
2020-10-31 08:53:27 +13:00
$response->dynamic($file, Response::MODEL_FILE);
2020-12-27 05:48:36 +13:00
});
2019-05-09 18:54:39 +12:00
App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->alias('/v1/storage/files/:fileId/preview', ['bucketId' => 'default'])
2019-08-25 20:10:28 +12:00
->desc('Get File Preview')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
->label('scope', 'files.read')
2022-08-15 21:05:41 +12:00
->label('cache', true)
->label('cache.resource', 'file/{request.fileId}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2019-05-09 18:54:39 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'getFilePreview')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/get-file-preview.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
2020-11-12 11:02:02 +13:00
->label('sdk.response.type', Response::CONTENT_TYPE_IMAGE)
2022-02-27 20:39:33 +13:00
->label('sdk.methodType', 'location')
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID')
2020-09-11 02:40:14 +12:00
->param('width', 0, new Range(0, 4000), 'Resize preview image width, Pass an integer between 0 to 4000.', true)
->param('height', 0, new Range(0, 4000), 'Resize preview image height, Pass an integer between 0 to 4000.', true)
->param('gravity', Image::GRAVITY_CENTER, new WhiteList(Image::getGravityTypes()), 'Image crop gravity. Can be one of ' . implode(",", Image::getGravityTypes()), true)
2020-09-11 02:40:14 +12:00
->param('quality', 100, new Range(0, 100), 'Preview image quality. Pass an integer between 0 to 100. Defaults to 100.', true)
->param('borderWidth', 0, new Range(0, 100), 'Preview image border in pixels. Pass an integer between 0 to 100. Defaults to 0.', true)
2021-03-22 19:54:42 +13:00
->param('borderColor', '', new HexColor(), 'Preview image border color. Use a valid HEX color, no # is needed for prefix.', true)
->param('borderRadius', 0, new Range(0, 4000), 'Preview image border radius in pixels. Pass an integer between 0 to 4000.', true)
->param('opacity', 1, new Range(0, 1, Range::TYPE_FLOAT), 'Preview image opacity. Only works with images having an alpha channel (like png). Pass a number between 0 to 1.', true)
->param('rotation', 0, new Range(-360, 360), 'Preview image rotation in degrees. Pass an integer between -360 and 360.', true)
2020-09-11 02:40:14 +12:00
->param('background', '', new HexColor(), 'Preview image background color. Only works with transparent images (png). Use a valid HEX color, no # is needed for prefix.', true)
->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true)
2020-12-27 05:48:36 +13:00
->inject('request')
->inject('response')
->inject('project')
->inject('dbForProject')
2021-11-19 22:45:42 +13:00
->inject('mode')
2021-12-13 21:08:27 +13:00
->inject('deviceFiles')
->inject('deviceLocal')
2022-08-11 14:18:22 +12:00
->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceFiles, Device $deviceLocal) {
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
if (!\extension_loaded('imagick')) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing');
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2021-06-21 20:05:11 +12:00
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2022-08-08 22:58:36 +12:00
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
2022-08-25 01:24:54 +12:00
$validator = new Authorization(Database::PERMISSION_READ);
2022-08-08 22:58:36 +12:00
$valid = $validator->isValid($bucket->getRead());
2022-08-25 01:24:54 +12:00
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2021-11-07 19:35:49 +13:00
}
2021-11-07 18:50:21 +13:00
if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support
2020-06-30 23:09:28 +12:00
$output = 'jpg';
}
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
$inputs = Config::getParam('storage-inputs');
$outputs = Config::getParam('storage-outputs');
$fileLogos = Config::getParam('storage-logos');
2020-06-30 16:32:36 +12:00
$date = \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT'; // 45 days cache
$key = \md5($fileId . $width . $height . $gravity . $quality . $borderWidth . $borderColor . $borderRadius . $opacity . $rotation . $background . $output);
2022-08-08 22:58:36 +12:00
2022-08-25 01:24:54 +12:00
if ($fileSecurity && !$valid) {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
2022-08-25 01:24:54 +12:00
} else {
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
2019-05-09 18:54:39 +12:00
2022-08-26 15:01:16 +12:00
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
$path = $file->getAttribute('path');
$type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
2022-08-31 01:46:55 +12:00
$algorithm = $file->getAttribute('algorithm', 'none');
2021-05-03 20:28:31 +12:00
$cipher = $file->getAttribute('openSSLCipher');
2020-06-30 23:09:28 +12:00
$mime = $file->getAttribute('mimeType');
if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) App::getEnv('_APP_STORAGE_PREVIEW_LIMIT', 20000000)) {
2022-05-24 02:54:50 +12:00
if (!\in_array($mime, $inputs)) {
$path = (\array_key_exists($mime, $fileLogos)) ? $fileLogos[$mime] : $fileLogos['default'];
} else {
// it was an image but the file size exceeded the limit
$path = $fileLogos['default_image'];
}
2022-08-31 01:46:55 +12:00
$algorithm = 'none';
2020-06-30 23:09:28 +12:00
$cipher = null;
$background = (empty($background)) ? 'eceff1' : $background;
2020-06-20 23:20:49 +12:00
$type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
$deviceFiles = $deviceLocal;
2020-06-30 23:09:28 +12:00
}
2019-05-12 16:56:55 +12:00
2021-12-13 21:08:27 +13:00
if (!$deviceFiles->exists($path)) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2022-05-24 02:54:50 +12:00
if (empty($output)) {
2022-03-20 15:48:40 +13:00
// when file extension is not provided and the mime type is not one of our supported outputs
// we fallback to `jpg` output format
2022-03-19 00:51:59 +13:00
$output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type;
2022-03-19 00:07:54 +13:00
}
2019-05-09 18:54:39 +12:00
2021-12-13 21:08:27 +13:00
$source = $deviceFiles->read($path);
2020-06-30 23:09:28 +12:00
if (!empty($cipher)) { // Decrypt
$source = OpenSSL::decrypt(
$source,
2021-05-03 20:28:31 +12:00
$file->getAttribute('openSSLCipher'),
2021-07-07 22:07:11 +12:00
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
2020-06-30 23:09:28 +12:00
0,
2021-05-03 20:28:31 +12:00
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
2020-06-30 23:09:28 +12:00
);
}
2019-05-09 18:54:39 +12:00
2022-08-31 14:56:44 +12:00
switch ($algorithm) {
2022-08-31 01:46:55 +12:00
case 'zstd':
$compressor = new Zstd();
$source = $compressor->decompress($source);
break;
case 'gzip':
$compressor = new GZIP();
$source = $compressor->decompress($source);
break;
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2021-02-20 02:59:46 +13:00
$image = new Image($source);
2019-05-09 18:54:39 +12:00
2021-06-12 00:26:04 +12:00
$image->crop((int) $width, (int) $height, $gravity);
if (!empty($opacity) || $opacity === 0) {
2021-04-09 19:42:45 +12:00
$image->setOpacity($opacity);
}
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
if (!empty($background)) {
2021-07-07 22:07:11 +12:00
$image->setBackground('#' . $background);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2021-07-07 22:07:11 +12:00
if (!empty($borderWidth)) {
$image->setBorder($borderWidth, '#' . $borderColor);
}
2019-05-09 18:54:39 +12:00
2021-03-22 19:54:42 +13:00
if (!empty($borderRadius)) {
$image->setBorderRadius($borderRadius);
}
if (!empty($rotation)) {
$image->setRotation(($rotation + 360) % 360);
2021-03-22 19:54:42 +13:00
}
2021-02-20 02:59:46 +13:00
$data = $image->output($output, $quality);
2020-07-03 15:11:16 +12:00
2022-07-24 21:49:51 +12:00
$contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg'];
2022-07-25 03:22:31 +12:00
2020-06-30 23:09:28 +12:00
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT')
2022-07-24 21:49:51 +12:00
->setContentType($contentType)
2022-08-15 03:01:34 +12:00
->file($data)
2020-06-30 23:09:28 +12:00
;
2019-05-09 18:54:39 +12:00
2021-02-20 02:59:46 +13:00
unset($image);
2020-12-27 05:48:36 +13:00
});
2019-05-09 18:54:39 +12:00
App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->alias('/v1/storage/files/:fileId/download', ['bucketId' => 'default'])
2019-08-25 20:10:28 +12:00
->desc('Get File for Download')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
->label('scope', 'files.read')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2019-05-09 18:54:39 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'getFileDownload')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/get-file-download.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
2020-11-12 18:12:25 +13:00
->label('sdk.response.type', '*/*')
2020-04-11 06:59:14 +12:00
->label('sdk.methodType', 'location')
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
2021-09-12 19:03:25 +12:00
->inject('request')
2020-12-27 05:48:36 +13:00
->inject('response')
->inject('dbForProject')
2021-11-19 22:45:42 +13:00
->inject('mode')
2021-12-13 21:08:27 +13:00
->inject('deviceFiles')
2022-08-11 14:18:22 +12:00
->action(function (string $bucketId, string $fileId, Request $request, Response $response, Database $dbForProject, string $mode, Device $deviceFiles) {
2019-05-09 18:54:39 +12:00
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2019-05-09 18:54:39 +12:00
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
2022-08-08 22:58:36 +12:00
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
2022-08-25 01:24:54 +12:00
$validator = new Authorization(Database::PERMISSION_READ);
2022-08-08 22:58:36 +12:00
$valid = $validator->isValid($bucket->getRead());
2022-08-25 01:24:54 +12:00
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2021-11-07 19:35:49 +13:00
}
2022-08-25 01:24:54 +12:00
if ($fileSecurity && !$valid) {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
2022-08-25 01:24:54 +12:00
} else {
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
2022-08-26 15:01:16 +12:00
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
$path = $file->getAttribute('path', '');
2019-05-09 18:54:39 +12:00
2021-12-13 21:08:27 +13:00
if (!$deviceFiles->exists($path)) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2021-07-13 19:49:19 +12:00
$response
->setContentType($file->getAttribute('mimeType'))
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
->addHeader('Content-Disposition', 'attachment; filename="' . $file->getAttribute('name', '') . '"')
;
2019-05-09 18:54:39 +12:00
2021-09-13 18:17:45 +12:00
$size = $file->getAttribute('sizeOriginal', 0);
2020-06-30 23:09:28 +12:00
2021-09-12 20:50:43 +12:00
$rangeHeader = $request->getHeader('range');
if (!empty($rangeHeader)) {
2021-09-30 22:46:45 +13:00
$start = $request->getRangeStart();
$end = $request->getRangeEnd();
$unit = $request->getRangeUnit();
2021-12-09 23:44:44 +13:00
if ($end === null) {
$end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1));
2021-09-30 22:46:45 +13:00
}
2021-12-09 23:44:44 +13:00
if ($unit !== 'bytes' || $start >= $end || $end >= $size) {
throw new Exception(Exception::STORAGE_INVALID_RANGE);
2021-09-12 19:03:25 +12:00
}
2021-09-30 22:46:45 +13:00
$response
->addHeader('Accept-Ranges', 'bytes')
->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size)
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
2021-09-12 19:03:25 +12:00
}
2021-07-15 22:35:08 +12:00
$source = '';
2021-05-03 20:28:31 +12:00
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
2021-12-13 21:08:27 +13:00
$source = $deviceFiles->read($path);
2020-06-30 23:09:28 +12:00
$source = OpenSSL::decrypt(
$source,
2021-05-03 20:28:31 +12:00
$file->getAttribute('openSSLCipher'),
2021-07-13 19:49:19 +12:00
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
2020-06-30 23:09:28 +12:00
0,
2021-05-03 20:28:31 +12:00
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
2020-06-30 23:09:28 +12:00
);
}
2019-05-09 18:54:39 +12:00
2022-08-31 14:56:44 +12:00
switch ($file->getAttribute('algorithm', 'none')) {
2022-08-31 01:46:55 +12:00
case 'zstd':
if (empty($source)) {
$source = $deviceFiles->read($path);
}
$compressor = new Zstd();
$source = $compressor->decompress($source);
break;
case 'gzip':
if (empty($source)) {
$source = $deviceFiles->read($path);
}
$compressor = new GZIP();
$source = $compressor->decompress($source);
break;
2021-06-22 20:36:44 +12:00
}
2021-12-09 23:44:44 +13:00
if (!empty($source)) {
if (!empty($rangeHeader)) {
2021-09-30 22:46:45 +13:00
$response->send(substr($source, $start, ($end - $start + 1)));
2021-09-12 20:50:43 +12:00
}
2021-07-15 22:35:08 +12:00
$response->send($source);
}
2019-05-09 18:54:39 +12:00
2021-12-09 23:44:44 +13:00
if (!empty($rangeHeader)) {
2021-12-13 21:08:27 +13:00
$response->send($deviceFiles->read($path, $start, ($end - $start + 1)));
2021-09-12 20:50:43 +12:00
}
2021-08-16 19:25:20 +12:00
if ($size > APP_STORAGE_READ_BUFFER) {
2021-12-13 21:08:27 +13:00
$response->addHeader('Content-Length', $deviceFiles->getFileSize($path));
for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
2021-12-10 23:29:59 +13:00
$response->chunk(
2021-12-13 21:08:27 +13:00
$deviceFiles->read(
2021-12-10 23:29:59 +13:00
$path,
($i * MAX_OUTPUT_CHUNK_SIZE),
min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))
),
(($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size
);
2021-07-15 22:35:08 +12:00
}
} else {
2021-12-13 21:08:27 +13:00
$response->send($deviceFiles->read($path));
2021-07-15 22:35:08 +12:00
}
2020-12-27 05:48:36 +13:00
});
2019-05-09 18:54:39 +12:00
App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->alias('/v1/storage/files/:fileId/view', ['bucketId' => 'default'])
2019-08-25 20:10:28 +12:00
->desc('Get File for View')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
->label('scope', 'files.read')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2019-05-09 18:54:39 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'getFileView')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/get-file-view.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
2020-11-12 18:12:25 +13:00
->label('sdk.response.type', '*/*')
2020-04-11 06:59:14 +12:00
->label('sdk.methodType', 'location')
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
2020-12-27 05:48:36 +13:00
->inject('response')
->inject('request')
->inject('dbForProject')
2021-11-19 22:45:42 +13:00
->inject('mode')
2021-12-13 21:08:27 +13:00
->inject('deviceFiles')
2022-08-11 14:18:22 +12:00
->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, string $mode, Device $deviceFiles) {
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
2022-08-08 22:58:36 +12:00
$fileSecurity = $bucket->getAttribute('fileSecurity', false);
2022-08-25 01:24:54 +12:00
$validator = new Authorization(Database::PERMISSION_READ);
2022-08-08 22:58:36 +12:00
$valid = $validator->isValid($bucket->getRead());
2022-08-25 01:24:54 +12:00
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2021-11-07 19:35:49 +13:00
}
2019-05-09 18:54:39 +12:00
2022-08-25 01:24:54 +12:00
if ($fileSecurity && !$valid) {
$file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
2022-08-25 01:24:54 +12:00
} else {
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
2019-05-09 18:54:39 +12:00
2022-08-26 15:01:16 +12:00
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2022-08-08 22:58:36 +12:00
$mimes = Config::getParam('storage-mimes');
2020-06-30 23:09:28 +12:00
$path = $file->getAttribute('path', '');
2019-05-09 18:54:39 +12:00
2021-12-13 21:08:27 +13:00
if (!$deviceFiles->exists($path)) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
$contentType = 'text/plain';
2019-05-09 18:54:39 +12:00
2020-06-30 23:09:28 +12:00
if (\in_array($file->getAttribute('mimeType'), $mimes)) {
$contentType = $file->getAttribute('mimeType');
}
2019-05-09 18:54:39 +12:00
2021-07-13 19:49:19 +12:00
$response
->setContentType($contentType)
->addHeader('Content-Security-Policy', 'script-src none;')
->addHeader('X-Content-Type-Options', 'nosniff')
->addHeader('Content-Disposition', 'inline; filename="' . $file->getAttribute('name', '') . '"')
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->addHeader('X-Peak', \memory_get_peak_usage())
;
$size = $file->getAttribute('sizeOriginal', 0);
$rangeHeader = $request->getHeader('range');
if (!empty($rangeHeader)) {
2021-09-30 22:47:53 +13:00
$start = $request->getRangeStart();
$end = $request->getRangeEnd();
$unit = $request->getRangeUnit();
2021-12-09 23:44:44 +13:00
if ($end === null) {
$end = min(($start + 2000000 - 1), ($size - 1));
2021-09-30 22:47:53 +13:00
}
2021-12-09 23:44:44 +13:00
if ($unit != 'bytes' || $start >= $end || $end >= $size) {
throw new Exception(Exception::STORAGE_INVALID_RANGE);
}
2020-06-30 23:09:28 +12:00
2021-09-30 22:47:53 +13:00
$response
->addHeader('Accept-Ranges', 'bytes')
2021-12-09 23:44:44 +13:00
->addHeader('Content-Range', "bytes $start-$end/$size")
2021-09-30 22:47:53 +13:00
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
}
2021-07-15 22:35:08 +12:00
$source = '';
2021-05-03 20:28:31 +12:00
if (!empty($file->getAttribute('openSSLCipher'))) { // Decrypt
2021-12-13 21:08:27 +13:00
$source = $deviceFiles->read($path);
2020-06-30 23:09:28 +12:00
$source = OpenSSL::decrypt(
$source,
2021-05-03 20:28:31 +12:00
$file->getAttribute('openSSLCipher'),
2021-07-13 19:49:19 +12:00
App::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
2020-06-30 23:09:28 +12:00
0,
2021-05-03 20:28:31 +12:00
\hex2bin($file->getAttribute('openSSLIV')),
\hex2bin($file->getAttribute('openSSLTag'))
2020-06-30 23:09:28 +12:00
);
}
2019-05-09 18:54:39 +12:00
2022-08-31 14:56:44 +12:00
switch ($file->getAttribute('algorithm', 'none')) {
2022-08-31 01:46:55 +12:00
case 'zstd':
if (empty($source)) {
$source = $deviceFiles->read($path);
}
$compressor = new Zstd();
$source = $compressor->decompress($source);
break;
case 'gzip':
if (empty($source)) {
$source = $deviceFiles->read($path);
}
$compressor = new GZIP();
$source = $compressor->decompress($source);
break;
2021-07-15 22:35:08 +12:00
}
2019-05-09 18:54:39 +12:00
2021-12-09 23:44:44 +13:00
if (!empty($source)) {
if (!empty($rangeHeader)) {
2021-09-30 22:47:53 +13:00
$response->send(substr($source, $start, ($end - $start + 1)));
}
2021-07-15 22:35:08 +12:00
$response->send($source);
}
2021-12-09 23:44:44 +13:00
if (!empty($rangeHeader)) {
2021-12-13 21:08:27 +13:00
$response->send($deviceFiles->read($path, $start, ($end - $start + 1)));
}
2021-12-13 21:08:27 +13:00
$size = $deviceFiles->getFileSize($path);
if ($size > APP_STORAGE_READ_BUFFER) {
2021-12-13 21:08:27 +13:00
$response->addHeader('Content-Length', $deviceFiles->getFileSize($path));
for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
2021-12-09 23:44:44 +13:00
$response->chunk(
2021-12-13 21:08:27 +13:00
$deviceFiles->read(
2021-12-10 23:29:59 +13:00
$path,
($i * MAX_OUTPUT_CHUNK_SIZE),
min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))
),
(($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size
);
2021-07-15 22:35:08 +12:00
}
} else {
2021-12-13 21:08:27 +13:00
$response->send($deviceFiles->read($path));
2021-07-15 22:35:08 +12:00
}
2020-12-27 05:48:36 +13:00
});
2019-05-09 18:54:39 +12:00
App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->alias('/v1/storage/files/:fileId', ['bucketId' => 'default'])
2019-08-29 00:29:45 +12:00
->desc('Update File')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
2019-08-29 00:29:45 +12:00
->label('scope', 'files.write')
->label('event', 'buckets.[bucketId].files.[fileId].update')
2022-09-05 20:00:08 +12:00
->label('audits.event', 'file.update')
2022-09-09 00:16:54 +12:00
->label('audits.resource', 'file/{response.$id}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.update')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2019-08-29 00:29:45 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'updateFile')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/update-file.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_FILE)
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File unique ID.')
2022-09-06 21:17:25 +12:00
->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE, [Database::PERMISSION_READ, Database::PERMISSION_UPDATE, Database::PERMISSION_DELETE, Database::PERMISSION_WRITE]), 'An array of permission string. By default the current permissions are inherited. [Learn more about permissions](/docs/permissions).', true)
2020-12-27 05:48:36 +13:00
->inject('response')
->inject('dbForProject')
->inject('user')
2021-11-19 22:45:42 +13:00
->inject('mode')
2022-02-16 14:30:27 +13:00
->inject('events')
->action(function (string $bucketId, string $fileId, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $events) {
2022-08-08 22:58:36 +12:00
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
2019-08-29 00:29:45 +12:00
2022-08-08 22:58:36 +12:00
$fileSecurity = $bucket->getAttributes('fileSecurity', false);
2022-08-25 01:24:54 +12:00
$validator = new Authorization(Database::PERMISSION_UPDATE);
2022-08-08 22:58:36 +12:00
$valid = $validator->isValid($bucket->getUpdate());
2022-08-25 01:24:54 +12:00
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2020-06-30 23:09:28 +12:00
}
2019-08-29 00:29:45 +12:00
// Read permission should not be required for update
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
2019-08-29 00:29:45 +12:00
2022-08-26 15:01:16 +12:00
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2022-08-25 15:51:21 +12:00
// Map aggregate permissions into the multiple permissions they represent.
$permissions = Permission::aggregate($permissions, [
Database::PERMISSION_READ,
Database::PERMISSION_UPDATE,
Database::PERMISSION_DELETE,
]);
2022-08-16 00:56:19 +12:00
// Users can only manage their own roles, API keys and Admin users can manage any
$roles = Authorization::getRoles();
2022-08-26 15:01:16 +12:00
if (!Auth::isAppUser($roles) && !Auth::isPrivilegedUser($roles) && !\is_null($permissions)) {
2022-08-16 00:56:19 +12:00
foreach (Database::PERMISSIONS as $type) {
foreach ($permissions as $permission) {
2022-08-16 23:26:38 +12:00
$permission = Permission::parse($permission);
if ($permission->getPermission() != $type) {
2022-08-16 00:56:19 +12:00
continue;
}
2022-08-16 23:26:38 +12:00
$role = (new Role(
$permission->getRole(),
$permission->getIdentifier(),
$permission->getDimension()
))->toString();
2022-08-16 00:56:19 +12:00
if (!Authorization::isRole($role)) {
2022-08-25 15:51:21 +12:00
throw new Exception(Exception::USER_UNAUTHORIZED, 'Permissions must be one of: (' . \implode(', ', $roles) . ')');
2022-08-16 00:56:19 +12:00
}
}
}
}
2022-08-26 15:01:16 +12:00
if (\is_null($permissions)) {
$permissions = $file->getPermissions() ?? [];
}
$file->setAttribute('$permissions', $permissions);
2022-03-15 22:51:51 +13:00
if ($fileSecurity && !$valid) {
try {
$file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
} else {
$file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
}
$events
->setParam('bucketId', $bucket->getId())
->setParam('fileId', $file->getId())
Database layer (#3338) * database response model * database collection config * new database scopes * database service update * database execption codes * remove read write permission from database model * updating tests and fixing some bugs * server side tests are now passing * databases api * tests for database endpoint * composer update * fix error * formatting * formatting fixes * get database test * more updates to events and usage * more usage updates * fix delete type * fix test * delete database * more fixes * databaseId in attributes and indexes * more fixes * fix issues * fix index subquery * fix console scope and index query * updating tests as required * fix phpcs errors and warnings * updates to review suggestions * UI progress * ui updates and cleaning up * fix type * rework database events * update tests * update types * event generation fixed * events config updated * updating context to support multiple * realtime updates * fix ids * update context * validator updates * fix naming conflict * fix tests * fix lint errors * fix wprler and realtime tests * fix webhooks test * fix event validator and other tests * formatting fixes * removing leftover var_dumps * remove leftover comment * update usage params * usage metrics updates * update database usage * fix usage * specs update * updates to usage * fix UI and usage * fix lints * internal id fixes * fixes for internal Id * renaming services and related files * rename tests * rename doc link * rename readme * fix test name * tests: fixes for 0.15.x sync Co-authored-by: Torsten Dittmann <torsten.dittmann@googlemail.com>
2022-06-22 22:51:49 +12:00
->setContext('bucket', $bucket)
2020-06-30 23:09:28 +12:00
;
2020-10-31 08:53:27 +13:00
$response->dynamic($file, Response::MODEL_FILE);
2020-12-27 05:48:36 +13:00
});
2019-08-29 00:29:45 +12:00
App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->alias('/v1/storage/files/:fileId', ['bucketId' => 'default'])
2019-05-09 18:54:39 +12:00
->desc('Delete File')
2020-06-26 06:32:12 +12:00
->groups(['api', 'storage'])
2019-08-14 23:53:07 +12:00
->label('scope', 'files.write')
->label('event', 'buckets.[bucketId].files.[fileId].delete')
2022-09-05 20:00:08 +12:00
->label('audits.event', 'file.delete')
2022-08-15 16:29:44 +12:00
->label('audits.resource', 'file/{request.fileId}')
2022-08-11 14:18:22 +12:00
->label('usage.metric', 'files.{scope}.requests.delete')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
2021-04-16 19:22:17 +12:00
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
2019-05-09 18:54:39 +12:00
->label('sdk.namespace', 'storage')
2020-01-31 09:58:49 +13:00
->label('sdk.method', 'deleteFile')
2019-10-08 20:09:35 +13:00
->label('sdk.description', '/docs/references/storage/delete-file.md')
2020-11-12 10:02:24 +13:00
->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
->label('sdk.response.model', Response::MODEL_NONE)
2022-09-19 22:05:42 +12:00
->param('bucketId', '', new UID(), 'Storage bucket unique ID. You can create a new storage bucket using the Storage service [server integration](/docs/server/storage#createBucket).')
->param('fileId', '', new UID(), 'File ID.')
2020-12-27 05:48:36 +13:00
->inject('response')
->inject('dbForProject')
2020-12-27 05:48:36 +13:00
->inject('events')
2021-11-19 22:45:42 +13:00
->inject('mode')
2021-12-13 21:08:27 +13:00
->inject('deviceFiles')
2022-08-15 21:05:41 +12:00
->inject('deletes')
->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $events, string $mode, Device $deviceFiles, Delete $deletes) {
2022-03-15 22:51:51 +13:00
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
2019-05-09 18:54:39 +12:00
2022-08-08 22:58:36 +12:00
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2019-05-09 18:54:39 +12:00
2022-08-08 22:58:36 +12:00
$fileSecurity = $bucket->getAttributes('fileSecurity', false);
$validator = new Authorization(Database::PERMISSION_DELETE);
2022-08-08 22:58:36 +12:00
$valid = $validator->isValid($bucket->getDelete());
2022-08-25 01:24:54 +12:00
if (!$fileSecurity && !$valid) {
throw new Exception(Exception::USER_UNAUTHORIZED);
2021-11-07 19:35:49 +13:00
}
2019-05-09 18:54:39 +12:00
// Read permission should not be required for delete
$file = Authorization::skip(fn() => $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId));
2019-05-09 18:54:39 +12:00
2022-08-26 15:01:16 +12:00
if ($file->isEmpty()) {
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
2020-06-30 23:09:28 +12:00
}
2022-08-13 02:46:50 +12:00
// Make sure we don't delete the file before the document permission check occurs
if ($fileSecurity && !$valid && !$validator->isValid($file->getDelete())) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
$deviceDeleted = false;
if ($file->getAttribute('chunksTotal') !== $file->getAttribute('chunksUploaded')) {
2022-01-16 20:47:03 +13:00
$deviceDeleted = $deviceFiles->abort(
$file->getAttribute('path'),
($file->getAttribute('metadata', [])['uploadId'] ?? '')
);
} else {
$deviceDeleted = $deviceFiles->delete($file->getAttribute('path'));
}
if ($deviceDeleted) {
2022-08-15 21:05:41 +12:00
$deletes
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
->setResource('file/' . $fileId)
;
2022-01-25 20:52:11 +13:00
if ($fileSecurity && !$valid) {
try {
$deleted = $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId);
} catch (AuthorizationException) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
} else {
$deleted = Authorization::skip(fn() => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId));
}
2022-08-08 22:58:36 +12:00
2021-11-07 19:35:49 +13:00
if (!$deleted) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB');
2019-05-09 18:54:39 +12:00
}
2021-07-08 23:26:11 +12:00
} else {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to delete file from device');
2020-06-30 23:09:28 +12:00
}
2021-07-07 22:07:11 +12:00
2020-12-07 11:14:57 +13:00
$events
->setParam('bucketId', $bucket->getId())
->setParam('fileId', $file->getId())
Database layer (#3338) * database response model * database collection config * new database scopes * database service update * database execption codes * remove read write permission from database model * updating tests and fixing some bugs * server side tests are now passing * databases api * tests for database endpoint * composer update * fix error * formatting * formatting fixes * get database test * more updates to events and usage * more usage updates * fix delete type * fix test * delete database * more fixes * databaseId in attributes and indexes * more fixes * fix issues * fix index subquery * fix console scope and index query * updating tests as required * fix phpcs errors and warnings * updates to review suggestions * UI progress * ui updates and cleaning up * fix type * rework database events * update tests * update types * event generation fixed * events config updated * updating context to support multiple * realtime updates * fix ids * update context * validator updates * fix naming conflict * fix tests * fix lint errors * fix wprler and realtime tests * fix webhooks test * fix event validator and other tests * formatting fixes * removing leftover var_dumps * remove leftover comment * update usage params * usage metrics updates * update database usage * fix usage * specs update * updates to usage * fix UI and usage * fix lints * internal id fixes * fixes for internal Id * renaming services and related files * rename tests * rename doc link * rename readme * fix test name * tests: fixes for 0.15.x sync Co-authored-by: Torsten Dittmann <torsten.dittmann@googlemail.com>
2022-06-22 22:51:49 +12:00
->setContext('bucket', $bucket)
->setPayload($response->output($file, Response::MODEL_FILE))
2020-10-31 08:53:27 +13:00
;
2020-06-30 23:09:28 +12:00
$response->noContent();
2020-12-27 05:48:36 +13:00
});
2019-05-09 18:54:39 +12:00
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
2022-05-05 00:23:34 +12:00
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
2021-11-07 18:50:21 +13:00
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
2021-10-28 11:17:15 +13:00
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
2022-08-11 14:18:22 +12:00
'project.$all.storage.size',
'buckets.$all.count.total',
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
2022-08-13 17:01:43 +12:00
'files.$all.storage.size',
'files.$all.count.total',
2022-08-11 14:18:22 +12:00
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
2021-10-28 11:17:15 +13:00
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
2022-08-12 11:53:52 +12:00
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
2021-10-27 02:19:28 +13:00
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
2022-05-24 02:54:50 +12:00
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
2022-05-24 02:54:50 +12:00
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
2022-09-16 11:48:09 +12:00
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
2021-10-27 02:19:28 +13:00
}
});
$usage = new Document([
'range' => $range,
2022-08-11 14:18:22 +12:00
'bucketsCount' => $stats['buckets.$all.count.total'],
'bucketsCreate' => $stats['buckets.$all.requests.create'],
'bucketsRead' => $stats['buckets.$all.requests.read'],
'bucketsUpdate' => $stats['buckets.$all.requests.update'],
'bucketsDelete' => $stats['buckets.$all.requests.delete'],
2022-08-13 17:01:43 +12:00
'storage' => $stats['project.$all.storage.size'],
'filesCount' => $stats['files.$all.count.total'],
2022-08-11 14:18:22 +12:00
'filesCreate' => $stats['files.$all.requests.create'],
'filesRead' => $stats['files.$all.requests.read'],
'filesUpdate' => $stats['files.$all.requests.update'],
'filesDelete' => $stats['files.$all.requests.delete'],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
2021-08-21 00:10:52 +12:00
->desc('Get usage stats for a storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucketUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
->param('bucketId', '', new UID(), 'Bucket ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
2022-05-05 00:23:34 +12:00
->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) {
$bucket = $dbForProject->getDocument('buckets', $bucketId);
2021-12-09 23:44:44 +13:00
if ($bucket->isEmpty()) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$usage = [];
2021-11-07 18:50:21 +13:00
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
2021-10-28 11:17:15 +13:00
$periods = [
'24h' => [
'period' => '30m',
'limit' => 48,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
2022-08-13 16:39:01 +12:00
"files.{$bucketId}.count.total",
"files.{$bucketId}.storage.size",
"files.{$bucketId}.requests.create",
"files.{$bucketId}.requests.read",
"files.{$bucketId}.requests.update",
"files.{$bucketId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
2021-10-28 11:17:15 +13:00
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
2022-09-03 02:19:36 +12:00
$requestDocs = $dbForProject->find('stats', [
2022-08-12 11:53:52 +12:00
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
2021-10-27 02:19:28 +13:00
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
2022-05-24 02:54:50 +12:00
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
2022-05-24 02:54:50 +12:00
$diff = match ($period) { // convert period to seconds for unix timestamp math
'30m' => 1800,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
2022-09-16 11:48:09 +12:00
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
2021-10-27 02:19:28 +13:00
}
});
$usage = new Document([
'range' => $range,
2022-08-13 16:39:01 +12:00
'filesCount' => $stats[$metrics[0]],
'filesStorage' => $stats[$metrics[1]],
'filesCreate' => $stats[$metrics[2]],
'filesRead' => $stats[$metrics[3]],
'filesUpdate' => $stats[$metrics[4]],
'filesDelete' => $stats[$metrics[5]],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
});