CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-01 18:39:57 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch '0.13.x' of github.com:appwrite/appwrite into feat-functions-refactor

Browse source
This commit is contained in:
Christy Jacob 2022-02-28 20:18:48 +04:00
parent 9c444c32a1 1c27b5a88f
commit 37c6fd75d8
20 changed files with 322 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.env
View file

@ -1,6 +1,7 @@
_APP_ENV=production
_APP_ENV=development
_APP_LOCALE=en
_APP_WORKER_PER_CORE=6
_APP_CONSOLE_WHITELIST_ROOT=disabled
_APP_CONSOLE_WHITELIST_EMAILS=
_APP_CONSOLE_WHITELIST_IPS=

1
Dockerfile
View file

@ -134,6 +134,7 @@ ENV DEBUG=$DEBUG
ENV _APP_SERVER=swoole \
_APP_ENV=production \
_APP_LOCALE=en \
_APP_WORKER_PER_CORE= \
_APP_DOMAIN=localhost \
_APP_DOMAIN_TARGET=localhost \
_APP_HOME=https://appwrite.io \

9
app/config/variables.php
View file

@ -176,6 +176,15 @@ return [
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_WORKER_PER_CORE',
'description' => 'Internal Worker per core for the API, Realtime and Executor containers. Can be configured to optimize performance.',
'introduction' => '0.13.0',
'default' => 6,
'required' => false,
'question' => '',
'filter' => ''
]
],
],

57
app/controllers/api/database.php
View file

@ -111,7 +111,7 @@ function createAttribute(string $collectionId, Document $attribute, Response $re
}
$dbForProject->deleteCachedDocument('collections', $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId());
// Pass clone of $attribute object to workers
// so we can later modify Document to fit response model
@ -166,7 +166,7 @@ App::post('/v1/database/collections')
$collectionId = $collectionId == 'unique()' ? $dbForProject->getId() : $collectionId;
try {
$collection = $dbForProject->createDocument('collections', new Document([
$dbForProject->createDocument('collections', new Document([
'$id' => $collectionId,
'$read' => $read ?? [], // Collection permissions for collection documents (based on permission model)
'$write' => $write ?? [], // Collection permissions for collection documents (based on permission model)
@ -177,8 +177,9 @@ App::post('/v1/database/collections')
'name' => $name,
'search' => implode(' ', [$collectionId, $name]),
]));
$collection = $dbForProject->getDocument('collections', $collectionId);
$dbForProject->createCollection('collection_' . $collectionId);
$dbForProject->createCollection('collection_' . $collection->getInternalId());
} catch (DuplicateException $th) {
throw new Exception('Collection already exists', 409, Exception::COLLECTION_ALREADY_EXISTS);
} catch (LimitException $th) {
@ -402,7 +403,8 @@ App::get('/v1/database/:collectionId/usage')
/** @var Utopia\Database\Database $dbForProject */
/** @var Utopia\Registry\Registry $register */
$collection = $dbForProject->getCollection('collection_' . $collectionId);
$collectionDocument = $dbForProject->getDocument('collections', $collectionId);
$collection = $dbForProject->getCollection('collection_' . $collectionDocument->getInternalId());
if ($collection->isEmpty()) {
throw new Exception('Collection not found', 404, Exception::COLLECTION_NOT_FOUND);
@ -513,7 +515,8 @@ App::get('/v1/database/collections/:collectionId/logs')
/** @var Utopia\Locale\Locale $locale */
/** @var MaxMind\Db\Reader $geodb */
$collection = $dbForProject->getCollection('collection_' . $collectionId);
$collectionDocument = $dbForProject->getDocument('collections', $collectionId);
$collection = $dbForProject->getCollection('collection_' . $collectionDocument->getInternalId());
if ($collection->isEmpty()) {
throw new Exception('Collection not found', 404, Exception::COLLECTION_NOT_FOUND);
@ -676,7 +679,7 @@ App::delete('/v1/database/collections/:collectionId')
throw new Exception('Failed to remove collection from DB', 500, Exception::GENERAL_SERVER_ERROR);
}
$dbForProject->deleteCachedCollection('collection_' . $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId());
$deletes
->setParam('type', DELETE_TYPE_DOCUMENT)
@ -1259,7 +1262,7 @@ App::delete('/v1/database/collections/:collectionId/attributes/:key')
}
$dbForProject->deleteCachedDocument('collections', $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId());
$database
->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE)
@ -1642,9 +1645,9 @@ App::post('/v1/database/collections/:collectionId/documents')
try {
if ($collection->getAttribute('permission') === 'collection') {
/** @var Document $document */
$document = Authorization::skip(fn() => $dbForProject->createDocument('collection_' . $collectionId, new Document($data)));
$document = Authorization::skip(fn() => $dbForProject->createDocument('collection_' . $collection->getInternalId(), new Document($data)));
} else {
$document = $dbForProject->createDocument('collection_' . $collectionId, new Document($data));
$document = $dbForProject->createDocument('collection_' . $collection->getInternalId(), new Document($data));
}
$document->setAttribute('$collection', $collectionId);
}
@ -1742,8 +1745,8 @@ App::get('/v1/database/collections/:collectionId/documents')
$cursorDocument = null;
if (!empty($cursor)) {
$cursorDocument = $collection->getAttribute('permission') === 'collection'
? Authorization::skip(fn () => $dbForProject->getDocument('collection_' . $collectionId, $cursor))
: $dbForProject->getDocument('collection_' . $collectionId, $cursor);
? Authorization::skip(fn () => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $cursor))
: $dbForProject->getDocument('collection_' . $collection->getInternalId(), $cursor);
if ($cursorDocument->isEmpty()) {
throw new Exception("Document '{$cursor}' for the 'cursor' value not found.", 400, Exception::GENERAL_CURSOR_NOT_FOUND);
@ -1752,11 +1755,11 @@ App::get('/v1/database/collections/:collectionId/documents')
if ($collection->getAttribute('permission') === 'collection') {
/** @var Document[] $documents */
$documents = Authorization::skip(fn() => $dbForProject->find('collection_' . $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection));
$sum = Authorization::skip(fn() => $dbForProject->count('collection_' . $collectionId, $queries, APP_LIMIT_COUNT));
$documents = Authorization::skip(fn() => $dbForProject->find('collection_' . $collection->getInternalId(), $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection));
$sum = Authorization::skip(fn() => $dbForProject->count('collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT));
} else {
$documents = $dbForProject->find('collection_' . $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection);
$sum = $dbForProject->count('collection_' . $collectionId, $queries, APP_LIMIT_COUNT);
$documents = $dbForProject->find('collection_' . $collection->getInternalId(), $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursorDocument ?? null, $cursorDirection);
$sum = $dbForProject->count('collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT);
}
/**
@ -1818,9 +1821,9 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId')
if ($collection->getAttribute('permission') === 'collection') {
/** @var Document $document */
$document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collectionId, $documentId));
$document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId));
} else {
$document = $dbForProject->getDocument('collection_' . $collectionId, $documentId);
$document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId);
}
/**
@ -1872,7 +1875,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId/logs')
throw new Exception('Collection not found', 404, Exception::COLLECTION_NOT_FOUND);
}
$document = $dbForProject->getDocument('collection_' . $collectionId, $documentId);
$document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId);
if ($document->isEmpty()) {
throw new Exception('No document found', 404, Exception::DOCUMENT_NOT_FOUND);
@ -1981,9 +1984,9 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId')
throw new Exception('Unauthorized permissions', 401, Exception::USER_UNAUTHORIZED);
}
$document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collectionId, $documentId));
$document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId));
} else {
$document = $dbForProject->getDocument('collection_' . $collectionId, $documentId);
$document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId);
}
@ -2031,9 +2034,9 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId')
try {
if ($collection->getAttribute('permission') === 'collection') {
/** @var Document $document */
$document = Authorization::skip(fn() => $dbForProject->updateDocument('collection_' . $collection->getId(), $document->getId(), new Document($data)));
$document = Authorization::skip(fn() => $dbForProject->updateDocument('collection_' . $collection->getInternalId(), $document->getId(), new Document($data)));
} else {
$document = $dbForProject->updateDocument('collection_' . $collection->getId(), $document->getId(), new Document($data));
$document = $dbForProject->updateDocument('collection_' . $collection->getInternalId(), $document->getId(), new Document($data));
}
/**
* Reset $collection attribute to remove prefix.
@ -2116,9 +2119,9 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId')
if ($collection->getAttribute('permission') === 'collection') {
/** @var Document $document */
$document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collectionId, $documentId));
$document = Authorization::skip(fn() => $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId));
} else {
$document = $dbForProject->getDocument('collection_' . $collectionId, $documentId);
$document = $dbForProject->getDocument('collection_' . $collection->getInternalId(), $documentId);
}
if ($document->isEmpty()) {
@ -2126,12 +2129,12 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId')
}
if ($collection->getAttribute('permission') === 'collection') {
Authorization::skip(fn() => $dbForProject->deleteDocument('collection_' . $collectionId, $documentId));
Authorization::skip(fn() => $dbForProject->deleteDocument('collection_' . $collection->getInternalId(), $documentId));
} else {
$dbForProject->deleteDocument('collection_' . $collectionId, $documentId);
$dbForProject->deleteDocument('collection_' . $collection->getInternalId(), $documentId);
}
$dbForProject->deleteCachedDocument('collection_' . $collectionId, $documentId);
$dbForProject->deleteCachedDocument('collection_' . $collection->getInternalId(), $documentId);
/**
* Reset $collection attribute to remove prefix.

8
app/controllers/api/projects.php
View file

@ -101,10 +101,10 @@ App::post('/v1/projects')
'auths' => $auths,
'search' => implode(' ', [$projectId, $name]),
]));
/** @var array $collections */
$collections = Config::getParam('collections', []);
$collections = Config::getParam('collections', []); /** @var array $collections */
$dbForProject->setNamespace('_project_' . $project->getId());
$dbForProject->setNamespace("_{$project->getId()}");
$dbForProject->create('appwrite');
$audit = new Audit($dbForProject);
@ -269,7 +269,7 @@ App::get('/v1/projects/:projectId/usage')
],
];
$dbForProject->setNamespace('_project_' . $projectId);
$dbForProject->setNamespace("_{$projectId}");
$metrics = [
'requests',

2
app/http.php
View file

@ -23,9 +23,11 @@ use Utopia\Logger\Log\User;
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
$payloadSize = 6 * (1024 * 1024); // 6MB
$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$http
->set([
'worker_num' => $workerNumber,
'open_http2_protocol' => true,
// 'document_root' => __DIR__.'/../public',
// 'enable_static_handler' => true,

4
app/init.php
View file

@ -786,7 +786,7 @@ App::setResource('dbForProject', function($db, $cache, $project) {
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace('_project_'.$project->getId());
$database->setNamespace("_{$project->getId()}");
return $database;
}, ['db', 'cache', 'project']);
@ -796,7 +796,7 @@ App::setResource('dbForConsole', function($db, $cache) {
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace('_project_console');
$database->setNamespace('_console');
return $database;
}, ['db', 'cache']);

17
app/realtime.php
View file

@ -46,9 +46,12 @@ $stats->create();
$containerId = uniqid();
$statsDocument = null;
$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
$adapter = new Adapter\Swoole(port: App::getEnv('PORT', 80));
$adapter->setPackageMaxLength(64000); // Default maximum Package Size (64kb)
$adapter
->setPackageMaxLength(64000) // Default maximum Package Size (64kb)
->setWorkerNumber($workerNumber);
$server = new Server($adapter);
@ -137,7 +140,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
*/
go(function () use ($register, $containerId, &$statsDocument, $logError) {
try {
[$database, $returnDatabase] = getDatabase($register, '_project_console');
[$database, $returnDatabase] = getDatabase($register, '_console');
$document = new Document([
'$id' => $database->getId(),
'$collection' => 'realtime',
@ -190,7 +193,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
}
try {
[$database, $returnDatabase] = getDatabase($register, '_project_console');
[$database, $returnDatabase] = getDatabase($register, '_console');
$statsDocument
->setAttribute('timestamp', time())
@ -217,7 +220,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
*/
if ($realtime->hasSubscriber('console', 'role:member', 'project')) {
[$database, $returnDatabase] = getDatabase($register, '_project_console');
[$database, $returnDatabase] = getDatabase($register, '_console');
$payload = [];
@ -321,7 +324,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
return;
}
[$database, $returnDatabase] = getDatabase($register, '_project_' . $projectId);
[$database, $returnDatabase] = getDatabase($register, "_{$projectId}");
$user = $database->getDocument('users', $userId);
@ -397,7 +400,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
$cache = new Cache(new RedisCache($redis));
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace('_project_' . $project->getId());
$database->setNamespace("_{$project->getId()}");
/*
* Project Check
@ -504,7 +507,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
$cache = new Cache(new RedisCache($redis));
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace('_project_' . $realtime->connections[$connection]['projectId']);
$database->setNamespace("_{$realtime->connections[$connection]['projectId']}");
/*
* Abuse Check

8
app/tasks/migrate.php
View file

@ -46,7 +46,13 @@ $cli
$offset = 0;
$projects = [$console];
$count = 0;
$totalProjects = $consoleDB->count('projects') + 1;
try {
$totalProjects = $consoleDB->count('projects') + 1;
} catch (\Throwable $th) {
$consoleDB->setNamespace('_console');
$totalProjects = $consoleDB->count('projects') + 1;
}
$class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
$migration = new $class();

26
app/tasks/usage.php
View file

@ -259,7 +259,7 @@ $cli
$dbForConsole = new Database(new MariaDB($db), $cacheAdapter);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForConsole->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForConsole->setNamespace('_project_console');
$dbForConsole->setNamespace('_console');
$latestTime = [];
@ -325,7 +325,7 @@ $cli
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId !== 'console') {
$dbForProject->setNamespace('_project_' . $projectId);
$dbForProject->setNamespace('_' . $projectId);
$metricUpdated = $metric;
if (!empty($groupBy)) {
@ -417,8 +417,8 @@ $cli
$projectId = $project->getId();
// Get total storage
$dbForProject->setNamespace('_project_' . $projectId);
$storageTotal = (int) $dbForProject->sum('deployments', 'size');
$dbForProject->setNamespace('_' . $projectId);
$storageTotal = $dbForProject->sum('tags', 'size');
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
$id = \md5($time . '_30m_storage.deployments.total'); //Construct unique id for each metric using time, period and metric
@ -497,7 +497,7 @@ $cli
foreach ($collections as $collection => $options) {
try {
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$count = $dbForProject->count($collection);
$metricPrefix = $options['metricPrefix'] ?? '';
$metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count";
@ -553,7 +553,7 @@ $cli
$subCollectionTotals = []; //total project level sum of sub collections
do { // Loop over all the parent collection document for each sub collection
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$parents = $dbForProject->find($collection, [], 100, cursor: $latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections
if (empty($parents)) {
@ -564,12 +564,12 @@ $cli
foreach ($parents as $parent) {
foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$count = $dbForProject->count(($subOptions['collectionPrefix'] ?? '') . $parent->getId());
$subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
@ -618,12 +618,12 @@ $cli
continue;
}
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$total = (int) $dbForProject->sum(($subOptions['collectionPrefix'] ?? '') . $parent->getId(), $sum['field']);
$subCollectionTotals[$subCollection] = ($ssubCollectionTotals[$subCollection] ?? 0) + $total; // Project level sum for sub collections like storage.total
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.total" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.total";
$time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes
@ -668,7 +668,7 @@ $cli
* Inserting project level counts for sub collections like database.documents.count
*/
foreach ($subCollectionCounts as $subCollection => $count) {
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count";
@ -717,7 +717,7 @@ $cli
* Inserting project level sums for sub collections like storage.total
*/
foreach ($subCollectionTotals as $subCollection => $count) {
$dbForProject->setNamespace("_project_{$projectId}");
$dbForProject->setNamespace("_{$projectId}");
$metric = empty($metricPrefix) ? "{$subCollection}.total" : "{$metricPrefix}.{$subCollection}.total";
@ -769,4 +769,4 @@ $cli
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
});
});

4
app/views/install/compose.phtml
View file

@ -66,6 +66,7 @@ services:
- influxdb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_LOCALE
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
@ -145,6 +146,7 @@ services:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
@ -155,6 +157,8 @@ services:
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-executor:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>

10
app/workers/database.php
View file

@ -86,7 +86,7 @@ class DatabaseV1 extends Worker
$project = $dbForConsole->getDocument('projects', $projectId);
try {
if(!$dbForProject->createAttribute('collection_' . $collectionId, $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) {
if(!$dbForProject->createAttribute('collection_' . $collection->getInternalId(), $key, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters)) {
throw new Exception('Failed to create Attribute');
}
$dbForProject->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'available'));
@ -135,7 +135,7 @@ class DatabaseV1 extends Worker
// - failed: attribute was never created
// - stuck: attribute was available but cannot be removed
try {
if($status !== 'failed' && !$dbForProject->deleteAttribute('collection_' . $collectionId, $key)) {
if($status !== 'failed' && !$dbForProject->deleteAttribute('collection_' . $collection->getInternalId(), $key)) {
throw new Exception('Failed to delete Attribute');
}
$dbForProject->deleteDocument('attributes', $attribute->getId());
@ -210,7 +210,7 @@ class DatabaseV1 extends Worker
}
$dbForProject->deleteCachedDocument('collections', $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collectionId);
$dbForProject->deleteCachedCollection('collection_' . $collection->getInternalId());
}
/**
@ -233,7 +233,7 @@ class DatabaseV1 extends Worker
$project = $dbForConsole->getDocument('projects', $projectId);
try {
if(!$dbForProject->createIndex('collection_' . $collectionId, $key, $type, $attributes, $lengths, $orders)) {
if(!$dbForProject->createIndex('collection_' . $collection->getInternalId(), $key, $type, $attributes, $lengths, $orders)) {
throw new Exception('Failed to create Index');
}
$dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available'));
@ -276,7 +276,7 @@ class DatabaseV1 extends Worker
$project = $dbForConsole->getDocument('projects', $projectId);
try {
if($status !== 'failed' && !$dbForProject->deleteIndex('collection_' . $collectionId, $key)) {
if($status !== 'failed' && !$dbForProject->deleteIndex('collection_' . $collection->getInternalId(), $key)) {
throw new Exception('Failed to delete index');
}
$dbForProject->deleteDocument('indexes', $index->getId());

16
app/workers/deletes.php
View file

@ -548,18 +548,18 @@ class DeletesV1 extends Worker
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_DEVICE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_DEVICE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_DEVICE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_DEVICE_S3_BUCKET', '');
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
$device = new S3(APP_STORAGE_UPLOADS . '/app-' . $projectId, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
break;
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DEVICE_DO_SPACES_BUCKET', '');
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
$device = new DOSpaces(APP_STORAGE_UPLOADS . '/app-' . $projectId, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
break;

2
composer.json
View file

@ -45,7 +45,7 @@
"utopia-php/cache": "0.4.*",
"utopia-php/cli": "0.12.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.14.*",
"utopia-php/database": "0.15.*",
"utopia-php/locale": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/preloader": "0.2.*",

23
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "deeea4a225c55fccc04729750dd8c5ec",
"content-hash": "318c53aaac3cdfbdb728acc5e59b8057",
"packages": [
{
"name": "adhocore/jwt",
@ -2129,16 +2129,16 @@
},
{
"name": "utopia-php/database",
"version": "0.14.1",
"version": "0.15.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "ecc143f2cfe16b23675407035c6b5375ba263285"
"reference": "eb4f61ec40d697acdfd574638ecd075e4f44b864"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/ecc143f2cfe16b23675407035c6b5375ba263285",
"reference": "ecc143f2cfe16b23675407035c6b5375ba263285",
"url": "https://api.github.com/repos/utopia-php/database/zipball/eb4f61ec40d697acdfd574638ecd075e4f44b864",
"reference": "eb4f61ec40d697acdfd574638ecd075e4f44b864",
"shasum": ""
},
"require": {
@ -2186,9 +2186,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.14.1"
"source": "https://github.com/utopia-php/database/tree/0.15.1"
},
"time": "2022-01-25T13:01:20+00:00"
"time": "2022-02-22T09:33:37+00:00"
},
{
"name": "utopia-php/domains",
@ -3075,7 +3075,7 @@
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator",
"reference": "36e459dfd97a5693746e733ca9947a02ce12b0dc"
"reference": "4a43aa70c2f0b243edfc689f618a85f7d0817287"
},
"require": {
"ext-curl": "*",
@ -3110,7 +3110,7 @@
}
],
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"time": "2022-02-27T08:06:11+00:00"
"time": "2022-02-22T10:51:55+00:00"
},
{
"name": "composer/pcre",
@ -3685,6 +3685,9 @@
"require": {
"php": "^7.1 || ^8.0"
},
"replace": {
"myclabs/deep-copy": "self.version"
},
"require-dev": {
"doctrine/collections": "^1.0",
"doctrine/common": "^2.6",
@ -6570,5 +6573,5 @@
"platform-overrides": {
"php": "8.0"
},
"plugin-api-version": "2.2.0"
"plugin-api-version": "2.1.0"
}

2
docker-compose.yml
View file

@ -88,6 +88,7 @@ services:
environment:
- _APP_ENV
- _APP_LOCALE
- _APP_WORKER_PER_CORE
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_IPS
@ -174,6 +175,7 @@ services:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST

27
src/Appwrite/Migration/Migration.php
View file

@ -38,6 +38,26 @@ abstract class Migration
'0.13.0' => 'V12',
];
/**
* @var array
*/
protected array $collections;
public function __construct()
{
$this->collections = array_merge([
'_metadata' => [
'$id' => '_metadata'
],
'audit' => [
'$id' => 'audit'
],
'abuse' => [
'$id' => 'abuse'
]
], Config::getParam('collections', []));
}
/**
* Set project for migration.
*
@ -51,7 +71,7 @@ abstract class Migration
{
$this->project = $project;
$this->projectDB = $projectDB;
$this->projectDB->setNamespace('_project_' . $this->project->getId());
$this->projectDB->setNamespace('_' . $this->project->getId());
$this->consoleDB = $consoleDB;
@ -67,10 +87,7 @@ abstract class Migration
{
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
/** @var array $collections */
$collections = Config::getParam('collections', []);
foreach ($collections as $collection) {
foreach ($this->collections as $collection) {
$sum = 0;
$nextDocument = null;
$collectionCount = $this->projectDB->count($collection['$id']);

170
src/Appwrite/Migration/Version/V12.php
View file

@ -4,17 +4,178 @@ namespace Appwrite\Migration\Version;
use Appwrite\Migration\Migration;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
class V12 extends Migration
{
/**
* @var \PDO $pdo
*/
private $pdo;
public function execute(): void
{
global $register;
Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
$this->pdo = $register->get('db');
Console::info('Migrating Project Schemas');
$this->migrateProjectSchema($this->project->getId());
/**
* Switch to migrated Console Project
*/
if ($this->project->getId() === 'console') {
$this->consoleDB->setNamespace('_console');
$this->projectDB->setNamespace('_console');
}
Console::info('Migrating Permissions');
$this->fixPermissions();
Console::info('Migrating Collections');
$this->fixCollections();
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
}
/**
* Migrate Project Tables.
*
* @param string $projectId
* @return void
* @throws \Exception
* @throws \PDOException
*/
private function migrateProjectSchema(string $projectId): void
{
/**
* Remove empty generated Console Project.
*/
if ($this->consoleDB->getNamespace() === '_project_console' && $projectId === 'console') {
$all = [];
foreach ($this->collections as $collection) {
$all[] = "_{$projectId}_{$collection['$id']}";
$all[] = "_{$projectId}_{$collection['$id']}_perms";
}
$this->pdo->prepare('DROP TABLE IF EXISTS ' . implode(', ', $all) . ';')->execute();
} elseif ($this->projectDB->getNamespace() === '_console') {
return;
}
/**
* Rename Database Tables.
*/
foreach ($this->collections as $collection) {
$id = $collection['$id'];
$this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_{$id} RENAME TO _{$projectId}_{$id}")->execute();
$this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_{$id}_perms (
`_id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`_type` VARCHAR(12) NOT NULL,
`_permission` VARCHAR(255) NOT NULL,
`_document` VARCHAR(255) NOT NULL,
PRIMARY KEY (`_id`),
UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`),
INDEX `_index2` (`_permission`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute();
}
}
/**
* Migrate all Collection Structure.
*
* @return void
*/
protected function fixCollections(): void
{
foreach ($this->collections as $collection) {
$id = $collection['$id'];
Console::log("- {$id}");
switch ($id) {
case 'sessions':
try {
$this->projectDB->renameAttribute($id, 'providerToken', 'providerAccessToken');
} catch (\Throwable $th) {
Console::warning("'providerAccessToken' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->createAttribute(collection: $id, id: 'providerRefreshToken', type: Database::VAR_STRING, size: 16384, signed: true, required: true, filters: ['encrypt']);
} catch (\Throwable $th) {
Console::warning("'providerRefreshToken' from {$id}: {$th->getMessage()}");
}
try {
$this->projectDB->createAttribute(collection: $id, id: 'providerAccessTokenExpiry', type: Database::VAR_INTEGER, size: 0, required: true);
} catch (\Throwable $th) {
Console::warning("'providerAccessTokenExpiry' from {$id}: {$th->getMessage()}");
}
break;
}
usleep(100000);
}
}
/**
* Migrate all Permission to new System with dedicated Table.
* @return void
* @throws \Exception
*/
protected function fixPermissions()
{
foreach ($this->collections as $collection) {
$id = $collection['$id'];
Console::log("- {$collection['$id']}");
$nextDocument = null;
do {
$documents = $this->projectDB->find($id, limit: $this->limit, cursor: $nextDocument);
$count = count($documents);
\Co\run(function (array $documents) {
foreach ($documents as $document) {
go(function (Document $document) {
$sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}` WHERE _uid = {$this->pdo->quote($document->getid())}";
$stmt = $this->pdo->prepare($sql);
$stmt->execute();
$permissions = $stmt->fetch();
$read = json_decode($permissions['_read'] ?? null) ?? [];
$write = json_decode($permissions['_write'] ?? null) ?? [];
$permissions = [];
foreach ($read as $permission) {
$permissions[] = "('read', '{$permission}', '{$document->getId()}')";
}
foreach ($write as $permission) {
$permissions[] = "('write', '{$permission}', '{$document->getId()}')";
}
if (!empty($permissions)) {
$queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions);
$stmtPermissions = $this->pdo->prepare($queryPermissions);
$stmtPermissions->execute();
}
}, $document);
}
}, $documents);
if ($count !== $this->limit) {
$nextDocument = null;
} else {
$nextDocument = end($documents);
}
} while (!is_null($nextDocument));
}
/**
* Timeout to give MariaDB some room to breath
*/
usleep(100000);
}
protected function fixDocument(Document $document)
{
switch ($document->getCollection()) {
@ -92,6 +253,15 @@ class V12 extends Migration
}
break;
case 'sessions':
$document
->setAttribute('providerRefreshToken', '')
->setAttribute('providerAccessTokenExpiry', 0)
->setAttribute('providerAccessToken', $document->getAttribute('providerToken', ''))
->removeAttribute('providerToken');
break;
}
return $document;

4
src/Appwrite/Resque/Worker.php
View file

@ -184,10 +184,10 @@ abstract class Worker
if (!$projectId) {
throw new \Exception('ProjectID not provided - cannot get database');
}
$namespace = "_project_{$projectId}";
$namespace = "_{$projectId}";
break;
case self::DATABASE_CONSOLE:
$namespace = "_project_console";
$namespace = "_console";
$sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created
break;
default:

17
tests/e2e/Services/Users/UsersBase.php
View file

@ -149,7 +149,21 @@ trait UsersBase
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'search' => 'manchester-united.co.uk'
'search' => 'united.co.uk'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertIsArray($response['body']);
$this->assertIsArray($response['body']['users']);
$this->assertIsInt($response['body']['sum']);
$this->assertEquals(1, $response['body']['sum']);
$this->assertCount(1, $response['body']['users']);
$response = $this->client->call(Client::METHOD_GET, '/users', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'search' => 'man'
]);
$this->assertEquals($response['headers']['status-code'], 200);
@ -165,6 +179,7 @@ trait UsersBase
], $this->getHeaders()), [
'search' => $data['userId']
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['users']);