CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-09-20 11:37:45 +12:00
Code Issues Projects Releases Wiki Activity

connection storage implementation

Browse source
This commit is contained in:
shimon 2024-08-07 19:29:36 +03:00
parent 5276024077
commit f1a518c018
2 changed files with 214 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

160
app/init.php
View file

@ -5,7 +5,9 @@
*
* Initializes both Appwrite API entry point, queue workers, and CLI tasks.
* Set configuration, framework resources & app constants
*
*/
if (\file_exists(__DIR__ . '/../vendor/autoload.php')) {
require_once __DIR__ . '/../vendor/autoload.php';
}
@ -343,7 +345,8 @@ Database::addFilter(
if (isset($formatOptions['min']) || isset($formatOptions['max'])) {
$attribute
->setAttribute('min', $formatOptions['min'])
->setAttribute('max', $formatOptions['max']);
->setAttribute('max', $formatOptions['max'])
;
}
return $value;
@ -352,7 +355,9 @@ Database::addFilter(
Database::addFilter(
'subQueryAttributes',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
$attributes = $database->find('attributes', [
Query::equal('collectionInternalId', [$document->getInternalId()]),
@ -376,7 +381,9 @@ Database::addFilter(
Database::addFilter(
'subQueryIndexes',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return $database
->find('indexes', [
@ -389,7 +396,9 @@ Database::addFilter(
Database::addFilter(
'subQueryPlatforms',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return $database
->find('platforms', [
@ -401,7 +410,9 @@ Database::addFilter(
Database::addFilter(
'subQueryKeys',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return $database
->find('keys', [
@ -413,7 +424,9 @@ Database::addFilter(
Database::addFilter(
'subQueryWebhooks',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return $database
->find('webhooks', [
@ -425,7 +438,9 @@ Database::addFilter(
Database::addFilter(
'subQuerySessions',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return Authorization::skip(fn () => $database->find('sessions', [
Query::equal('userInternalId', [$document->getInternalId()]),
@ -436,7 +451,9 @@ Database::addFilter(
Database::addFilter(
'subQueryTokens',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return Authorization::skip(fn () => $database
->find('tokens', [
@ -448,7 +465,9 @@ Database::addFilter(
Database::addFilter(
'subQueryChallenges',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return Authorization::skip(fn () => $database
->find('challenges', [
@ -460,7 +479,9 @@ Database::addFilter(
Database::addFilter(
'subQueryAuthenticators',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return Authorization::skip(fn () => $database
->find('authenticators', [
@ -472,7 +493,9 @@ Database::addFilter(
Database::addFilter(
'subQueryMemberships',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return Authorization::skip(fn () => $database
->find('memberships', [
@ -484,7 +507,9 @@ Database::addFilter(
Database::addFilter(
'subQueryVariables',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return $database
->find('variables', [
@ -523,12 +548,14 @@ Database::addFilter(
Database::addFilter(
'subQueryProjectVariables',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return $database
->find('variables', [
Query::equal('resourceType', ['project']),
Query::limit(APP_LIMIT_SUBQUERY),
Query::limit(APP_LIMIT_SUBQUERY)
]);
}
);
@ -540,7 +567,7 @@ Database::addFilter(
$user->getId(),
$user->getAttribute('email', ''),
$user->getAttribute('name', ''),
$user->getAttribute('phone', ''),
$user->getAttribute('phone', '')
];
foreach ($user->getAttribute('labels', []) as $label) {
@ -558,33 +585,36 @@ Database::addFilter(
Database::addFilter(
'subQueryTargets',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
return Authorization::skip(fn () => $database
->find('targets', [
Query::equal('userInternalId', [$document->getInternalId()]),
Query::limit(APP_LIMIT_SUBQUERY),
Query::limit(APP_LIMIT_SUBQUERY)
]));
}
);
Database::addFilter(
'subQueryTopicTargets',
function (mixed $value) {},
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
$targetIds = Authorization::skip(fn () => \array_map(
fn ($document) => $document->getAttribute('targetInternalId'),
$database->find('subscribers', [
Query::equal('topicInternalId', [$document->getInternalId()]),
Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY),
Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY)
])
));
if (\count($targetIds) > 0) {
return $database->skipValidation(fn () => $database->find('targets', [
Query::equal('$internalId', $targetIds),
Query::equal('$internalId', $targetIds)
]));
}
return [];
}
);
@ -596,7 +626,7 @@ Database::addFilter(
$provider->getId(),
$provider->getAttribute('name', ''),
$provider->getAttribute('provider', ''),
$provider->getAttribute('type', ''),
$provider->getAttribute('type', '')
];
$search = \implode(' ', \array_filter($searchValues));
@ -668,7 +698,6 @@ Structure::addFormat(APP_DATABASE_ATTRIBUTE_DATETIME, function () {
Structure::addFormat(APP_DATABASE_ATTRIBUTE_ENUM, function ($attribute) {
$elements = $attribute['formatOptions']['elements'];
return new WhiteList($elements, true);
}, Database::VAR_STRING);
@ -683,14 +712,12 @@ Structure::addFormat(APP_DATABASE_ATTRIBUTE_URL, function () {
Structure::addFormat(APP_DATABASE_ATTRIBUTE_INT_RANGE, function ($attribute) {
$min = $attribute['formatOptions']['min'] ?? -INF;
$max = $attribute['formatOptions']['max'] ?? INF;
return new Range($min, $max, Range::TYPE_INTEGER);
}, Database::VAR_INTEGER);
Structure::addFormat(APP_DATABASE_ATTRIBUTE_FLOAT_RANGE, function ($attribute) {
$min = $attribute['formatOptions']['min'] ?? -INF;
$max = $attribute['formatOptions']['max'] ?? INF;
return new Range($min, $max, Range::TYPE_FLOAT);
}, Database::VAR_FLOAT);
@ -707,12 +734,12 @@ $register->set('logger', function () {
}
if (!Logger::hasProvider($providerName)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Logging provider not supported. Logging is disabled');
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Logging provider not supported. Logging is disabled");
}
// Old Sentry Format conversion. Fallback until the old syntax is completely deprecated.
if (str_contains($providerConfig, ';') && strtolower($providerName) == 'sentry') {
$configChunks = \explode(';', $providerConfig);
$configChunks = \explode(";", $providerConfig);
$sentryKey = $configChunks[0];
$projectId = $configChunks[1];
@ -722,7 +749,6 @@ $register->set('logger', function () {
$classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
$adapter = new $classname($providerConfig);
return new Logger($adapter);
});
$register->set('pools', function () {
@ -819,7 +845,7 @@ $register->set('pools', function () {
$dsnDatabase = $dsn->getPath();
if (!in_array($dsnScheme, $schemes)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid console database scheme');
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme");
}
/**
@ -833,13 +859,13 @@ $register->set('pools', function () {
'mysql',
'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, [
return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true,
]);
PDO::ATTR_STRINGIFY_FETCHES => true
));
});
},
'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) {
@ -884,7 +910,7 @@ $register->set('pools', function () {
break;
default:
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Server error: Missing adapter implementation.');
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation.");
}
return $adapter;
@ -950,7 +976,6 @@ $register->set('passwordsDictionary', function () {
$content = \file_get_contents(__DIR__ . '/assets/security/10k-common-passwords');
$content = explode("\n", $content);
$content = array_flip($content);
return $content;
});
$register->set('promiseAdapter', function () {
@ -1106,11 +1131,12 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
/** @var Utopia\Database\Database $dbForProject */
/** @var Utopia\Database\Database $dbForConsole */
/** @var string $mode */
Authorization::setDefaultStatus(true);
Auth::setCookieName('a_session_' . $project->getId());
if ($mode === APP_MODE_ADMIN) {
if (APP_MODE_ADMIN === $mode) {
Auth::setCookieName('a_session_' . $console->getId());
}
@ -1147,7 +1173,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
Auth::$unique = $session['id'] ?? '';
Auth::$secret = $session['secret'] ?? '';
if ($mode !== APP_MODE_ADMIN) {
if (APP_MODE_ADMIN !== $mode) {
if ($project->isEmpty()) {
$user = new Document([]);
} else {
@ -1168,7 +1194,7 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
$user = new Document([]);
}
if ($mode === APP_MODE_ADMIN) {
if (APP_MODE_ADMIN === $mode) {
if ($user->find('teamInternalId', $project->getAttribute('teamInternalId'), 'memberships')) {
Authorization::setDefaultStatus(false); // Cancel security segmentation for admin users.
} else {
@ -1209,6 +1235,7 @@ App::setResource('project', function ($dbForConsole, $request, $console) {
/** @var Appwrite\Utopia\Request $request */
/** @var Utopia\Database\Database $dbForConsole */
/** @var Utopia\Database\Document $console */
$projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', ''));
if (empty($projectId) || $projectId === 'console') {
@ -1238,6 +1265,7 @@ App::setResource('session', function (Document $user) {
}
}
return;
}, ['user']);
App::setResource('console', function () {
@ -1275,7 +1303,7 @@ App::setResource('console', function () {
'oAuthProviders' => [
'githubEnabled' => true,
'githubSecret' => System::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''),
'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', ''),
'githubAppid' => System::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '')
],
]);
}, []);
@ -1365,12 +1393,6 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
->setMetadata('project', $project->getId())
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
var_dump([
'location' => 'Init::getProjectDB',
'_APP_DATABASE_SHARED_TABLES' => System::getEnv('_APP_DATABASE_SHARED_TABLES', ''),
'dsn' => $dsn,
]);
$sharedTablesKeys = explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (in_array($dsn->getHost(), $sharedTablesKeys)) {
$database
@ -1388,7 +1410,6 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
if (isset($databases[$dsn->getHost()])) {
$database = $databases[$dsn->getHost()];
$configure($database);
return $database;
}
@ -1413,7 +1434,8 @@ App::setResource('cache', function (Group $pools) {
$adapters[] = $pools
->get($value)
->pop()
->getResource();
->getResource()
;
}
return new Cache(new Sharding($adapters));
@ -1423,28 +1445,29 @@ App::setResource('deviceForLocal', function () {
return new Local();
});
App::setResource('deviceForFiles', function ($project, $connectionStorage) {
return getDevice(APP_STORAGE_UPLOADS.'/app-'.$project->getId(), $connectionStorage);
}, ['project', 'connectionStorage']);
App::setResource('deviceForFiles', function ($project, $connectionString) {
return getDevice(APP_STORAGE_UPLOADS.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
App::setResource('deviceForFunctions', function ($project, $connectionStorage) {
return getDevice(APP_STORAGE_FUNCTIONS.'/app-'.$project->getId(), $connectionStorage);
}, ['project', 'connectionStorage']);
App::setResource('deviceForFunctions', function ($project, $connectionString) {
return getDevice(APP_STORAGE_FUNCTIONS.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
App::setResource('deviceForBuilds', function ($project, $connectionStorage) {
return getDevice(APP_STORAGE_BUILDS.'/app-'.$project->getId(), $connectionStorage);
}, ['project', 'connectionStorage']);
App::setResource('deviceForBuilds', function ($project, $connectionString) {
return getDevice(APP_STORAGE_BUILDS.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
App::setResource('connectionStorage', function () {
return '';
App::setResource('connectionString', function () {
return System::getEnv('_APP_CONNECTIONS_STORAGE', '');
});
function getDevice(string $root, string $connectionStorage = ''): Device
function getDevice(string $root, string $connectionString = ''): Device
{
$connectionStorage = ! empty($connectionStorage) ? $connectionStorage : System::getEnv('_APP_CONNECTIONS_STORAGE', '');
var_dump('*****$connectionStorag****');
var_dump($connectionStorage);
if (! empty($connectionStorage)) {
var_dump('*****$connectionStorage****');
var_dump($connectionString);
if (! empty($connectionString)) {
$acl = 'private';
$device = Storage::DEVICE_LOCAL;
$accessKey = '';
@ -1453,7 +1476,7 @@ function getDevice(string $root, string $connectionStorage = ''): Device
$region = '';
try {
$dsn = new DSN($connectionStorage);
$dsn = new DSN($connectionString);
$device = $dsn->getScheme();
$accessKey = $dsn->getUser() ?? '';
$accessSecret = $dsn->getPassword() ?? '';
@ -1469,7 +1492,6 @@ function getDevice(string $root, string $connectionStorage = ''): Device
case STORAGE::DEVICE_DO_SPACES:
$device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
$device->setHttpVersion(S3::HTTP_VERSION_1_1);
return $device;
case Storage::DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
@ -1492,7 +1514,6 @@ function getDevice(string $root, string $connectionStorage = ''): Device
$s3Region = System::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = System::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
@ -1502,7 +1523,6 @@ function getDevice(string $root, string $connectionStorage = ''): Device
$doSpacesAcl = 'private';
$device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
$device->setHttpVersion(S3::HTTP_VERSION_1_1);
return $device;
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
@ -1510,7 +1530,6 @@ function getDevice(string $root, string $connectionStorage = ''): Device
$backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
@ -1518,7 +1537,6 @@ function getDevice(string $root, string $connectionStorage = ''): Device
$linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
@ -1526,7 +1544,6 @@ function getDevice(string $root, string $connectionStorage = ''): Device
$wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
@ -1553,6 +1570,7 @@ App::setResource('passwordsDictionary', function ($register) {
return $register->get('passwordsDictionary');
}, ['register']);
App::setResource('servers', function () {
$platforms = Config::getParam('platforms');
$server = $platforms[APP_PLATFORM_SERVER];
@ -1657,21 +1675,18 @@ App::setResource('schema', function ($utopia, $dbForProject) {
App::setResource('contributors', function () {
$path = 'app/config/contributors.json';
$list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : [];
return $list;
});
App::setResource('employees', function () {
$path = 'app/config/employees.json';
$list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : [];
return $list;
});
App::setResource('heroes', function () {
$path = 'app/config/heroes.json';
$list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : [];
return $list;
});
@ -1690,7 +1705,6 @@ App::setResource('requestTimestamp', function ($request) {
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value');
}
}
return $requestTimestamp;
}, ['request']);
App::setResource('plan', function (array $plan = []) {

24
app/worker.php
View file

@ -258,21 +258,21 @@ Server::setResource('pools', function (Registry $register) {
return $register->get('pools');
}, ['register']);
Server::setResource('deviceForFunctions', function (Document $project) {
return getDevice(APP_STORAGE_FUNCTIONS.'/app-'.$project->getId());
}, ['project']);
Server::setResource('deviceForFunctions', function (Document $project, $connectionString) {
return getDevice(APP_STORAGE_FUNCTIONS.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
Server::setResource('deviceForFiles', function (Document $project) {
return getDevice(APP_STORAGE_UPLOADS.'/app-'.$project->getId());
}, ['project']);
Server::setResource('deviceForFiles', function (Document $project, $connectionString) {
return getDevice(APP_STORAGE_UPLOADS.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
Server::setResource('deviceForBuilds', function (Document $project) {
return getDevice(APP_STORAGE_BUILDS.'/app-'.$project->getId());
}, ['project']);
Server::setResource('deviceForBuilds', function (Document $project, $connectionString) {
return getDevice(APP_STORAGE_BUILDS.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
Server::setResource('deviceForCache', function (Document $project) {
return getDevice(APP_STORAGE_CACHE.'/app-'.$project->getId());
}, ['project']);
Server::setResource('deviceForCache', function (Document $project, $connectionString) {
return getDevice(APP_STORAGE_CACHE.'/app-'.$project->getId(), $connectionString);
}, ['project', 'connectionString']);
$pools = $register->get('pools');
$platform = new Appwrite();