CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-05 14:40:42 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6940 from appwrite/remove-obsolete-tasks

Browse source 
chore: cleanup obsolete CLI tasks
This commit is contained in:
Christy Jacob 2023-10-18 14:59:41 -04:00 committed by GitHub
parent 8a52dc3485 688f2e5a56
commit eed70454ce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 50 additions and 974 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/controllers/api/databases.php
View file

@ -2264,7 +2264,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->desc('Delete attribute')
->groups(['api', 'database', 'schema'])
->label('scope', 'collections.write')
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete')
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
@ -2642,7 +2642,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->desc('Delete index')
->groups(['api', 'database'])
->label('scope', 'collections.write')
->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete')
->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update')
->label('audits.event', 'index.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')

1
app/views/install/compose.phtml
View file

@ -188,7 +188,6 @@ services:
- traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`)
- traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime
- traefik.http.routers.appwrite_realtime_wss.tls=true
- traefik.http.routers.appwrite_realtime_wss.tls.certresolver=dns
networks:
- appwrite
depends_on:

1
docker-compose.yml
View file

@ -211,7 +211,6 @@ services:
- traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`)
- traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime
- traefik.http.routers.appwrite_realtime_wss.tls=true
- traefik.http.routers.appwrite_realtime_wss.tls.certresolver=dns
networks:
- appwrite
volumes:

8
phpunit.xml
View file

@ -19,9 +19,11 @@
<file>./tests/e2e/Client.php</file>
<directory>./tests/e2e/General</directory>
<directory>./tests/e2e/Scopes</directory>
<directory>./tests/e2e/Services/Account</directory>
<directory>./tests/e2e/Services/Console</directory>
<directory>./tests/e2e/Services/Teams</directory>
<directory>./tests/e2e/Services/Realtime</directory>
<directory>./tests/e2e/Services/Account</directory>
<directory>./tests/e2e/Services/Users</directory>
<directory>./tests/e2e/Services/Console</directory>
<directory>./tests/e2e/Services/Avatars</directory>
<directory>./tests/e2e/Services/Databases</directory>
<directory>./tests/e2e/Services/GraphQL</directory>
@ -29,8 +31,6 @@
<directory>./tests/e2e/Services/Locale</directory>
<directory>./tests/e2e/Services/Projects</directory>
<directory>./tests/e2e/Services/Storage</directory>
<directory>./tests/e2e/Services/Teams</directory>
<directory>./tests/e2e/Services/Users</directory>
<directory>./tests/e2e/Services/Webhooks</directory>
<file>./tests/e2e/Services/Functions/FunctionsBase.php</file>
<file>./tests/e2e/Services/Functions/FunctionsCustomServerTest.php</file>

10
src/Appwrite/Platform/Services/Tasks.php
View file

@ -8,20 +8,15 @@ use Appwrite\Platform\Tasks\Install;
use Appwrite\Platform\Tasks\Maintenance;
use Appwrite\Platform\Tasks\Migrate;
use Appwrite\Platform\Tasks\Schedule;
use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\PatchDeleteScheduleUpdatedAtAttribute;
use Appwrite\Platform\Tasks\ClearCardCache;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcUsersStats;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\PatchDeleteProjectCollections;
use Appwrite\Platform\Tasks\Upgrade;
class Tasks extends Service
@ -39,17 +34,12 @@ class Tasks extends Service
->addAction(Install::getName(), new Install())
->addAction(Upgrade::getName(), new Upgrade())
->addAction(Maintenance::getName(), new Maintenance())
->addAction(PatchCreateMissingSchedules::getName(), new PatchCreateMissingSchedules())
->addAction(ClearCardCache::getName(), new ClearCardCache())
->addAction(PatchDeleteScheduleUpdatedAtAttribute::getName(), new PatchDeleteScheduleUpdatedAtAttribute())
->addAction(Schedule::getName(), new Schedule())
->addAction(Migrate::getName(), new Migrate())
->addAction(SDKs::getName(), new SDKs())
->addAction(VolumeSync::getName(), new VolumeSync())
->addAction(Specs::getName(), new Specs())
->addAction(CalcUsersStats::getName(), new CalcUsersStats())
->addAction(CalcTierStats::getName(), new CalcTierStats())
->addAction(PatchDeleteProjectCollections::getName(), new PatchDeleteProjectCollections())
;
}
}

176
src/Appwrite/Platform/Tasks/CalcUsersStats.php
View file

@ -1,176 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class CalcUsersStats extends Action
{
private array $columns = [
'Project ID',
'Project Name',
'Team ID',
'Team name',
'Users'
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'calc-users-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite calc-users-stats
Console::title('Cloud Users calculation V1');
Console::success(APP_NAME . ' cloud Users calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/users_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Project Name */
$stats['Project Name'] = $project->getAttribute('name');
/** Get Team Name and Id */
$teamId = $project->getAttribute('teamId', null);
$teamName = null;
if ($teamId) {
$team = $dbForConsole->getDocument('teams', $teamId);
$teamName = $team->getAttribute('name');
}
$stats['Team ID'] = $teamId;
$stats['Team name'] = $teamName;
/** Get Total Users */
$stats['users'] = $dbForProject->count('users', []);
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}

62
src/Appwrite/Platform/Tasks/ClearCardCache.php
View file

@ -1,62 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
class ClearCardCache extends Action
{
public static function getName(): string
{
return 'clear-card-cache';
}
public function __construct()
{
$this
->desc('Deletes card cache for specific user')
->param('userId', '', new UID(), 'User UID.', false)
->inject('dbForConsole')
->callback(fn (string $userId, Database $dbForConsole) => $this->action($userId, $dbForConsole));
}
public function action(string $userId, Database $dbForConsole): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('ClearCardCache V1');
Console::success(APP_NAME . ' ClearCardCache v1 has started');
$resources = ['card/' . $userId, 'card-back/' . $userId, 'card-og/' . $userId];
$caches = Authorization::skip(fn () => $dbForConsole->find('cache', [
Query::equal('resource', $resources),
Query::limit(100)
]));
$count = \count($caches);
Console::info("Going to delete {$count} cache records in 10 seconds...");
\sleep(10);
foreach ($caches as $cache) {
$key = $cache->getId();
$cacheFolder = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-console')
);
$cacheFolder->purge($key);
Authorization::skip(fn () => $dbForConsole->deleteDocument('cache', $cache->getId()));
}
Console::success(APP_NAME . ' ClearCardCache v1 has finished');
}
}

97
src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php
View file

@ -1,97 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
class PatchCreateMissingSchedules extends Action
{
public static function getName(): string
{
return 'patch-create-missing-schedules';
}
public function __construct()
{
$this
->desc('Ensure every function has a schedule')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchCreateMissingSchedules V1');
Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
$functionCursor = null;
while (true) {
$functionsQueries = [Query::limit($limit)];
if ($functionCursor !== null) {
$functionsQueries[] = Query::cursorAfter($functionCursor);
}
$functions = $dbForProject->find('functions', $functionsQueries);
if (count($functions) === 0) {
break;
}
foreach ($functions as $function) {
$scheduleId = $function->getAttribute('scheduleId');
$schedule = $dbForConsole->getDocument('schedules', $scheduleId);
if ($schedule->isEmpty()) {
$functionId = $function->getId();
$schedule = $dbForConsole->createDocument('schedules', new Document([
'$id' => ID::custom($scheduleId),
'region' => $project->getAttribute('region', 'default'),
'resourceType' => 'function',
'resourceId' => $functionId,
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')),
]));
Console::success('Recreated schedule for function ' . $functionId);
}
}
$functionCursor = $functions[array_key_last($functions)];
}
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}

129
src/Appwrite/Platform/Tasks/PatchDeleteProjectCollections.php
View file

@ -1,129 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Pools\Group;
use Utopia\Validator\Numeric;
class PatchDeleteProjectCollections extends Action
{
private array $names = [
'webhooks',
'platforms',
'schedules',
'projects',
'domains',
'certificates',
'keys',
'realtime',
];
public static function getName(): string
{
return 'patch-delete-project-collections';
}
public function __construct()
{
$this
->desc('Delete unnecessary project collections')
->param('offset', 0, new Numeric(), 'Resume deletion from param pos', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(function (int $offset, Group $pools, Cache $cache, Database $dbForConsole) {
$this->action($offset, $pools, $cache, $dbForConsole);
});
}
public function action(int $offset, Group $pools, Cache $cache, Database $dbForConsole): void
{
//docker compose exec -t appwrite patch-delete-project-collections
Console::title('Delete project collections V1');
Console::success(APP_NAME . ' delete project collections has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 50;
$sum = 50;
$offset = $offset;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Deleting collections for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace('_' . $project->getInternalId());
foreach ($this->names as $name) {
if (empty($name)) {
continue;
}
if ($dbForProject->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
if ($dbForProject->deleteCollection($name)) {
Console::log('Deleted ' . $name);
} else {
Console::error('Failed to delete ' . $name);
}
}
}
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
if (!empty($projects)) {
Console::log('Querying..... offset=' . $offset . ' , limit=' . $limit . ', count=' . $count);
}
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
}
}

74
src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php
View file

@ -1,74 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Pools\Group;
class PatchDeleteScheduleUpdatedAtAttribute extends Action
{
public static function getName(): string
{
return 'patch-delete-schedule-updated-at-attribute';
}
public function __construct()
{
$this
->desc('Ensure function collections do not have scheduleUpdatedAt attribute')
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchDeleteScheduleUpdatedAtAttribute V1');
Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
try {
/**
* Delete 'scheduleUpdatedAt' attribute
*/
$dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt');
$dbForProject->deleteCachedCollection('functions');
Console::success("'scheduleUpdatedAt' deleted.");
} catch (\Throwable $th) {
Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}");
}
$pools->reclaim();
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}

1
src/Appwrite/Platform/Workers/Databases.php
View file

@ -466,6 +466,7 @@ class Databases extends Action
throw new DatabaseException('Failed to delete index');
}
$dbForProject->deleteDocument('indexes', $index->getId());
$index->setAttribute('status', 'deleted');
} catch (\Exception $e) {
Console::error($e->getMessage());

409
src/Appwrite/Resque/Worker.php
View file

@ -1,409 +0,0 @@
<?php
namespace Appwrite\Resque;
use Appwrite\Event\Usage;
use Exception;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Cache\Adapter\Sharding;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Pools\Group;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Database\Validator\Authorization;
use Utopia\DSN\DSN;
use Utopia\Storage\Device\Wasabi;
use Utopia\Storage\Storage;
abstract class Worker
{
/**
* Callbacks that will be executed when an error occurs
*
* @var array
*/
protected static array $errorCallbacks = [];
/**
* Associative array holding all information passed into the worker
*
* @return array
*/
public array $args = [];
/**
* Function for identifying the worker needs to be set to unique name
*
* @return string
* @throws Exception
*/
public function getName(): string
{
throw new Exception("Please implement getName method in worker");
}
/**
* Function executed before running first task.
* Can include any preparations, such as connecting to external services or loading files
*
* @return void
* @throws \Exception|\Throwable
*/
public function init(): void
{
throw new Exception("Please implement init method in worker");
}
/**
* Function executed when new task requests is received.
* You can access $args here, it will contain event information
*
* @return void
* @throws \Exception|\Throwable
*/
public function run(): void
{
throw new Exception("Please implement run method in worker");
}
/**
* Function executed just before shutting down the worker.
* You can do cleanup here, such as disconnecting from services or removing temp files
*
* @return void
* @throws \Exception|\Throwable
*/
public function shutdown(): void
{
throw new Exception("Please implement shutdown method in worker");
}
public const DATABASE_PROJECT = 'project';
public const DATABASE_CONSOLE = 'console';
/**
* A wrapper around 'init' function with non-worker-specific code
*
* @return void
* @throws \Exception|\Throwable
*/
public function setUp(): void
{
try {
$this->init();
} catch (\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
$errorCallback($error, "init", $this->getName());
}
throw $error;
}
}
/**
* A wrapper around 'run' function with non-worker-specific code
*
* @return void
* @throws \Exception|\Throwable
*/
public function perform(): void
{
try {
/**
* Disabling global authorization in workers.
*/
Authorization::disable();
Authorization::setDefaultStatus(false);
$this->run();
} catch (\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
$errorCallback($error, "run", $this->getName(), $this->args);
}
throw $error;
}
}
/**
* A wrapper around 'shutdown' function with non-worker-specific code
*
* @return void
* @throws \Exception|\Throwable
*/
public function tearDown(): void
{
global $register;
try {
$pools = $register->get('pools'); /** @var Group $pools */
$pools->reclaim();
$this->shutdown();
} catch (\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
$errorCallback($error, "shutdown", $this->getName());
}
throw $error;
}
}
/**
* Register callback. Will be executed when error occurs.
* @param callable $callback
* @return void
*/
public static function error(callable $callback): void
{
self::$errorCallbacks[] = $callback;
}
/**
* Get internal project database
* @param Document $project
* @return Database
* @throws Exception
*/
protected static $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
protected function getProjectDB(Document $project): Database
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
if ($project->isEmpty() || $project->getId() === 'console') {
return $this->getConsoleDB();
}
$databaseName = $project->getAttribute('database');
if (isset(self::$databases[$databaseName])) {
$database = self::$databases[$databaseName];
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, $this->getCache());
self::$databases[$databaseName] = $database;
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
/**
* Get console database
* @return Database
* @throws Exception
*/
protected function getConsoleDB(): Database
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$databaseName = 'console';
if (isset(self::$databases[$databaseName])) {
$database = self::$databases[$databaseName];
$database->setNamespace('_console');
return $database;
}
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, $this->getCache());
self::$databases[$databaseName] = $database;
$database->setNamespace('_console');
return $database;
}
/**
* Get Cache
* @return Cache
*/
protected function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}
/**
* Get usage queue
* @return Usage
* @throws Exception
*/
protected function getUsageQueue(): Usage
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$queue = $pools
->get('queue')
->pop()
->getResource();
return new Usage($queue);
}
/**
* Get Functions Storage Device
* @param string $projectId of the project
* @return Device
*/
protected function getFunctionsDevice(string $projectId): Device
{
return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
}
/**
* Get Files Storage Device
* @param string $projectId of the project
* @return Device
*/
protected function getFilesDevice(string $projectId): Device
{
return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
}
/**
* Get Builds Storage Device
* @param string $projectId of the project
* @return Device
*/
protected function getBuildsDevice(string $projectId): Device
{
return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
}
protected function getCacheDevice(string $projectId): Device
{
return $this->getDevice(APP_STORAGE_CACHE . '/app-' . $projectId);
}
/**
* Get Device based on selected storage environment
* @param string $root path of the device
* @return Device
*/
public function getDevice(string $root): Device
{
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
if (!empty($connection)) {
$acl = 'private';
$device = Storage::DEVICE_LOCAL;
$accessKey = '';
$accessSecret = '';
$bucket = '';
$region = '';
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
$accessKey = $dsn->getUser() ?? '';
$accessSecret = $dsn->getPassword() ?? '';
$bucket = $dsn->getPath() ?? '';
$region = $dsn->getParam('region');
} catch (\Exception $e) {
Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
}
switch ($device) {
case Storage::DEVICE_S3:
return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE::DEVICE_DO_SPACES:
return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LINODE:
return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_WASABI:
return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LOCAL:
default:
return new Local($root);
}
} else {
switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) {
case Storage::DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
}
}

2
tests/e2e/Services/Account/AccountCustomClientTest.php
View file

@ -1011,7 +1011,7 @@ class AccountCustomClientTest extends Scope
$smsRequest = $this->getLastRequest();
return \array_merge($data, [
'token' => $smsRequest['data']['message']
'token' => $smsRequest['data']['secret']
]);
}

42
tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php
View file

@ -346,15 +346,32 @@ class RealtimeConsoleClientTest extends Scope
/**
* Test Delete Index
*/
$attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $actorsId . '/indexes/key_name', array_merge([
$indexKey = 'key_name';
$attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $actorsId . '/indexes/' . $indexKey, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals($attribute['headers']['status-code'], 204);
$indexKey = 'key_name';
$response = json_decode($client->receive(), true);
$this->assertArrayHasKey('type', $response);
$this->assertArrayHasKey('data', $response);
$this->assertEquals('event', $response['type']);
$this->assertNotEmpty($response['data']);
$this->assertArrayHasKey('timestamp', $response['data']);
$this->assertCount(1, $response['data']['channels']);
$this->assertContains('console', $response['data']['channels']);
$this->assertContains("databases.{$databaseId}.collections.{$actorsId}.indexes.*.update", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.{$actorsId}.indexes.*", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.{$actorsId}", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.*.indexes.*.update", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.*.indexes.*", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.*", $response['data']['events']);
$this->assertNotEmpty($response['data']['payload']);
/** Delete index generates two events. One from the API and one from the database worker */
$response = json_decode($client->receive(), true);
$this->assertArrayHasKey('type', $response);
$this->assertArrayHasKey('data', $response);
$this->assertEquals('event', $response['type']);
@ -402,13 +419,30 @@ class RealtimeConsoleClientTest extends Scope
/**
* Test Delete Attribute
*/
$attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $data['actorsId'] . '/attributes/name', array_merge([
$attributeKey = 'name';
$attribute = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $data['actorsId'] . '/attributes/' . $attributeKey, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
$this->assertEquals($attribute['headers']['status-code'], 204);
$attributeKey = 'name';
$response = json_decode($client->receive(), true);
$this->assertArrayHasKey('type', $response);
$this->assertArrayHasKey('data', $response);
$this->assertEquals('event', $response['type']);
$this->assertNotEmpty($response['data']);
$this->assertArrayHasKey('timestamp', $response['data']);
$this->assertCount(1, $response['data']['channels']);
$this->assertContains('console', $response['data']['channels']);
$this->assertContains("databases.{$databaseId}.collections.{$actorsId}.attributes.*.update", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.{$actorsId}.attributes.*", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.{$actorsId}", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.*.attributes.*.update", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.*.attributes.*", $response['data']['events']);
$this->assertContains("databases.{$databaseId}.collections.*", $response['data']['events']);
$this->assertNotEmpty($response['data']['payload']);
$response = json_decode($client->receive(), true);
$this->assertArrayHasKey('type', $response);

4
tests/e2e/Services/Webhooks/WebhooksBase.php
View file

@ -167,10 +167,10 @@ trait WebhooksBase
$this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io');
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*.attributes.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*.attributes.*.delete', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*.attributes.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.attributes.*", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.attributes.*.delete", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.attributes.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);

4
tests/e2e/Services/Webhooks/WebhooksCustomServerTest.php
View file

@ -124,10 +124,10 @@ class WebhooksCustomServerTest extends Scope
$this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io');
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*.indexes.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*.indexes.*.delete', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('databases.' . $databaseId . '.collections.*.indexes.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.indexes.*", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.indexes.*.delete", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("databases.{$databaseId}.collections.{$actorsId}.indexes.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);