CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-07-02 05:00:33 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4330 from appwrite/feat-db-pools-eldad

Browse source 
Feat db pools eldad
This commit is contained in:
Christy Jacob 2022-10-25 01:27:07 +05:30 committed by GitHub
parent 584ec3160a 27c4e24fa5
commit 6f00c98805
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 1210 additions and 1126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.env
View file

@ -23,8 +23,11 @@ _APP_DB_SCHEMA=appwrite
_APP_DB_USER=user
_APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite
_APP_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
_APP_STORAGE_DEVICE=Local
_APP_STORAGE_S3_ACCESS_KEY=
_APP_STORAGE_S3_SECRET=

13
Dockerfile
View file

@ -35,6 +35,7 @@ ENV PHP_REDIS_VERSION=5.3.7 \
PHP_IMAGICK_VERSION=3.7.0 \
PHP_YAML_VERSION=2.2.2 \
PHP_MAXMINDDB_VERSION=v1.11.0 \
PHP_MEMCACHED_VERSION=v3.2.0 \
PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 "
RUN \
@ -52,6 +53,7 @@ RUN \
imagemagick \
imagemagick-dev \
libmaxminddb-dev \
libmemcached-dev \
zstd-dev
RUN docker-php-ext-install sockets
@ -125,6 +127,15 @@ RUN \
./configure && \
make && make install
# Memcached Extension
FROM compile as memcached
RUN \
git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \
cd php-memcached && \
phpize && \
./configure && \
make && make install
# Zstd Compression
FROM compile as zstd
RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \
@ -134,7 +145,6 @@ RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \
&& ./configure --with-libzstd \
&& make && make install
# Rust Extensions Compile Image
FROM php:8.0.18-cli as rust_compile
@ -304,6 +314,7 @@ COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imag
COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/

80
app/cli.php
View file

@ -6,7 +6,78 @@ require_once __DIR__ . '/controllers/general.php';
use Utopia\App;
use Utopia\CLI\CLI;
use Utopia\CLI\Console;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use InfluxDB\Database as InfluxDatabase;
function getInfluxDB(): InfluxDatabase
{
global $register;
$client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}
function getConsoleDB(): Database
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database->setNamespace('console');
return $database;
}
function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}
Authorization::disable();
@ -29,4 +100,13 @@ $cli
Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN'));
});
$cli
->error(function ($error) {
if (App::getEnv('_APP_ENV', 'development')) {
Console::error($error);
} else {
Console::error($error->getMessage());
}
});
$cli->run();

36
app/config/variables.php
View file

@ -306,24 +306,24 @@ return [
'question' => '',
'filter' => 'password'
],
[
'name' => '_APP_DB_PROJECT',
'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
'introduction' => 'TBD',
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
'required' => true,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_DB__APP_DB_CONSOLEROOT_PASS',
'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
'introduction' => 'TBD',
'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
'required' => true,
'question' => '',
'filter' => ''
]
// [
// 'name' => '_APP_CONNECTIONS_DB_PROJECT',
// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ],
// [
// 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ]
],
],
[

2
app/controllers/api/account.php
View file

@ -5,7 +5,6 @@ use Appwrite\Auth\Auth;
use Appwrite\Auth\Validator\Password;
use Appwrite\Auth\Validator\Phone;
use Appwrite\Detector\Detector;
use Appwrite\Event\Audit;
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Event\Phone as EventPhone;
@ -39,7 +38,6 @@ use Utopia\Database\Validator\UID;
use Utopia\Locale\Locale;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;

2
app/controllers/api/functions.php
View file

@ -25,7 +25,6 @@ use Appwrite\Task\Validator\Cron;
use Appwrite\Utopia\Database\Validator\Queries\Deployments;
use Appwrite\Utopia\Database\Validator\Queries\Executions;
use Appwrite\Utopia\Database\Validator\Queries\Functions;
use Appwrite\Utopia\Database\Validator\Queries\Variables;
use Utopia\App;
use Utopia\Database\Database;
use Utopia\Database\Document;
@ -33,7 +32,6 @@ use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Text;
use Utopia\Validator\Range;
use Utopia\Validator\WhiteList;

233
app/controllers/api/health.php
View file

@ -5,7 +5,9 @@ use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
@ -26,6 +28,7 @@ App::get('/v1/health')
->action(function (Response $response) {
$output = [
'name' => 'http',
'status' => 'pass',
'ping' => 0
];
@ -42,7 +45,6 @@ App::get('/v1/health/version')
->label('sdk.response.model', Response::MODEL_HEALTH_VERSION)
->inject('response')
->action(function (Response $response) {
$response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION);
});
@ -58,33 +60,50 @@ App::get('/v1/health/db')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('utopia')
->action(function (Response $response, App $utopia) {
->inject('pools')
->action(function (Response $response, Group $pools) {
$checkStart = \microtime(true);
$output = [];
try {
$dbPool = $utopia->getResource('dbPool');
$database = $dbPool->getConsoleDB();
/* @var $consoleDB PDO */
$consoleDB = $dbPool->getPDO($database);
// Run a small test to check the connection
$statement = $consoleDB->prepare("SELECT 1;");
$statement->closeCursor();
$statement->execute();
} catch (Exception $_e) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available');
}
$output = [
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
$configs = [
'Console.DB' => Config::getParam('pools-console'),
'Projects.DB' => Config::getParam('pools-database'),
];
$response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/cache')
@ -99,23 +118,163 @@ App::get('/v1/health/cache')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('utopia')
->action(function (Response $response, App $utopia) {
->inject('pools')
->action(function (Response $response, Group $pools) {
$checkStart = \microtime(true);
$output = [];
$redis = $utopia->getResource('cache');
if (!$redis->ping(true)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available');
}
$output = [
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
$configs = [
'Cache' => Config::getParam('pools-cache'),
];
$response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/queue')
->desc('Get Queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getQueue')
->label('sdk.description', '/docs/references/health/get-queue.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('pools')
->action(function (Response $response, Group $pools) {
$output = [];
$configs = [
'Queue' => Config::getParam('pools-queue'),
];
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/pubsub')
->desc('Get PubSub')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getPubSub')
->label('sdk.description', '/docs/references/health/get-pubsub.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('pools')
->action(function (Response $response, Group $pools) {
$output = [];
$configs = [
'PubSub' => Config::getParam('pools-pubsub'),
];
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/time')

21
app/controllers/api/projects.php
View file

@ -2,7 +2,6 @@
use Appwrite\Auth\Auth;
use Appwrite\Auth\Validator\Password;
use Appwrite\Database\DatabasePool;
use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Appwrite\Event\Validator\Event;
@ -29,10 +28,9 @@ use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
use Utopia\Registry\Registry;
use Appwrite\Extend\Exception;
use Utopia\Cache\Adapter\Redis;
use Utopia\Cache\Cache;
use Utopia\Database\Adapter\MariaDB;
use Appwrite\Utopia\Database\Validator\Queries\Projects;
use Utopia\Cache\Cache;
use Utopia\Pools\Group;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\Hostname;
@ -74,8 +72,9 @@ App::post('/v1/projects')
->inject('response')
->inject('dbForConsole')
->inject('cache')
->inject('dbPool')
->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, \Redis $cache, DatabasePool $dbPool) {
->inject('pools')
->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) {
$team = $dbForConsole->getDocument('teams', $teamId);
@ -90,13 +89,13 @@ App::post('/v1/projects')
}
$projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
$databases = Config::getParam('pools-database', []);
$database = $databases[array_rand($databases)];
if ($projectId === 'console') {
throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
}
$pdo = $dbPool->getAnyFromPool();
$project = $dbForConsole->createDocument('projects', new Document([
'$id' => $projectId,
'$permissions' => [
@ -127,10 +126,12 @@ App::post('/v1/projects')
'domains' => null,
'auths' => $auths,
'search' => implode(' ', [$projectId, $name]),
'database' => $pdo->getName()
'database' => $database,
]));
$dbForProject = DatabasePool::getDatabase($pdo->getConnection(), $cache, "_{$project->getInternalId()}");
$dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache);
$dbForProject->setNamespace("_{$project->getInternalId()}");
$dbForProject->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$audit = new Audit($dbForProject);

3
app/controllers/api/teams.php
View file

@ -20,7 +20,6 @@ use Appwrite\Utopia\Response;
use MaxMind\Db\Reader;
use Utopia\App;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
@ -36,9 +35,7 @@ use Utopia\Database\Validator\Key;
use Utopia\Database\Validator\UID;
use Utopia\Locale\Locale;
use Utopia\Validator\Text;
use Utopia\Validator\Range;
use Utopia\Validator\ArrayList;
use Utopia\Validator\WhiteList;
App::post('/v1/teams')
->desc('Create Team')

33
app/http.php
View file

@ -2,7 +2,6 @@
require_once __DIR__ . '/../vendor/autoload.php';
use Appwrite\Database\DatabasePool;
use Appwrite\Utopia\Response;
use Swoole\Process;
use Swoole\Http\Server;
@ -23,6 +22,7 @@ use Utopia\Swoole\Files;
use Appwrite\Utopia\Request;
use Utopia\Logger\Log;
use Utopia\Logger\Log\User;
use Utopia\Pools\Group;
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
@ -62,11 +62,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
go(function () use ($register, $app) {
$redis = $register->get('redisPool')->get();
App::setResource('cache', fn() => $redis);
$dbPool = $register->get('dbPool');
App::setResource('dbPool', fn() => $dbPool);
$pools = $register->get('pools'); /** @var Group $pools */
App::setResource('pools', fn() => $pools);
// wait for database to be ready
$attempts = 0;
@ -93,7 +90,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$collections = Config::getParam('collections', []);
try {
$redis->flushAll();
$cache = $app->getResource('cache'); /** @var Utopia\Cache\Cache $cache */
$cache->flush();
Console::success('[Setup] - Creating database: appwrite...');
$dbForConsole->create(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
} catch (\Exception $e) {
@ -217,7 +215,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
$dbPool->reset();
$pools->reclaim();
Console::success('[Setup] - Server database init completed...');
});
@ -250,11 +248,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$app = new App('UTC');
$redis = $register->get('redisPool')->get();
App::setResource('cache', fn() => $redis);
$dbPool = $register->get('dbPool');
App::setResource('dbPool', fn() => $dbPool);
$pools = $register->get('pools');
App::setResource('pools', fn() => $pools);
try {
Authorization::cleanRoles();
@ -321,13 +316,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
/**
* Reset Database connection if PDOException was thrown.
*/
if ($th instanceof PDOException) {
$db = null;
}
$swooleResponse->setStatusCode(500);
$output = ((App::isDevelopment())) ? [
@ -345,10 +333,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$swooleResponse->end(\json_encode($output));
} finally {
$dbPool->reset();
/** @var RedisPool $redisPool */
$redisPool = $register->get('redisPool');
$redisPool->put($redis);
$pools->reclaim();
}
});

314
app/init.php
View file

@ -18,8 +18,6 @@ ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Extend\Exception;
use Appwrite\Auth\Auth;
use Appwrite\SMS\Adapter\Mock;
@ -34,35 +32,28 @@ use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Event\Phone;
use Appwrite\Event\Delete;
use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\IP;
use Appwrite\Network\Validator\URL;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\View;
use Utopia\App;
use Utopia\Validator\Range;
use Utopia\Validator\WhiteList;
use Utopia\Database\ID;
use Utopia\Database\Document;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\DatetimeValidator;
use Utopia\Database\Validator\Structure;
use Utopia\Logger\Logger;
use Utopia\Config\Config;
use Utopia\Locale\Locale;
use Utopia\Registry\Registry;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Database\Document;
use Utopia\Database\Database;
use Appwrite\Database\DatabasePool;
use Appwrite\Event\Delete;
use Utopia\Database\Validator\Structure;
use Utopia\Database\Validator\Authorization;
use Utopia\Cache\Cache;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Validator\Range;
use Utopia\Validator\WhiteList;
use Swoole\Database\RedisConfig;
use Swoole\Database\RedisPool;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Query;
use Utopia\Database\Validator\DatetimeValidator;
use Utopia\Storage\Device;
use Utopia\Storage\Storage;
use Utopia\Storage\Device\Backblaze;
@ -71,6 +62,19 @@ use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Wasabi;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Adapter\MySQL;
use Utopia\Pools\Group;
use Utopia\Pools\Pool;
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
const APP_NAME = 'Appwrite';
const APP_DOMAIN = 'appwrite.io';
@ -493,53 +497,180 @@ $register->set('logger', function () {
$adapter = new $classname($providerConfig);
return new Logger($adapter);
});
$register->set('pools', function () {
$group = new Group();
$register->set('dbPool', function () {
/** Parse the console databases */
$consoleDB = App::getEnv('_APP_DB_CONSOLE', '');
$consoleDB = explode(',', $consoleDB)[0];
$consoleDB = explode('=', $consoleDB);
$name = $consoleDB[0];
$dsn = $consoleDB[1];
$consoleDBs[$name] = $dsn;
$fallbackForDB = AppwriteURL::unparse([
'scheme' => 'mariadb',
'host' => App::getEnv('_APP_DB_HOST', 'mariadb'),
'port' => App::getEnv('_APP_DB_PORT', '3306'),
'user' => App::getEnv('_APP_DB_USER', ''),
'pass' => App::getEnv('_APP_DB_PASS', ''),
]);
$fallbackForRedis = AppwriteURL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]);
/** Parse the project databases */
$projectDBs = [];
$projectDB = App::getEnv('_APP_DB_PROJECT', '');
$projectDB = explode(',', $projectDB);
foreach ($projectDB as $db) {
$db = explode('=', $db);
$name = $db[0];
$dsn = $db[1];
$projectDBs[$name] = $dsn;
$connections = [
'console' => [
'type' => 'database',
'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB),
'multiple' => false,
'schemes' => ['mariadb', 'mysql'],
],
'database' => [
'type' => 'database',
'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB),
'multiple' => true,
'schemes' => ['mariadb', 'mysql'],
],
'queue' => [
'type' => 'queue',
'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis),
'multiple' => false,
'schemes' => ['redis'],
],
'pubsub' => [
'type' => 'pubsub',
'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis),
'multiple' => false,
'schemes' => ['redis'],
],
'cache' => [
'type' => 'cache',
'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis),
'multiple' => true,
'schemes' => ['redis'],
],
];
foreach ($connections as $key => $connection) {
$type = $connection['type'] ?? '';
$dsns = $connection['dsns'] ?? '';
$multipe = $connection['multiple'] ?? false;
$schemes = $connection['schemes'] ?? [];
$config = [];
$dsns = explode(',', $connection['dsns'] ?? '');
foreach ($dsns as &$dsn) {
$dsn = explode('=', $dsn);
$name = ($multipe) ? $key . '_' . $dsn[0] : $key;
$dsn = $dsn[1] ?? '';
$config[] = $name;
if (empty($dsn)) {
//throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}");
continue;
}
$dsn = new DSN($dsn);
$dsnHost = $dsn->getHost();
$dsnPort = $dsn->getPort();
$dsnUser = $dsn->getUser();
$dsnPass = $dsn->getPassword();
$dsnScheme = $dsn->getScheme();
$dsnDatabase = $dsn->getDatabase();
if (!in_array($dsnScheme, $schemes)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme");
}
/**
* Get Resource
*
* Creation could be reused accross connection types like database, cache, queue, etc.
*
* Resource assignment to an adapter will happen below.
*/
switch ($dsnScheme) {
case 'mysql':
case 'mariadb':
$resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true
));
});
};
break;
case 'redis':
$resource = function () use ($dsnHost, $dsnPort, $dsnPass) {
$redis = new Redis();
@$redis->pconnect($dsnHost, (int)$dsnPort);
if ($dsnPass) {
$redis->auth($dsnPass);
}
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
};
break;
default:
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme");
break;
}
$pool = new Pool($name, 64, function () use ($type, $resource, $dsn) {
// Get Adapter
$adapter = null;
switch ($type) {
case 'database':
$adapter = match ($dsn->getScheme()) {
'mariadb' => new MariaDB($resource()),
'mysql' => new MySQL($resource()),
default => null
};
$adapter->setDefaultDatabase($dsn->getDatabase());
break;
case 'queue':
$adapter = $resource();
break;
case 'pubsub':
$adapter = $resource();
break;
case 'cache':
$adapter = match ($dsn->getScheme()) {
'redis' => new RedisCache($resource()),
default => null
};
break;
default:
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation.");
break;
}
return $adapter;
});
$group->add($pool);
}
Config::setParam('pools-' . $key, $config);
}
$pool = new DatabasePool($consoleDBs, $projectDBs);
return $pool;
});
$register->set('redisPool', function () {
$redisHost = App::getEnv('_APP_REDIS_HOST', '');
$redisPort = App::getEnv('_APP_REDIS_PORT', '');
$redisUser = App::getEnv('_APP_REDIS_USER', '');
$redisPass = App::getEnv('_APP_REDIS_PASS', '');
$redisAuth = '';
if ($redisUser && $redisPass) {
$redisAuth = $redisUser . ':' . $redisPass;
try {
$group->fill();
} catch (\Throwable $th) {
Console::error('Connection failure: ' . $th->getMessage());
}
$pool = new RedisPool(
(new RedisConfig())
->withHost($redisHost)
->withPort($redisPort)
->withAuth($redisAuth)
->withDbIndex(0),
64
);
return $pool;
return $group;
});
$register->set('influxdb', function () {
// Register DB connection
@ -556,7 +687,7 @@ $register->set('influxdb', function () {
return $client;
});
$register->set('statsd', function () {
// Register DB connection
// Register DB connection
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
@ -596,14 +727,6 @@ $register->set('smtp', function () {
$register->set('geodb', function () {
return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-06.mmdb');
});
$register->set('cache', function () {
// This is usually for our workers or CLI commands scope
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
});
/*
* Localization
@ -901,22 +1024,51 @@ App::setResource('console', function () {
]);
}, []);
App::setResource('dbForProject', function ($dbPool, $cache, Document $project) {
$database = $project->getAttribute('database', '');
if (empty($database)) {
$database = $dbPool->getConsoleDB();
App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) {
if ($project->isEmpty() || $project->getId() === 'console') {
return $dbForConsole;
}
$pdo = $dbPool->getPDOFromPool($database);
$database = DatabasePool::getDatabase($pdo->getConnection(), $cache, "_{$project->getInternalId()}");
return $database;
}, ['dbPool', 'cache', 'project']);
App::setResource('dbForConsole', function ($dbPool, $cache) {
$database = $dbPool->getConsoleDB();
$pdo = $dbPool->getPDOFromPool($database);
$database = DatabasePool::getDatabase($pdo->getConnection(), $cache, '_console');
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, $cache);
$database->setNamespace('_' . $project->getInternalId());
return $database;
}, ['dbPool', 'cache']);
}, ['pools', 'dbForConsole', 'cache', 'project']);
App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, $cache);
$database->setNamespace('console');
return $database;
}, ['pools', 'cache']);
App::setResource('cache', function (Group $pools) {
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}, ['pools']);
App::setResource('deviceLocal', function () {
return new Local();

1
app/preload.php
View file

@ -35,6 +35,7 @@ foreach (
realpath(__DIR__ . '/../vendor/symfony'),
realpath(__DIR__ . '/../vendor/mongodb'),
realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload
realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload
] as $key => $value
) {
if ($value !== false) {

168
app/realtime.php
View file

@ -1,7 +1,6 @@
<?php
use Appwrite\Auth\Auth;
use Appwrite\Database\DatabasePool;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Network\Validator\Origin;
use Appwrite\Utopia\Response;
@ -21,8 +20,11 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Appwrite\Utopia\Request;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\WebSocket\Server;
use Utopia\WebSocket\Adapter;
@ -30,6 +32,68 @@ require_once __DIR__ . '/init.php';
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
function getConsoleDB(): Database
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database->setNamespace('console');
return $database;
}
function getProjectDB(Document $project): Database
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}
$realtime = new Realtime();
/**
@ -92,38 +156,6 @@ $logError = function (Throwable $error, string $action) use ($register) {
$server->error($logError);
function getDatabase(Registry &$register, string $projectId)
{
$redis = $register->get('redisPool')->get();
$dbPool = $register->get('dbPool');
/** Get the console DB */
$database = $dbPool->getConsoleDB();
$pdo = $dbPool->getPDOFromPool($database);
$database = DatabasePool::wait(
DatabasePool::getDatabase($pdo->getConnection(), $redis, '_console'),
'realtime'
);
if ($projectId !== 'console') {
$project = Authorization::skip(fn() => $database->getDocument('projects', $projectId));
$database = $project->getAttribute('database', '');
$pdo = $dbPool->getPDOFromPool($database);
$database = DatabasePool::wait(
DatabasePool::getDatabase($pdo->getConnection(), $redis, "_{$project->getInternalId()}"),
'realtime'
);
}
return [
$database,
function () use ($register, $redis) {
$register->get('dbPool')->reset();
$register->get('redisPool')->put($redis);
}
];
}
$server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) {
sleep(5); // wait for the initial database schema to be ready
Console::success('Server started successfully');
@ -133,7 +165,8 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
*/
go(function () use ($register, $containerId, &$statsDocument) {
$attempts = 0;
[$database, $returnDatabase] = getDatabase($register, 'console');
$database = getConsoleDB();
do {
try {
$attempts++;
@ -153,7 +186,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
sleep(DATABASE_RECONNECT_SLEEP);
}
} while (true);
call_user_func($returnDatabase);
$register->get('pools')->reclaim();
});
/**
@ -169,7 +202,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
}
try {
[$database, $returnDatabase] = getDatabase($register, 'console');
$database = getConsoleDB();
$statsDocument
->setAttribute('timestamp', DateTime::now())
@ -179,7 +212,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
} catch (\Throwable $th) {
call_user_func($logError, $th, "updateWorkerDocument");
} finally {
call_user_func($returnDatabase);
$register->get('pools')->reclaim();
}
});
});
@ -195,7 +228,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
* Sending current connections to project channels on the console project every 5 seconds.
*/
if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) {
[$database, $returnDatabase] = getDatabase($register, '_console');
$database = getConsoleDB();
$payload = [];
@ -240,7 +273,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
]));
}
call_user_func($returnDatabase);
$register->get('pools')->reclaim();
}
/**
* Sending test message for SDK E2E tests every 5 seconds.
@ -275,8 +308,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
}
$start = time();
/** @var Redis $redis */
$redis = $register->get('redisPool')->get();
$redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
if ($redis->ping(true)) {
@ -295,9 +327,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) {
$connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId]));
[$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, 'console');
$consoleDatabase = getConsoleDB();
$project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId));
[$database, $returnDatabase] = getDatabase($register, $project->getId());
$database = getProjectDB($project);
$user = $database->getDocument('users', $userId);
@ -305,8 +337,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
$realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']);
call_user_func($returnDatabase);
call_user_func($returnConsoleDatabase);
$register->get('pools')->reclaim();
}
}
@ -334,7 +365,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
call_user_func($logError, $th, "pubSubConnection");
Console::error('Pub/sub error: ' . $th->getMessage());
$register->get('redisPool')->put($redis);
$register->get('pools')->reclaim();
$attempts++;
sleep(DATABASE_RECONNECT_SLEEP);
continue;
@ -349,15 +380,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
$request = new Request($request);
$response = new Response(new SwooleResponse());
/** @var PDO $db */
$dbPool = $register->get('dbPool');
/** @var Redis $redis */
$redis = $register->get('redisPool')->get();
Console::info("Connection open (user: {$connection})");
App::setResource('dbPool', fn() => $dbPool);
App::setResource('cache', fn() => $redis);
App::setResource('pools', fn() => $register->get('pools'));
App::setResource('request', fn() => $request);
App::setResource('response', fn() => $response);
@ -372,13 +397,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
throw new Exception('Missing or unknown project ID', 1008);
}
$dbForProject = $app->getResource('dbForProject');
/** @var \Utopia\Database\Document $console */
$console = $app->getResource('console');
/** @var \Utopia\Database\Document $user */
$user = $app->getResource('user');
$dbForProject = getProjectDB($project);
$console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */
$user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */
/*
* Abuse Check
@ -456,36 +477,20 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
Console::error('[Error] Code: ' . $response['data']['code']);
Console::error('[Error] Message: ' . $response['data']['message']);
}
if ($th instanceof PDOException) {
$db = null;
}
} finally {
/**
* Put used PDO and Redis Connections back into their pools.
*/
$dbPool->reset();
$register->get('redisPool')->put($redis);
$register->get('pools')->reclaim();
}
});
$server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) {
try {
$response = new Response(new SwooleResponse());
$redis = $register->get('redisPool')->get();
$dbPool = $register->get('dbPool');
$projectId = $realtime->connections[$connection]['projectId'];
/** Get the console DB */
$database = $dbPool->getConsoleDB();
$pdo = $dbPool->getPDOFromPool($database);
$database = DatabasePool::getDatabase($pdo->getConnection(), $redis, '_console');
$database = getConsoleDB();
if ($projectId !== 'console') {
$project = Authorization::skip(fn() => $database->getDocument('projects', $projectId));
$database = $project->getAttribute('database', '');
$pdo = $dbPool->getPDOFromPool($database);
$database = DatabasePool::getDatabase($pdo->getConnection(), $redis, "_{$project->getInternalId()}");
$database = getProjectDB($project);
}
/*
@ -569,8 +574,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
$server->close($connection, $th->getCode());
}
} finally {
$dbPool->reset();
$register->get('redisPool')->put($redis);
$register->get('pools')->reclaim();
}
});

87
app/tasks/doctor.php
View file

@ -8,6 +8,7 @@ use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Domains\Domain;
$cli
@ -21,7 +22,7 @@ $cli
Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n");
Console::log('Checking for production best practices...');
Console::log('[Settings]');
$domain = new Domain(App::getEnv('_APP_DOMAIN'));
@ -77,7 +78,6 @@ $cli
Console::log('🟢 HTTPS force option is enabled');
}
$providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
@ -90,32 +90,55 @@ $cli
\sleep(0.2);
try {
Console::log("\n" . 'Checking connectivity...');
Console::log("\n" . '[Connectivity]');
} catch (\Throwable $th) {
//throw $th;
}
try {
$dbPool = $register->get('dbPool'); /* @var $dbPool DatabasePool */
$database = $dbPool->getConsoleDB();
$pdo = $dbPool->getPDO($database);
Console::success('Database............connected 👍');
} catch (\Throwable $th) {
Console::error('Database.........disconnected 👎');
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$configs = [
'Console.DB' => Config::getParam('pools-console'),
'Projects.DB' => Config::getParam('pools-database'),
];
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
if ($adapter->ping()) {
Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected');
} else {
Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected');
}
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected');
}
}
}
try {
$register->get('cache');
Console::success('Queue...............connected 👍');
} catch (\Throwable $th) {
Console::error('Queue............disconnected 👎');
}
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$configs = [
'Cache' => Config::getParam('pools-cache'),
'Queue' => Config::getParam('pools-queue'),
'PubSub' => Config::getParam('pools-pubsub'),
];
try {
$register->get('cache');
Console::success('Cache...............connected 👍');
} catch (\Throwable $th) {
Console::error('Cache............disconnected 👎');
foreach ($configs as $key => $config) {
foreach ($config as $pool) {
try {
$adapter = $pools->get($pool)->pop()->getResource();
if ($adapter->ping()) {
Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected');
} else {
Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected');
}
} catch (\Throwable $th) {
Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected');
}
}
}
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled
@ -126,12 +149,12 @@ $cli
);
if ((@$antivirus->ping())) {
Console::success('Antivirus...........connected 👍');
Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected');
} else {
Console::error('Antivirus........disconnected 👎');
Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected');
}
} catch (\Throwable $th) {
Console::error('Antivirus........disconnected 👎');
Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected');
}
}
@ -144,35 +167,35 @@ $cli
$mail->AltBody = 'Hello World';
$mail->send();
Console::success('SMTP................connected 👍');
Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected');
} catch (\Throwable $th) {
Console::error('SMTP.............disconnected 👎');
Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
}
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) {
Console::success('StatsD..............connected 👍');
Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected');
\fclose($fp);
} else {
Console::error('StatsD...........disconnected 👎');
Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected');
}
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) {
Console::success('InfluxDB............connected 👍');
Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected');
\fclose($fp);
} else {
Console::error('InfluxDB.........disconnected 👎');
Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected');
}
\sleep(0.2);
Console::log('');
Console::log('Checking volumes...');
Console::log('[Volumes]');
foreach (
[
@ -200,7 +223,7 @@ $cli
\sleep(0.2);
Console::log('');
Console::log('Checking disk space usage...');
Console::log('[Disk Space]');
foreach (
[

15
app/tasks/maintenance.php
View file

@ -3,7 +3,6 @@
global $cli;
use Appwrite\Auth\Auth;
use Appwrite\Database\DatabasePool;
use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Utopia\App;
@ -16,8 +15,6 @@ $cli
->task('maintenance')
->desc('Schedules maintenance tasks and publishes them to resque')
->action(function () {
global $register;
Console::title('Maintenance V1');
Console::success(APP_NAME . ' maintenance process v1 has started');
@ -115,16 +112,8 @@ $cli
$usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days
$cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
Console::loop(function () use ($register, $interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) {
$redis = $register->get('cache');
$dbPool = $register->get('dbPool');
$database = $dbPool->getConsoleDB();
$pdo = $dbPool->getPDO($database);
$database = DatabasePool::wait(
DatabasePool::getDatabase($pdo, $redis, '_console'),
'certificates',
);
Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) {
$database = getConsoleDB();
$time = DateTime::now();

11
app/tasks/specs.php
View file

@ -9,8 +9,12 @@ use Appwrite\Specification\Specification;
use Appwrite\Utopia\Response;
use Swoole\Http\Response as HttpResponse;
use Utopia\App;
use Utopia\Cache\Adapter\None;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Adapter\MySQL;
use Utopia\Database\Database;
use Utopia\Request;
use Utopia\Validator\WhiteList;
@ -19,16 +23,15 @@ $cli
->param('version', 'latest', new Text(16), 'Spec version', true)
->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true)
->action(function ($version, $mode) use ($register) {
$consoleDB = $register->get('dbPool')->getConsoleDB();
$redis = $register->get('cache');
$appRoutes = App::getRoutes();
$response = new Response(new HttpResponse());
$mocks = ($mode === 'mocks');
// Mock dependencies
App::setResource('request', fn () => new Request());
App::setResource('response', fn () => $response);
App::setResource('consoleDB', fn () => $consoleDB);
App::setResource('cache', fn () => $redis);
App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None())));
App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None())));
$platforms = [
'client' => APP_PLATFORM_CLIENT,

44
app/tasks/usage.php
View file

@ -2,7 +2,6 @@
global $cli, $register;
use Appwrite\Database\DatabasePool;
use Appwrite\Usage\Calculators\Aggregator;
use Appwrite\Usage\Calculators\Database;
use Appwrite\Usage\Calculators\TimeSeries;
@ -11,39 +10,12 @@ use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Utopia\Logger\Log;
use Utopia\Validator\WhiteList;
Authorization::disable();
Authorization::setDefaultStatus(false);
function getInfluxDB(Registry &$register): InfluxDatabase
{
/** @var InfluxDB\Client $client */
$client = $register->get('influxdb');
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}
$logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) {
$logger = $register->get('logger');
@ -78,7 +50,6 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use (
Console::warning($error->getTraceAsString());
};
function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
$interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
@ -120,21 +91,12 @@ $cli
->task('usage')
->param('type', 'timeseries', new WhiteList(['timeseries', 'database']))
->desc('Schedules syncing data from influxdb to Appwrite console db')
->action(function (string $type) use ($register, $logError) {
->action(function (string $type) use ($logError) {
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$redis = $register->get('cache');
$dbPool = $register->get('dbPool');
$database = $dbPool->getConsoleDB();
$pdo = $dbPool->getPDO($database);
$database = DatabasePool::wait(
DatabasePool::getDatabase($pdo, $redis, '_console'),
'projects',
);
$influxDB = getInfluxDB($register);
$database = getConsoleDB();
$influxDB = getInfluxDB();
switch ($type) {
case 'timeseries':

88
app/views/install/compose.phtml
View file

@ -88,12 +88,15 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
@ -181,10 +184,15 @@ services:
- _APP_WORKER_PER_CORE
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -203,12 +211,15 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -254,12 +265,15 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
@ -300,12 +314,15 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -325,12 +342,15 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -354,12 +374,15 @@ services:
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -378,12 +401,15 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_FUNCTIONS_TIMEOUT
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
@ -511,12 +537,15 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -539,16 +568,19 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -573,14 +605,14 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG

1
app/workers/audits.php
View file

@ -1,6 +1,5 @@
<?php
use Appwrite\Event\Event;
use Appwrite\Resque\Worker;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;

2
app/workers/builds.php
View file

@ -7,7 +7,6 @@ use Appwrite\Utopia\Response\Model\Deployment;
use Cron\CronExpression;
use Executor\Executor;
use Appwrite\Usage\Stats;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\App;
use Utopia\CLI\Console;
@ -15,7 +14,6 @@ use Utopia\Database\ID;
use Utopia\Storage\Storage;
use Utopia\Database\Document;
use Utopia\Config\Config;
use Utopia\Database\Query;
require_once __DIR__ . '/../init.php';

1
app/workers/certificates.php
View file

@ -1,6 +1,5 @@
<?php
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Network\Validator\CNAME;
use Appwrite\Resque\Worker;

5
composer.json
View file

@ -48,10 +48,10 @@
"utopia-php/abuse": "0.14.*",
"utopia-php/analytics": "0.2.*",
"utopia-php/audit": "0.15.*",
"utopia-php/cache": "0.6.*",
"utopia-php/cache": "0.8.*",
"utopia-php/cli": "0.13.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.26.*",
"utopia-php/database": "dev-feat-update-cache-lib as 0.26.1",
"utopia-php/locale": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/preloader": "0.2.*",
@ -61,6 +61,7 @@
"utopia-php/websocket": "0.1.0",
"utopia-php/image": "0.5.*",
"utopia-php/orchestration": "0.6.*",
"utopia-php/pools": "0.1.*",
"resque/php-resque": "1.3.6",
"matomo/device-detector": "6.0.0",
"dragonmantank/cron-expression": "3.3.1",

124
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "08fdd139ad1285b02c4b4e555679e7de",
"content-hash": "f3beee3a829a19e53b311052111bde2c",
"packages": [
{
"name": "adhocore/jwt",
@ -1897,24 +1897,26 @@
},
{
"name": "utopia-php/cache",
"version": "0.6.1",
"version": "0.8.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cache.git",
"reference": "9889235a6d3da6cbb1f435201529da4d27c30e79"
"reference": "212e66100a1f32e674fca5d9bc317cc998303089"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cache/zipball/9889235a6d3da6cbb1f435201529da4d27c30e79",
"reference": "9889235a6d3da6cbb1f435201529da4d27c30e79",
"url": "https://api.github.com/repos/utopia-php/cache/zipball/212e66100a1f32e674fca5d9bc317cc998303089",
"reference": "212e66100a1f32e674fca5d9bc317cc998303089",
"shasum": ""
},
"require": {
"ext-json": "*",
"ext-memcached": "*",
"ext-redis": "*",
"php": ">=8.0"
},
"require-dev": {
"laravel/pint": "1.2.*",
"phpunit/phpunit": "^9.3",
"vimeo/psalm": "4.13.1"
},
@ -1928,12 +1930,6 @@
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
}
],
"description": "A simple cache library to manage application cache storing, loading and purging",
"keywords": [
"cache",
@ -1944,9 +1940,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/cache/issues",
"source": "https://github.com/utopia-php/cache/tree/0.6.1"
"source": "https://github.com/utopia-php/cache/tree/0.8.0"
},
"time": "2022-08-10T08:12:46+00:00"
"time": "2022-10-16T16:48:09+00:00"
},
{
"name": "utopia-php/cli",
@ -2054,16 +2050,16 @@
},
{
"name": "utopia-php/database",
"version": "0.26.0",
"version": "dev-feat-update-cache-lib",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "d172af2541137c83a86d066f82f48914b5a3a610"
"reference": "44ae47dfd49c9c7c0cba29f6867347e25c23b57b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/d172af2541137c83a86d066f82f48914b5a3a610",
"reference": "d172af2541137c83a86d066f82f48914b5a3a610",
"url": "https://api.github.com/repos/utopia-php/database/zipball/44ae47dfd49c9c7c0cba29f6867347e25c23b57b",
"reference": "44ae47dfd49c9c7c0cba29f6867347e25c23b57b",
"shasum": ""
},
"require": {
@ -2072,7 +2068,7 @@
"ext-redis": "*",
"mongodb/mongodb": "1.8.0",
"php": ">=8.0",
"utopia-php/cache": "0.6.*",
"utopia-php/cache": "0.8.*",
"utopia-php/framework": "0.*.*"
},
"require-dev": {
@ -2092,16 +2088,6 @@
"license": [
"MIT"
],
"authors": [
{
"name": "Eldad Fux",
"email": "eldad@appwrite.io"
},
{
"name": "Brandon Leckemby",
"email": "brandon@appwrite.io"
}
],
"description": "A simple library to manage application persistency using multiple database adapters",
"keywords": [
"database",
@ -2112,9 +2098,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.26.0"
"source": "https://github.com/utopia-php/database/tree/feat-update-cache-lib"
},
"time": "2022-10-03T17:12:01+00:00"
"time": "2022-10-16T17:35:26+00:00"
},
{
"name": "utopia-php/domains",
@ -2443,6 +2429,59 @@
},
"time": "2022-07-13T16:47:18+00:00"
},
{
"name": "utopia-php/pools",
"version": "0.1.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/pools.git",
"reference": "5a467a569a80aefc846a97dc195b4adc2fd71805"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/pools/zipball/5a467a569a80aefc846a97dc195b4adc2fd71805",
"reference": "5a467a569a80aefc846a97dc195b4adc2fd71805",
"shasum": ""
},
"require": {
"ext-mongodb": "*",
"ext-pdo": "*",
"ext-redis": "*",
"php": ">=8.0"
},
"require-dev": {
"phpunit/phpunit": "^9.4",
"vimeo/psalm": "4.0.1"
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\Pools\\": "src/Pools"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Team Appwrite",
"email": "team@appwrite.io"
}
],
"description": "A simple library to manage connection pools",
"keywords": [
"framework",
"php",
"pools",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/pools/issues",
"source": "https://github.com/utopia-php/pools/tree/0.1.0"
},
"time": "2022-10-11T19:31:07+00:00"
},
{
"name": "utopia-php/preloader",
"version": "0.2.4",
@ -3535,16 +3574,16 @@
},
{
"name": "phpunit/php-code-coverage",
"version": "9.2.16",
"version": "9.2.17",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "2593003befdcc10db5e213f9f28814f5aa8ac073"
"reference": "aa94dc41e8661fe90c7316849907cba3007b10d8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/2593003befdcc10db5e213f9f28814f5aa8ac073",
"reference": "2593003befdcc10db5e213f9f28814f5aa8ac073",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/aa94dc41e8661fe90c7316849907cba3007b10d8",
"reference": "aa94dc41e8661fe90c7316849907cba3007b10d8",
"shasum": ""
},
"require": {
@ -3600,7 +3639,7 @@
],
"support": {
"issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.16"
"source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.17"
},
"funding": [
{
@ -3608,7 +3647,7 @@
"type": "github"
}
],
"time": "2022-08-20T05:26:47+00:00"
"time": "2022-08-30T12:24:04+00:00"
},
{
"name": "phpunit/php-file-iterator",
@ -5357,9 +5396,18 @@
"time": "2022-09-28T08:42:51+00:00"
}
],
"aliases": [],
"aliases": [
{
"package": "utopia-php/database",
"version": "dev-feat-update-cache-lib",
"alias": "0.26.1",
"alias_normalized": "0.26.1.0"
}
],
"minimum-stability": "stable",
"stability-flags": [],
"stability-flags": {
"utopia-php/database": 20
},
"prefer-stable": false,
"prefer-lowest": false,
"platform": {

146
docker-compose.yml
View file

@ -133,12 +133,20 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_PROJECT
- _APP_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_CONNECTIONS_PUBSUB
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
@ -211,10 +219,19 @@ services:
- _APP_WORKER_PER_CORE
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_PUBSUB
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -235,12 +252,19 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -266,6 +290,7 @@ services:
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -290,12 +315,19 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- *x-env-storage
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -319,12 +351,19 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -346,12 +385,19 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -376,12 +422,19 @@ services:
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -402,12 +455,19 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_FUNCTIONS_TIMEOUT
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
@ -479,6 +539,7 @@ services:
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
@ -505,6 +566,7 @@ services:
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_LOGGING_PROVIDER
@ -528,12 +590,18 @@ services:
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -559,16 +627,22 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -591,16 +665,22 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_CONSOLE
- _APP_DB_PROJECT
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_USAGE_TIMESERIES_INTERVAL
- _APP_USAGE_DATABASE_INTERVAL
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -622,6 +702,7 @@ services:
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
mariadb:
image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
@ -644,7 +725,6 @@ services:
# smtp:
# image: appwrite/smtp:1.2.0
# container_name: appwrite-smtp
# restart: unless-stopped
# networks:
# - appwrite
# environment:
@ -731,7 +811,6 @@ services:
image: adminer
container_name: appwrite-adminer
<<: *x-logging
restart: always
ports:
- 9506:8080
networks:
@ -739,7 +818,6 @@ services:
# redis-commander:
# image: rediscommander/redis-commander:latest
# restart: unless-stopped
# networks:
# - appwrite
# environment:
@ -749,7 +827,6 @@ services:
# resque:
# image: appwrite/resque-web:1.1.0
# restart: unless-stopped
# networks:
# - appwrite
# ports:
@ -763,7 +840,6 @@ services:
# chronograf:
# image: chronograf:1.6
# container_name: appwrite-chronograf
# restart: unless-stopped
# networks:
# - appwrite
# volumes:

2
docs/references/health/get-cache.md
View file

@ -1 +1 @@
Check the Appwrite in-memory cache server is up and connection is successful.
Check the Appwrite in-memory cache servers are up and connection is successful.

2
docs/references/health/get-db.md
View file

@ -1 +1 @@
Check the Appwrite database server is up and connection is successful.
Check the Appwrite database servers are up and connection is successful.

1
docs/references/health/get-pubsub.md Normal file
View file

@ -0,0 +1 @@
Check the Appwrite pub-sub servers are up and connection is successful.

1
docs/references/health/get-queue.md Normal file
View file

@ -0,0 +1 @@
Check the Appwrite queue messaging servers are up and connection is successful.

221
src/Appwrite/Database/DatabasePool.php
View file

@ -1,221 +0,0 @@
<?php
namespace Appwrite\Database;
use Appwrite\Database\PDO as DatabasePDO;
use PDO;
use Utopia\App;
use Appwrite\DSN\DSN;
use Utopia\Cache\Cache;
use Swoole\Database\PDOProxy;
use Utopia\Database\Database;
use Appwrite\Extend\Exception;
use Appwrite\Database\PDOPool;
use Swoole\Database\PDOConfig;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\CLI\Console;
class DatabasePool
{
/**
* @var array
*
* Array to store mappings from database names to PDOPool instances.
*/
protected array $pools = [];
/**
* @var array
*
* Array to store mappings from database names to DSNs
*/
protected array $dsns = [];
/**
* @var string
*
* The name of the console Database
*/
protected string $consoleDB = '';
/**
* Constructor for Database pools
*
* @param array $consoleDB
* @param array $projectDB
*
*/
public function __construct(array $consoleDB, array $projectDB)
{
if (count($consoleDB) != 1) {
throw new Exception('Console DB should contain only one entry', 500);
}
if (empty($projectDB)) {
throw new Exception('Project DB is not defined', 500);
}
$this->consoleDB = array_key_first($consoleDB);
$this->dsns = array_merge($consoleDB, $projectDB);
/** Create PDO pool instances for all the dsns */
foreach ($this->dsns as $name => $dsn) {
$dsn = new DSN($dsn);
$pdoConfig = (new PDOConfig())
->withHost($dsn->getHost())
->withPort($dsn->getPort())
->withDbName($dsn->getDatabase())
->withCharset('utf8mb4')
->withUsername($dsn->getUser())
->withPassword($dsn->getPassword())
->withOptions([
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true
]);
$pool = new PDOPool($pdoConfig, $name, 64);
$this->pools[$name] = $pool;
}
}
/**
* Get a single PDO instance by database name
*
* @param string $name
*
* @return ?PDO
*/
public function getPDO(string $name): ?PDO
{
$dsn = $this->dsns[$name] ?? throw new Exception("Database with name : $name not found.", 500);
$dsn = new DSN($dsn);
$dbHost = $dsn->getHost();
$dbPort = $dsn->getPort();
$dbUser = $dsn->getUser();
$dbPass = $dsn->getPassword();
$dbScheme = $dsn->getDatabase();
$pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true
));
return $pdo;
}
/**
* Get a PDO instance from the list of available database pools. Meant to be used in co-routines
*
* @param string $projectId
*
* @return array
*/
public function getPDOFromPool(string $name): PDOWrapper
{
$pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_DB_PROJECT in .env", 500);
$pdo = $pool->get();
return $pdo;
}
/**
* Get a random PDO instance from the available database pools
*
* @return PDOWrapper
*/
public function getAnyFromPool(): PDOWrapper
{
$name = array_rand($this->pools);
$pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_DB_PROJECT in .env", 500);
$pdo = $pool->get();
return $pdo;
}
public function reset(): void
{
foreach ($this->pools as $pool) {
$pool->reset();
}
}
/**
* Return a PDO instance back to its database pool
*
* @param PDOProxy $db
* @param string $name
*
* @return void
*/
public function put(PDOProxy $db, string $name): void
{
$pool = $this->pools[$name] ?? null;
if ($pool === null) {
throw new Exception("Failed to put PDO into database pool. Database pool with name : $name not found", 500);
}
$pool->put($db);
}
/**
* Get the name of the console DB
*
* @return ?string
*/
public function getConsoleDB(): ?string
{
if (empty($this->consoleDB)) {
throw new Exception('Console DB is not defined', 500);
};
return $this->consoleDB;
}
public static function wait(Database $database, string $collection)
{
$attempts = 0;
do {
try {
$attempts++;
if (!$database->exists($database->getDefaultDatabase(), $collection)) {
throw new Exception('Collection not ready');
}
break; // leave loop if successful
} catch (\Exception $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep(DATABASE_RECONNECT_SLEEP);
}
} while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
return $database;
}
/**
* Get a database instance from a PDO and cache
*
* @param PDO|PDOProxy $pdo
* @param \Redis $redis
* @param string $namespace
*
* @return Database
*/
public static function getDatabase(PDO|PDOProxy $pdo, \Redis $redis, string $namespace = ''): Database
{
$cache = new Cache(new RedisCache($redis));
$database = new Database(new MariaDB($pdo), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace($namespace);
return $database;
}
}

47
src/Appwrite/Database/PDOPool.php
View file

@ -1,47 +0,0 @@
<?php
namespace Appwrite\Database;
use Swoole\Database\PDOConfig;
use Swoole\Database\PDOPool as SwoolePDOPool;
class PDOPool
{
private SwoolePDOPool $pool;
private string $name;
private array $activeConnections = [];
public function __construct(PDOConfig $pdoConfig, string $name, int $size = SwoolePDOPool::DEFAULT_SIZE)
{
$this->pool = new SwoolePDOPool($pdoConfig, $size);
$this->name = $name;
}
public function getActiveConnections()
{
return $this->activeConnections;
}
public function get(float $timeout = -1): PDOWrapper
{
$pdo = $this->pool->get($timeout);
$this->activeConnections[] = $pdo;
return new PDOWrapper($pdo, $this->name);
}
public function put(PDOWrapper $pdo): void
{
$this->pool->put($pdo->getConnection());
unset($this->activeConnections[array_search($pdo, $this->activeConnections)]);
}
public function reset(): void
{
foreach ($this->activeConnections as $connection) {
$this->pool->put($connection);
}
$this->activeConnections = [];
}
}

27
src/Appwrite/Database/PDOWrapper.php
View file

@ -1,27 +0,0 @@
<?php
namespace Appwrite\Database;
use Swoole\Database\PDOProxy;
class PDOWrapper
{
private string $name;
private PDOProxy $connection;
public function __construct(PDOProxy $connection, string $name)
{
$this->connection = $connection;
$this->name = $name;
}
public function getName()
{
return $this->name;
}
public function getConnection()
{
return $this->connection;
}
}

110
src/Appwrite/Extend/PDO.php
View file

@ -1,110 +0,0 @@
<?php
namespace Appwrite\Extend;
use PDO as PDONative;
class PDO extends PDONative
{
/**
* @var PDONative
*/
protected $pdo;
/**
* @var mixed
*/
protected $dsn;
/**
* @var mixed
*/
protected $username;
/**
* @var mixed
*/
protected $passwd;
/**
* @var mixed
*/
protected $options;
/**
* Create A Proxy PDO Object
*/
public function __construct($dsn, $username = null, $passwd = null, $options = null)
{
$this->dsn = $dsn;
$this->username = $username;
$this->passwd = $passwd;
$this->options = $options;
$this->pdo = new PDONative($dsn, $username, $passwd, $options);
}
public function setAttribute($attribute, $value)
{
return $this->pdo->setAttribute($attribute, $value);
}
public function prepare($statement, $driver_options = null)
{
return new PDOStatement($this, $this->pdo->prepare($statement, []));
}
public function quote($string, $parameter_type = PDONative::PARAM_STR)
{
return $this->pdo->quote($string, $parameter_type);
}
public function beginTransaction()
{
try {
$result = $this->pdo->beginTransaction();
} catch (\Throwable $th) {
$this->pdo = $this->reconnect();
$result = $this->pdo->beginTransaction();
}
return $result;
}
public function rollBack()
{
try {
$result = $this->pdo->rollBack();
} catch (\Throwable $th) {
$this->pdo = $this->reconnect();
return false;
}
return $result;
}
public function commit()
{
try {
$result = $this->pdo->commit();
} catch (\Throwable $th) {
$this->pdo = $this->reconnect();
$result = $this->pdo->commit();
}
return $result;
}
public function reconnect(): PDONative
{
$this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options);
echo '[PDO] MySQL connection restarted' . PHP_EOL;
// Connection settings
$this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays
$this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions
return $this->pdo;
}
}

115
src/Appwrite/Extend/PDOStatement.php
View file

@ -1,115 +0,0 @@
<?php
namespace Appwrite\Extend;
use PDO as PDONative;
use PDOStatement as PDOStatementNative;
class PDOStatement extends PDOStatementNative
{
/**
* @var PDO
*/
protected $pdo;
/**
* Params
*/
protected $params = [];
/**
* Values
*/
protected $values = [];
/**
* Columns
*/
protected $columns = [];
/**
* @var PDOStatementNative
*/
protected $PDOStatement;
public function __construct(PDO &$pdo, PDOStatementNative $PDOStatement)
{
$this->pdo = &$pdo;
$this->PDOStatement = $PDOStatement;
}
public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR)
{
$this->values[$parameter] = ['value' => $value, 'data_type' => $data_type];
$result = $this->PDOStatement->bindValue($parameter, $value, $data_type);
return $result;
}
public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null)
{
$this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options];
$result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options);
return $result;
}
public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null)
{
$this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata];
$result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata);
return $result;
}
public function execute($input_parameters = null)
{
try {
$result = $this->PDOStatement->execute($input_parameters);
} catch (\Throwable $th) {
$this->pdo = $this->pdo->reconnect();
$this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []);
foreach ($this->values as $key => $set) {
$this->PDOStatement->bindValue($key, $set['value'], $set['data_type']);
}
foreach ($this->params as $key => $set) {
$this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']);
}
foreach ($this->columns as $key => $set) {
$this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']);
}
$result = $this->PDOStatement->execute($input_parameters);
}
return $result;
}
public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0)
{
$result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset);
return $result;
}
/**
* Fetch All
*
* @param int $fetch_style
* @param mixed $fetch_args
*
* @return array|false
*/
public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args)
{
$result = $this->PDOStatement->fetchAll();
return $result;
}
}

89
src/Appwrite/Resque/Worker.php
View file

@ -2,8 +2,11 @@
namespace Appwrite\Resque;
use Appwrite\Database\DatabasePool;
use Exception;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Database\Database;
use Utopia\Storage\Device;
use Utopia\Storage\Storage;
@ -13,7 +16,6 @@ use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Wasabi;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\S3;
use Exception;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
@ -134,7 +136,12 @@ abstract class Worker
*/
public function tearDown(): void
{
global $register;
try {
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools->reclaim();
$this->shutdown();
} catch (\Throwable $error) {
foreach (self::$errorCallbacks as $errorCallback) {
@ -165,22 +172,23 @@ abstract class Worker
protected function getProjectDB(Document $project): Database
{
global $register;
$database = $project->getAttribute('database', '');
$internalId = $project->getInternalId();
if (empty($database)) {
throw new \Exception('Database name not provided - cannot get database');
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
if ($project->isEmpty() || $project->getId() === 'console') {
return $this->getConsoleDB();
}
$cache = $register->get('cache');
$dbPool = $register->get('dbPool');
$namespace = "_$internalId";
$pdo = $dbPool->getPDO($database);
$dbForProject = DatabasePool::wait(
DatabasePool::getDatabase($pdo, $cache, $namespace),
'projects'
);
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
return $dbForProject;
$database = new Database($dbAdapter, $this->getCache());
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
/**
@ -190,21 +198,45 @@ abstract class Worker
protected function getConsoleDB(): Database
{
global $register;
$cache = $register->get('cache');
$dbPool = $register->get('dbPool');
$database = $dbPool->getConsoleDB();
if (empty($database)) {
throw new \Exception('Database name not provided - cannot get database');
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, $this->getCache());
$database->setNamespace('console');
return $database;
}
/**
* Get Cache
* @return Cache
*/
protected function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
$namespace = "_console";
$pdo = $dbPool->getPDO($database);
$dbForConsole = DatabasePool::wait(
DatabasePool::getDatabase($pdo, $cache, $namespace),
'_metadata'
);
return $dbForConsole;
return new Cache(new Sharding($adapters));
}
/**
@ -227,7 +259,6 @@ abstract class Worker
return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
}
/**
* Get Builds Storage Device
* @param string $projectId of the project

2
src/Appwrite/Utopia/Response.php
View file

@ -207,6 +207,7 @@ class Response extends SwooleResponse
public const MODEL_HEALTH_QUEUE = 'healthQueue';
public const MODEL_HEALTH_TIME = 'healthTime';
public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus';
public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList';
// Deprecated
public const MODEL_PERMISSIONS = 'permissions';
@ -268,6 +269,7 @@ class Response extends SwooleResponse
->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE))
->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false))
->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE))
->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS))
// Entities
->setModel(new Database())
->setModel(new Collection())

6
src/Appwrite/Utopia/Response/Model/HealthStatus.php
View file

@ -10,6 +10,12 @@ class HealthStatus extends Model
public function __construct()
{
$this
->addRule('name', [
'type' => self::TYPE_STRING,
'description' => 'Name of the service.',
'default' => '',
'example' => 'database',
])
->addRule('ping', [
'type' => self::TYPE_INTEGER,
'description' => 'Duration in milliseconds how long the health check took.',

220
tests/e2e/Services/Account/AccountBase.php
View file

@ -317,137 +317,137 @@ trait AccountBase
return $data;
}
// /**
// * @depends testCreateAccountSession
// */
// public function testGetAccountLogs($data): array
// {
// sleep(10);
// $session = $data['session'] ?? '';
// $sessionId = $data['sessionId'] ?? '';
// $userId = $data['id'] ?? '';
// /**
// * Test for SUCCESS
// */
// $response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
// 'origin' => 'http://localhost',
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id'],
// 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
// ]));
/**
* @depends testCreateAccountSession
*/
public function testGetAccountLogs($data): array
{
sleep(10);
$session = $data['session'] ?? '';
$sessionId = $data['sessionId'] ?? '';
$userId = $data['id'] ?? '';
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
// $this->assertEquals($response['headers']['status-code'], 200);
// $this->assertIsArray($response['body']['logs']);
// $this->assertNotEmpty($response['body']['logs']);
// $this->assertCount(3, $response['body']['logs']);
// $this->assertIsNumeric($response['body']['total']);
// $this->assertContains($response['body']['logs'][1]['event'], ["session.create"]);
// $this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP));
// $this->assertEquals(true, DateTime::isValid($response['body']['logs'][1]['time']));
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertIsArray($response['body']['logs']);
$this->assertNotEmpty($response['body']['logs']);
$this->assertCount(3, $response['body']['logs']);
$this->assertIsNumeric($response['body']['total']);
$this->assertContains($response['body']['logs'][1]['event'], ["session.create"]);
$this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals(true, DateTime::isValid($response['body']['logs'][1]['time']));
// $this->assertEquals('Windows', $response['body']['logs'][1]['osName']);
// $this->assertEquals('WIN', $response['body']['logs'][1]['osCode']);
// $this->assertEquals('10', $response['body']['logs'][1]['osVersion']);
$this->assertEquals('Windows', $response['body']['logs'][1]['osName']);
$this->assertEquals('WIN', $response['body']['logs'][1]['osCode']);
$this->assertEquals('10', $response['body']['logs'][1]['osVersion']);
// $this->assertEquals('browser', $response['body']['logs'][1]['clientType']);
// $this->assertEquals('Chrome', $response['body']['logs'][1]['clientName']);
// $this->assertEquals('CH', $response['body']['logs'][1]['clientCode']);
// $this->assertEquals('70.0', $response['body']['logs'][1]['clientVersion']);
// $this->assertEquals('Blink', $response['body']['logs'][1]['clientEngine']);
$this->assertEquals('browser', $response['body']['logs'][1]['clientType']);
$this->assertEquals('Chrome', $response['body']['logs'][1]['clientName']);
$this->assertEquals('CH', $response['body']['logs'][1]['clientCode']);
$this->assertEquals('70.0', $response['body']['logs'][1]['clientVersion']);
$this->assertEquals('Blink', $response['body']['logs'][1]['clientEngine']);
// $this->assertEquals('desktop', $response['body']['logs'][1]['deviceName']);
// $this->assertEquals('', $response['body']['logs'][1]['deviceBrand']);
// $this->assertEquals('', $response['body']['logs'][1]['deviceModel']);
// $this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals('desktop', $response['body']['logs'][1]['deviceName']);
$this->assertEquals('', $response['body']['logs'][1]['deviceBrand']);
$this->assertEquals('', $response['body']['logs'][1]['deviceModel']);
$this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP));
// $this->assertEquals('--', $response['body']['logs'][1]['countryCode']);
// $this->assertEquals('Unknown', $response['body']['logs'][1]['countryName']);
$this->assertEquals('--', $response['body']['logs'][1]['countryCode']);
$this->assertEquals('Unknown', $response['body']['logs'][1]['countryName']);
// $this->assertContains($response['body']['logs'][2]['event'], ["user.create"]);
// $this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP));
// $this->assertEquals(true, DateTime::isValid($response['body']['logs'][2]['time']));
$this->assertContains($response['body']['logs'][2]['event'], ["user.create"]);
$this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals(true, DateTime::isValid($response['body']['logs'][2]['time']));
// $this->assertEquals('Windows', $response['body']['logs'][2]['osName']);
// $this->assertEquals('WIN', $response['body']['logs'][2]['osCode']);
// $this->assertEquals('10', $response['body']['logs'][2]['osVersion']);
$this->assertEquals('Windows', $response['body']['logs'][2]['osName']);
$this->assertEquals('WIN', $response['body']['logs'][2]['osCode']);
$this->assertEquals('10', $response['body']['logs'][2]['osVersion']);
// $this->assertEquals('browser', $response['body']['logs'][2]['clientType']);
// $this->assertEquals('Chrome', $response['body']['logs'][2]['clientName']);
// $this->assertEquals('CH', $response['body']['logs'][2]['clientCode']);
// $this->assertEquals('70.0', $response['body']['logs'][2]['clientVersion']);
// $this->assertEquals('Blink', $response['body']['logs'][2]['clientEngine']);
$this->assertEquals('browser', $response['body']['logs'][2]['clientType']);
$this->assertEquals('Chrome', $response['body']['logs'][2]['clientName']);
$this->assertEquals('CH', $response['body']['logs'][2]['clientCode']);
$this->assertEquals('70.0', $response['body']['logs'][2]['clientVersion']);
$this->assertEquals('Blink', $response['body']['logs'][2]['clientEngine']);
// $this->assertEquals('desktop', $response['body']['logs'][2]['deviceName']);
// $this->assertEquals('', $response['body']['logs'][2]['deviceBrand']);
// $this->assertEquals('', $response['body']['logs'][2]['deviceModel']);
// $this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals('desktop', $response['body']['logs'][2]['deviceName']);
$this->assertEquals('', $response['body']['logs'][2]['deviceBrand']);
$this->assertEquals('', $response['body']['logs'][2]['deviceModel']);
$this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP));
// $this->assertEquals('--', $response['body']['logs'][2]['countryCode']);
// $this->assertEquals('Unknown', $response['body']['logs'][2]['countryName']);
$this->assertEquals('--', $response['body']['logs'][2]['countryCode']);
$this->assertEquals('Unknown', $response['body']['logs'][2]['countryName']);
// $responseLimit = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
// 'origin' => 'http://localhost',
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id'],
// 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
// ]), [
// 'queries' => [ 'limit(1)' ],
// ]);
$responseLimit = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'queries' => [ 'limit(1)' ],
]);
// $this->assertEquals($responseLimit['headers']['status-code'], 200);
// $this->assertIsArray($responseLimit['body']['logs']);
// $this->assertNotEmpty($responseLimit['body']['logs']);
// $this->assertCount(1, $responseLimit['body']['logs']);
// $this->assertIsNumeric($responseLimit['body']['total']);
$this->assertEquals($responseLimit['headers']['status-code'], 200);
$this->assertIsArray($responseLimit['body']['logs']);
$this->assertNotEmpty($responseLimit['body']['logs']);
$this->assertCount(1, $responseLimit['body']['logs']);
$this->assertIsNumeric($responseLimit['body']['total']);
// $this->assertEquals($response['body']['logs'][0], $responseLimit['body']['logs'][0]);
$this->assertEquals($response['body']['logs'][0], $responseLimit['body']['logs'][0]);
// $responseOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
// 'origin' => 'http://localhost',
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id'],
// 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
// ]), [
// 'queries' => [ 'offset(1)' ],
// ]);
$responseOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'queries' => [ 'offset(1)' ],
]);
// $this->assertEquals($responseOffset['headers']['status-code'], 200);
// $this->assertIsArray($responseOffset['body']['logs']);
// $this->assertNotEmpty($responseOffset['body']['logs']);
// $this->assertCount(2, $responseOffset['body']['logs']);
// $this->assertIsNumeric($responseOffset['body']['total']);
$this->assertEquals($responseOffset['headers']['status-code'], 200);
$this->assertIsArray($responseOffset['body']['logs']);
$this->assertNotEmpty($responseOffset['body']['logs']);
$this->assertCount(2, $responseOffset['body']['logs']);
$this->assertIsNumeric($responseOffset['body']['total']);
// $this->assertEquals($response['body']['logs'][1], $responseOffset['body']['logs'][0]);
$this->assertEquals($response['body']['logs'][1], $responseOffset['body']['logs'][0]);
// $responseLimitOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
// 'origin' => 'http://localhost',
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id'],
// 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
// ]), [
// 'queries' => [ 'limit(1)', 'offset(1)' ],
// ]);
$responseLimitOffset = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'queries' => [ 'limit(1)', 'offset(1)' ],
]);
// $this->assertEquals($responseLimitOffset['headers']['status-code'], 200);
// $this->assertIsArray($responseLimitOffset['body']['logs']);
// $this->assertNotEmpty($responseLimitOffset['body']['logs']);
// $this->assertCount(1, $responseLimitOffset['body']['logs']);
// $this->assertIsNumeric($responseLimitOffset['body']['total']);
$this->assertEquals($responseLimitOffset['headers']['status-code'], 200);
$this->assertIsArray($responseLimitOffset['body']['logs']);
$this->assertNotEmpty($responseLimitOffset['body']['logs']);
$this->assertCount(1, $responseLimitOffset['body']['logs']);
$this->assertIsNumeric($responseLimitOffset['body']['total']);
// $this->assertEquals($response['body']['logs'][1], $responseLimitOffset['body']['logs'][0]);
// /**
// * Test for FAILURE
// */
// $response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
// 'origin' => 'http://localhost',
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id'],
// ]));
$this->assertEquals($response['body']['logs'][1], $responseLimitOffset['body']['logs'][0]);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/account/logs', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
]));
// $this->assertEquals($response['headers']['status-code'], 401);
$this->assertEquals($response['headers']['status-code'], 401);
// return $data;
// }
return $data;
}
// TODO Add tests for OAuth2 session creation

56
tests/e2e/Services/Health/HealthCustomServerTest.php
View file

@ -47,9 +47,9 @@ class HealthCustomServerTest extends Scope
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('pass', $response['body']['status']);
$this->assertIsInt($response['body']['ping']);
$this->assertLessThan(100, $response['body']['ping']);
$this->assertEquals('pass', $response['body']['statuses'][0]['status']);
$this->assertIsInt($response['body']['statuses'][0]['ping']);
$this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
/**
* Test for FAILURE
@ -69,9 +69,53 @@ class HealthCustomServerTest extends Scope
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('pass', $response['body']['status']);
$this->assertIsInt($response['body']['ping']);
$this->assertLessThan(100, $response['body']['ping']);
$this->assertEquals('pass', $response['body']['statuses'][0]['status']);
$this->assertIsInt($response['body']['statuses'][0]['ping']);
$this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
/**
* Test for FAILURE
*/
return [];
}
public function testQueueSuccess(): array
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('pass', $response['body']['statuses'][0]['status']);
$this->assertIsInt($response['body']['statuses'][0]['ping']);
$this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
/**
* Test for FAILURE
*/
return [];
}
public function testPubSubSuccess(): array
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/pubsub', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('pass', $response['body']['statuses'][0]['status']);
$this->assertIsInt($response['body']['statuses'][0]['ping']);
$this->assertLessThan(100, $response['body']['statuses'][0]['ping']);
/**
* Test for FAILURE