2022-11-10 06:01:43 +13:00
|
|
|
<?php
|
|
|
|
|
|
|
|
require_once __DIR__ . '/init.php';
|
|
|
|
|
2022-12-21 00:07:49 +13:00
|
|
|
use Appwrite\Event\Event;
|
2022-12-07 04:06:40 +13:00
|
|
|
use Appwrite\Event\Audit;
|
2022-12-21 01:22:58 +13:00
|
|
|
use Appwrite\Event\Build;
|
2022-12-14 00:16:12 +13:00
|
|
|
use Appwrite\Event\Certificate;
|
2022-12-14 04:54:01 +13:00
|
|
|
use Appwrite\Event\Database as EventDatabase;
|
2022-12-21 01:22:58 +13:00
|
|
|
use Appwrite\Event\Delete;
|
2022-11-16 07:13:17 +13:00
|
|
|
use Appwrite\Event\Func;
|
2023-11-28 23:19:55 +13:00
|
|
|
use Appwrite\Event\Hamster;
|
2022-12-21 01:22:58 +13:00
|
|
|
use Appwrite\Event\Mail;
|
2023-10-16 06:41:09 +13:00
|
|
|
use Appwrite\Event\Messaging;
|
2023-10-02 06:39:26 +13:00
|
|
|
use Appwrite\Event\Migration;
|
2023-05-30 01:58:45 +12:00
|
|
|
use Appwrite\Platform\Appwrite;
|
2023-08-21 00:29:43 +12:00
|
|
|
use Appwrite\Usage\Stats;
|
2022-11-10 06:01:43 +13:00
|
|
|
use Swoole\Runtime;
|
|
|
|
use Utopia\App;
|
|
|
|
use Utopia\Cache\Adapter\Sharding;
|
|
|
|
use Utopia\Cache\Cache;
|
2022-11-16 23:40:41 +13:00
|
|
|
use Utopia\CLI\Console;
|
2022-11-10 06:01:43 +13:00
|
|
|
use Utopia\Config\Config;
|
|
|
|
use Utopia\Database\Database;
|
2022-11-13 03:35:42 +13:00
|
|
|
use Utopia\Database\Document;
|
2023-08-24 07:08:26 +12:00
|
|
|
use Utopia\Database\Validator\Authorization;
|
2023-05-30 01:58:45 +12:00
|
|
|
use Utopia\Platform\Service;
|
2022-11-13 03:35:42 +13:00
|
|
|
use Utopia\Queue\Message;
|
2022-11-10 06:01:43 +13:00
|
|
|
use Utopia\Queue\Server;
|
|
|
|
use Utopia\Registry\Registry;
|
2022-11-16 23:40:41 +13:00
|
|
|
use Utopia\Logger\Log;
|
2022-11-17 02:34:11 +13:00
|
|
|
use Utopia\Logger\Logger;
|
2022-11-17 08:39:35 +13:00
|
|
|
use Utopia\Pools\Group;
|
2022-12-21 01:22:58 +13:00
|
|
|
use Utopia\Queue\Connection;
|
2022-11-16 23:40:41 +13:00
|
|
|
|
2023-05-30 01:58:45 +12:00
|
|
|
Authorization::disable();
|
2022-11-16 23:40:41 +13:00
|
|
|
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
|
2022-11-10 06:01:43 +13:00
|
|
|
|
2022-12-21 00:11:36 +13:00
|
|
|
Server::setResource('register', fn () => $register);
|
2022-11-10 06:01:43 +13:00
|
|
|
|
|
|
|
Server::setResource('dbForConsole', function (Cache $cache, Registry $register) {
|
|
|
|
$pools = $register->get('pools');
|
2022-11-16 17:17:46 +13:00
|
|
|
$database = $pools
|
2022-11-10 06:01:43 +13:00
|
|
|
->get('console')
|
|
|
|
->pop()
|
|
|
|
->getResource()
|
|
|
|
;
|
|
|
|
|
2022-11-16 17:17:46 +13:00
|
|
|
$adapter = new Database($database, $cache);
|
2023-08-17 11:34:54 +12:00
|
|
|
$adapter->setNamespace('_console');
|
2022-11-10 06:01:43 +13:00
|
|
|
|
2022-11-16 17:17:46 +13:00
|
|
|
return $adapter;
|
2022-11-10 06:01:43 +13:00
|
|
|
}, ['cache', 'register']);
|
|
|
|
|
2022-11-13 03:35:42 +13:00
|
|
|
Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) {
|
2022-11-16 17:17:46 +13:00
|
|
|
$payload = $message->getPayload() ?? [];
|
|
|
|
$project = new Document($payload['project'] ?? []);
|
2022-11-13 03:35:42 +13:00
|
|
|
|
|
|
|
if ($project->isEmpty() || $project->getId() === 'console') {
|
|
|
|
return $dbForConsole;
|
|
|
|
}
|
|
|
|
|
|
|
|
$pools = $register->get('pools');
|
2022-11-16 17:17:46 +13:00
|
|
|
$database = $pools
|
2022-11-13 03:35:42 +13:00
|
|
|
->get($project->getAttribute('database'))
|
|
|
|
->pop()
|
|
|
|
->getResource()
|
|
|
|
;
|
|
|
|
|
2022-11-16 17:17:46 +13:00
|
|
|
$adapter = new Database($database, $cache);
|
|
|
|
$adapter->setNamespace('_' . $project->getInternalId());
|
|
|
|
return $adapter;
|
2022-11-13 03:35:42 +13:00
|
|
|
}, ['cache', 'register', 'message', 'dbForConsole']);
|
|
|
|
|
2023-06-06 04:13:00 +12:00
|
|
|
Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
|
|
|
|
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
|
|
|
|
|
2023-10-02 06:39:26 +13:00
|
|
|
return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases): Database {
|
2023-06-06 04:13:00 +12:00
|
|
|
if ($project->isEmpty() || $project->getId() === 'console') {
|
|
|
|
return $dbForConsole;
|
|
|
|
}
|
|
|
|
|
|
|
|
$databaseName = $project->getAttribute('database');
|
|
|
|
|
|
|
|
if (isset($databases[$databaseName])) {
|
|
|
|
$database = $databases[$databaseName];
|
|
|
|
$database->setNamespace('_' . $project->getInternalId());
|
|
|
|
return $database;
|
|
|
|
}
|
|
|
|
|
|
|
|
$dbAdapter = $pools
|
|
|
|
->get($databaseName)
|
|
|
|
->pop()
|
|
|
|
->getResource();
|
|
|
|
|
|
|
|
$database = new Database($dbAdapter, $cache);
|
|
|
|
|
|
|
|
$databases[$databaseName] = $database;
|
|
|
|
|
|
|
|
$database->setNamespace('_' . $project->getInternalId());
|
|
|
|
|
|
|
|
return $database;
|
|
|
|
};
|
|
|
|
}, ['pools', 'dbForConsole', 'cache']);
|
|
|
|
|
2022-11-10 06:01:43 +13:00
|
|
|
Server::setResource('cache', function (Registry $register) {
|
|
|
|
$pools = $register->get('pools');
|
|
|
|
$list = Config::getParam('pools-cache', []);
|
|
|
|
$adapters = [];
|
|
|
|
|
|
|
|
foreach ($list as $value) {
|
|
|
|
$adapters[] = $pools
|
|
|
|
->get($value)
|
|
|
|
->pop()
|
|
|
|
->getResource()
|
|
|
|
;
|
|
|
|
}
|
|
|
|
|
|
|
|
return new Cache(new Sharding($adapters));
|
|
|
|
}, ['register']);
|
2023-07-11 16:39:59 +12:00
|
|
|
Server::setResource('log', fn() => new Log());
|
2023-10-17 16:44:36 +13:00
|
|
|
Server::setResource('usage', function ($register) {
|
|
|
|
return new Stats($register->get('statsd'));
|
2022-11-16 17:17:46 +13:00
|
|
|
}, ['register']);
|
2022-12-21 01:22:58 +13:00
|
|
|
Server::setResource('queue', function (Group $pools) {
|
|
|
|
return $pools->get('queue')->pop()->getResource();
|
|
|
|
}, ['pools']);
|
2023-06-06 04:13:00 +12:00
|
|
|
Server::setResource('queueForDatabase', function (Connection $queue) {
|
|
|
|
return new EventDatabase($queue);
|
|
|
|
}, ['queue']);
|
2022-12-21 05:11:30 +13:00
|
|
|
Server::setResource('queueForMessaging', function (Connection $queue) {
|
2023-10-20 22:25:17 +13:00
|
|
|
return new Messaging($queue);
|
2022-12-21 01:22:58 +13:00
|
|
|
}, ['queue']);
|
2023-06-12 02:08:48 +12:00
|
|
|
Server::setResource('queueForMails', function (Connection $queue) {
|
2022-12-21 01:22:58 +13:00
|
|
|
return new Mail($queue);
|
|
|
|
}, ['queue']);
|
2022-12-21 05:11:30 +13:00
|
|
|
Server::setResource('queueForBuilds', function (Connection $queue) {
|
2022-12-21 01:22:58 +13:00
|
|
|
return new Build($queue);
|
|
|
|
}, ['queue']);
|
2022-12-21 05:11:30 +13:00
|
|
|
Server::setResource('queueForDeletes', function (Connection $queue) {
|
2022-12-21 01:22:58 +13:00
|
|
|
return new Delete($queue);
|
|
|
|
}, ['queue']);
|
2022-12-21 05:11:30 +13:00
|
|
|
Server::setResource('queueForEvents', function (Connection $queue) {
|
2023-06-06 04:13:00 +12:00
|
|
|
return new Event($queue);
|
2022-12-21 01:22:58 +13:00
|
|
|
}, ['queue']);
|
2022-12-21 05:11:30 +13:00
|
|
|
Server::setResource('queueForAudits', function (Connection $queue) {
|
2022-12-21 01:22:58 +13:00
|
|
|
return new Audit($queue);
|
|
|
|
}, ['queue']);
|
|
|
|
Server::setResource('queueForFunctions', function (Connection $queue) {
|
|
|
|
return new Func($queue);
|
|
|
|
}, ['queue']);
|
2022-12-21 05:11:30 +13:00
|
|
|
Server::setResource('queueForCertificates', function (Connection $queue) {
|
2022-12-21 01:22:58 +13:00
|
|
|
return new Certificate($queue);
|
|
|
|
}, ['queue']);
|
2023-10-02 06:39:26 +13:00
|
|
|
Server::setResource('queueForMigrations', function (Connection $queue) {
|
|
|
|
return new Migration($queue);
|
|
|
|
}, ['queue']);
|
2023-11-28 23:19:55 +13:00
|
|
|
Server::setResource('queueForHamster', function (Connection $queue) {
|
|
|
|
return new Hamster($queue);
|
|
|
|
}, ['queue']);
|
2023-07-12 20:12:56 +12:00
|
|
|
Server::setResource('logger', function (Registry $register) {
|
2022-11-10 06:01:43 +13:00
|
|
|
return $register->get('logger');
|
2023-08-21 00:29:43 +12:00
|
|
|
}, ['register']);
|
2023-07-12 20:12:56 +12:00
|
|
|
Server::setResource('pools', function (Registry $register) {
|
2022-11-17 08:39:35 +13:00
|
|
|
return $register->get('pools');
|
|
|
|
}, ['register']);
|
2023-10-02 06:39:26 +13:00
|
|
|
Server::setResource('getFunctionsDevice', function () {
|
2023-06-11 22:29:04 +12:00
|
|
|
return function (string $projectId) {
|
|
|
|
return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
|
|
|
|
};
|
2023-05-24 01:43:03 +12:00
|
|
|
});
|
2023-10-02 06:39:26 +13:00
|
|
|
Server::setResource('getFilesDevice', function () {
|
2023-06-11 22:29:04 +12:00
|
|
|
return function (string $projectId) {
|
|
|
|
return getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
|
|
|
|
};
|
2023-05-24 01:43:03 +12:00
|
|
|
});
|
2023-10-02 06:39:26 +13:00
|
|
|
Server::setResource('getBuildsDevice', function () {
|
2023-06-11 22:29:04 +12:00
|
|
|
return function (string $projectId) {
|
|
|
|
return getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
|
|
|
|
};
|
|
|
|
});
|
2023-10-02 06:39:26 +13:00
|
|
|
Server::setResource('getCacheDevice', function () {
|
2023-06-11 22:29:04 +12:00
|
|
|
return function (string $projectId) {
|
|
|
|
return getDevice(APP_STORAGE_CACHE . '/app-' . $projectId);
|
|
|
|
};
|
2023-05-24 01:43:03 +12:00
|
|
|
});
|
2022-11-17 08:39:35 +13:00
|
|
|
|
2022-11-10 06:01:43 +13:00
|
|
|
$pools = $register->get('pools');
|
2023-05-30 01:58:45 +12:00
|
|
|
$platform = new Appwrite();
|
2023-05-30 20:53:52 +12:00
|
|
|
$args = $_SERVER['argv'];
|
2023-05-30 01:58:45 +12:00
|
|
|
|
2023-10-12 17:55:30 +13:00
|
|
|
if (!isset($args[1])) {
|
2023-06-02 15:54:34 +12:00
|
|
|
Console::error('Missing worker name');
|
2023-10-12 17:55:30 +13:00
|
|
|
Console::exit(1);
|
|
|
|
}
|
2023-02-06 10:06:32 +13:00
|
|
|
|
2023-10-12 17:55:30 +13:00
|
|
|
\array_shift($args);
|
|
|
|
$workerName = $args[0];
|
|
|
|
$workerIndex = $args[1] ?? '';
|
|
|
|
|
2023-10-17 16:44:36 +13:00
|
|
|
if (!empty($workerIndex)) {
|
2023-10-12 17:55:30 +13:00
|
|
|
$workerName .= '_' . $workerIndex;
|
2023-05-30 01:58:45 +12:00
|
|
|
}
|
|
|
|
|
2023-06-02 15:54:34 +12:00
|
|
|
try {
|
2023-10-12 19:11:08 +13:00
|
|
|
/**
|
|
|
|
* Any worker can be configured with the following env vars:
|
|
|
|
* - _APP_WORKERS_NUM The total number of worker processes
|
|
|
|
* - _APP_WORKER_PER_CORE The number of worker processes per core (ignored if _APP_WORKERS_NUM is set)
|
2023-10-12 20:15:16 +13:00
|
|
|
* - _APP_QUEUE_NAME The name of the queue to read for database events
|
2023-10-12 19:11:08 +13:00
|
|
|
*/
|
2023-10-18 02:55:08 +13:00
|
|
|
if ($workerName === 'databases') {
|
|
|
|
$queueName = App::getEnv('_APP_QUEUE_NAME', 'database_db_main');
|
|
|
|
} else {
|
|
|
|
$queueName = App::getEnv('_APP_QUEUE_NAME', 'v1-' . strtolower($workerName));
|
|
|
|
}
|
|
|
|
|
2023-06-02 15:54:34 +12:00
|
|
|
$platform->init(Service::TYPE_WORKER, [
|
2023-10-18 02:55:08 +13:00
|
|
|
'workersNum' => App::getEnv('_APP_WORKERS_NUM', 1),
|
2023-06-02 15:54:34 +12:00
|
|
|
'connection' => $pools->get('queue')->pop()->getResource(),
|
2023-06-05 04:25:56 +12:00
|
|
|
'workerName' => strtolower($workerName) ?? null,
|
2023-10-18 02:55:08 +13:00
|
|
|
'queueName' => $queueName
|
2023-06-02 15:54:34 +12:00
|
|
|
]);
|
|
|
|
} catch (\Exception $e) {
|
2023-06-04 20:19:49 +12:00
|
|
|
Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine());
|
2022-11-16 23:40:41 +13:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-05-30 01:58:45 +12:00
|
|
|
$worker = $platform->getWorker();
|
|
|
|
|
|
|
|
$worker
|
2022-11-17 08:39:35 +13:00
|
|
|
->shutdown()
|
|
|
|
->inject('pools')
|
|
|
|
->action(function (Group $pools) {
|
|
|
|
$pools->reclaim();
|
|
|
|
});
|
|
|
|
|
2023-05-30 01:58:45 +12:00
|
|
|
$worker
|
2022-11-16 23:40:41 +13:00
|
|
|
->error()
|
|
|
|
->inject('error')
|
|
|
|
->inject('logger')
|
2023-05-25 06:14:58 +12:00
|
|
|
->inject('log')
|
2023-07-12 19:17:40 +12:00
|
|
|
->action(function (Throwable $error, ?Logger $logger, Log $log) {
|
2022-11-16 23:40:41 +13:00
|
|
|
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
|
|
|
|
|
|
|
|
if ($error instanceof PDOException) {
|
|
|
|
throw $error;
|
|
|
|
}
|
|
|
|
|
2023-05-25 07:07:41 +12:00
|
|
|
if ($logger && ($error->getCode() >= 500 || $error->getCode() === 0)) {
|
2022-11-16 23:40:41 +13:00
|
|
|
$log->setNamespace("appwrite-worker");
|
|
|
|
$log->setServer(\gethostname());
|
|
|
|
$log->setVersion($version);
|
|
|
|
$log->setType(Log::TYPE_ERROR);
|
|
|
|
$log->setMessage($error->getMessage());
|
2022-11-17 02:34:11 +13:00
|
|
|
$log->setAction('appwrite-queue-' . App::getEnv('QUEUE'));
|
2022-11-16 23:40:41 +13:00
|
|
|
$log->addTag('verboseType', get_class($error));
|
|
|
|
$log->addTag('code', $error->getCode());
|
|
|
|
$log->addExtra('file', $error->getFile());
|
|
|
|
$log->addExtra('line', $error->getLine());
|
|
|
|
$log->addExtra('trace', $error->getTraceAsString());
|
|
|
|
$log->addExtra('detailedTrace', $error->getTrace());
|
2023-08-24 07:08:26 +12:00
|
|
|
$log->addExtra('roles', Authorization::getRoles());
|
2022-11-16 23:40:41 +13:00
|
|
|
|
|
|
|
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
|
|
|
|
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
|
|
|
|
|
2023-05-25 07:07:41 +12:00
|
|
|
$responseCode = $logger->addLog($log);
|
|
|
|
Console::info('Usage stats log pushed with status code: ' . $responseCode);
|
2022-11-16 23:40:41 +13:00
|
|
|
}
|
|
|
|
|
|
|
|
Console::error('[Error] Type: ' . get_class($error));
|
|
|
|
Console::error('[Error] Message: ' . $error->getMessage());
|
|
|
|
Console::error('[Error] File: ' . $error->getFile());
|
|
|
|
Console::error('[Error] Line: ' . $error->getLine());
|
2022-11-16 23:54:21 +13:00
|
|
|
});
|
2023-05-30 01:58:45 +12:00
|
|
|
|
2023-07-12 20:12:56 +12:00
|
|
|
$worker->workerStart()
|
|
|
|
->action(function () use ($workerName) {
|
|
|
|
Console::info("Worker $workerName started");
|
|
|
|
});
|
2023-07-10 20:07:54 +12:00
|
|
|
|
2023-10-17 16:44:36 +13:00
|
|
|
$worker->start();
|