2021-03-10 21:08:17 +13:00
|
|
|
<?php
|
|
|
|
|
|
|
|
namespace Appwrite\Resque;
|
|
|
|
|
2021-12-29 01:04:58 +13:00
|
|
|
use Utopia\App;
|
2021-07-26 11:14:19 +12:00
|
|
|
use Utopia\Cache\Cache;
|
|
|
|
use Utopia\Cache\Adapter\Redis as RedisCache;
|
|
|
|
use Utopia\CLI\Console;
|
|
|
|
use Utopia\Database\Database;
|
|
|
|
use Utopia\Database\Adapter\MariaDB;
|
2022-02-17 22:13:39 +13:00
|
|
|
use Utopia\Storage\Device;
|
|
|
|
use Utopia\Storage\Storage;
|
|
|
|
use Utopia\Storage\Device\Local;
|
|
|
|
use Utopia\Storage\Device\DOSpaces;
|
2022-03-19 05:03:04 +13:00
|
|
|
use Utopia\Storage\Device\Linode;
|
2022-03-19 06:17:43 +13:00
|
|
|
use Utopia\Storage\Device\Wasabi;
|
2022-05-10 23:15:56 +12:00
|
|
|
use Utopia\Storage\Device\Backblaze;
|
2022-02-17 22:13:39 +13:00
|
|
|
use Utopia\Storage\Device\S3;
|
2022-01-01 02:40:14 +13:00
|
|
|
use Exception;
|
2022-06-20 21:22:53 +12:00
|
|
|
use Utopia\Database\Validator\Authorization;
|
2022-02-17 22:13:39 +13:00
|
|
|
|
2021-06-12 02:20:18 +12:00
|
|
|
abstract class Worker
|
2021-03-10 21:08:17 +13:00
|
|
|
{
|
2021-12-01 01:12:17 +13:00
|
|
|
/**
|
|
|
|
* Callbacks that will be executed when an error occurs
|
|
|
|
*
|
|
|
|
* @var array
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
protected static array $errorCallbacks = [];
|
2021-12-01 01:12:17 +13:00
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
2021-12-06 00:08:08 +13:00
|
|
|
* Associative array holding all information passed into the worker
|
2021-11-24 23:09:10 +13:00
|
|
|
*
|
|
|
|
* @return array
|
|
|
|
*/
|
2021-09-01 21:09:04 +12:00
|
|
|
public array $args = [];
|
2021-03-10 21:08:17 +13:00
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* Function for identifying the worker needs to be set to unique name
|
|
|
|
*
|
|
|
|
* @return string
|
2021-12-06 02:07:45 +13:00
|
|
|
* @throws Exception
|
2021-11-24 23:09:10 +13:00
|
|
|
*/
|
2021-12-06 02:07:45 +13:00
|
|
|
public function getName(): string
|
|
|
|
{
|
|
|
|
throw new Exception("Please implement getName method in worker");
|
|
|
|
}
|
2021-11-24 03:24:25 +13:00
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* Function executed before running first task.
|
|
|
|
* Can include any preparations, such as connecting to external services or loading files
|
|
|
|
*
|
|
|
|
* @return void
|
|
|
|
* @throws \Exception|\Throwable
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
public function init()
|
|
|
|
{
|
2022-01-26 12:45:41 +13:00
|
|
|
throw new Exception("Please implement init method in worker");
|
2021-12-06 02:07:45 +13:00
|
|
|
}
|
2021-06-12 02:20:18 +12:00
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* Function executed when new task requests is received.
|
|
|
|
* You can access $args here, it will contain event information
|
|
|
|
*
|
|
|
|
* @return void
|
|
|
|
* @throws \Exception|\Throwable
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
public function run()
|
|
|
|
{
|
2022-01-26 12:45:41 +13:00
|
|
|
throw new Exception("Please implement run method in worker");
|
2021-12-06 02:07:45 +13:00
|
|
|
}
|
2021-06-12 02:20:18 +12:00
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* Function executed just before shutting down the worker.
|
|
|
|
* You can do cleanup here, such as disconnecting from services or removing temp files
|
|
|
|
*
|
|
|
|
* @return void
|
|
|
|
* @throws \Exception|\Throwable
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
public function shutdown()
|
|
|
|
{
|
2022-01-26 12:45:41 +13:00
|
|
|
throw new Exception("Please implement shutdown method in worker");
|
2021-12-06 02:07:45 +13:00
|
|
|
}
|
2021-03-10 21:08:17 +13:00
|
|
|
|
2022-06-03 01:03:37 +12:00
|
|
|
public const DATABASE_PROJECT = 'project';
|
|
|
|
public const DATABASE_CONSOLE = 'console';
|
2021-07-27 02:59:38 +12:00
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* A wrapper around 'init' function with non-worker-specific code
|
|
|
|
*
|
|
|
|
* @return void
|
|
|
|
* @throws \Exception|\Throwable
|
|
|
|
*/
|
2021-03-10 21:08:17 +13:00
|
|
|
public function setUp(): void
|
|
|
|
{
|
2021-11-24 03:24:25 +13:00
|
|
|
try {
|
|
|
|
$this->init();
|
2022-05-24 02:54:50 +12:00
|
|
|
} catch (\Throwable $error) {
|
2021-12-07 01:55:59 +13:00
|
|
|
foreach (self::$errorCallbacks as $errorCallback) {
|
|
|
|
$errorCallback($error, "init", $this->getName());
|
2021-11-26 03:13:14 +13:00
|
|
|
}
|
2021-11-24 03:24:25 +13:00
|
|
|
|
|
|
|
throw $error;
|
|
|
|
}
|
2021-03-10 21:08:17 +13:00
|
|
|
}
|
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* A wrapper around 'run' function with non-worker-specific code
|
|
|
|
*
|
|
|
|
* @return void
|
|
|
|
* @throws \Exception|\Throwable
|
|
|
|
*/
|
2021-09-01 21:09:04 +12:00
|
|
|
public function perform(): void
|
2021-03-10 21:08:17 +13:00
|
|
|
{
|
2021-11-24 03:24:25 +13:00
|
|
|
try {
|
2022-06-20 21:22:53 +12:00
|
|
|
/**
|
|
|
|
* Disabling global authorization in workers.
|
|
|
|
*/
|
|
|
|
Authorization::disable();
|
|
|
|
Authorization::setDefaultStatus(false);
|
2021-11-24 03:24:25 +13:00
|
|
|
$this->run();
|
2022-05-24 02:54:50 +12:00
|
|
|
} catch (\Throwable $error) {
|
2021-12-07 01:55:59 +13:00
|
|
|
foreach (self::$errorCallbacks as $errorCallback) {
|
|
|
|
$errorCallback($error, "run", $this->getName(), $this->args);
|
2021-11-26 03:13:14 +13:00
|
|
|
}
|
2021-11-24 03:24:25 +13:00
|
|
|
|
|
|
|
throw $error;
|
|
|
|
}
|
2021-03-10 21:08:17 +13:00
|
|
|
}
|
|
|
|
|
2021-11-24 23:09:10 +13:00
|
|
|
/**
|
|
|
|
* A wrapper around 'shutdown' function with non-worker-specific code
|
|
|
|
*
|
|
|
|
* @return void
|
|
|
|
* @throws \Exception|\Throwable
|
|
|
|
*/
|
2021-03-10 21:08:17 +13:00
|
|
|
public function tearDown(): void
|
|
|
|
{
|
2021-11-24 03:24:25 +13:00
|
|
|
try {
|
|
|
|
$this->shutdown();
|
2022-05-24 02:54:50 +12:00
|
|
|
} catch (\Throwable $error) {
|
2021-12-07 01:55:59 +13:00
|
|
|
foreach (self::$errorCallbacks as $errorCallback) {
|
|
|
|
$errorCallback($error, "shutdown", $this->getName());
|
2021-11-26 03:13:14 +13:00
|
|
|
}
|
2021-11-24 03:24:25 +13:00
|
|
|
|
|
|
|
throw $error;
|
|
|
|
}
|
2021-03-10 21:08:17 +13:00
|
|
|
}
|
2021-12-01 01:12:17 +13:00
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Register callback. Will be executed when error occurs.
|
|
|
|
* @param callable $callback
|
|
|
|
* @param Throwable $error
|
|
|
|
* @return self
|
|
|
|
*/
|
2021-12-06 02:07:45 +13:00
|
|
|
public static function error(callable $callback): void
|
2021-12-01 01:12:17 +13:00
|
|
|
{
|
2021-12-06 02:07:45 +13:00
|
|
|
\array_push(self::$errorCallbacks, $callback);
|
2021-12-01 01:12:17 +13:00
|
|
|
}
|
2021-07-26 11:14:19 +12:00
|
|
|
/**
|
|
|
|
* Get internal project database
|
|
|
|
* @param string $projectId
|
|
|
|
* @return Database
|
|
|
|
*/
|
2021-12-28 01:45:23 +13:00
|
|
|
protected function getProjectDB(string $projectId): Database
|
2021-07-26 11:14:19 +12:00
|
|
|
{
|
2022-06-20 21:22:53 +12:00
|
|
|
$consoleDB = $this->getConsoleDB();
|
|
|
|
|
|
|
|
if ($projectId === 'console') {
|
|
|
|
return $consoleDB;
|
|
|
|
}
|
|
|
|
|
|
|
|
/** @var Document $project */
|
|
|
|
$project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId));
|
|
|
|
|
|
|
|
return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId());
|
2021-07-26 11:14:19 +12:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get console database
|
|
|
|
* @return Database
|
|
|
|
*/
|
|
|
|
protected function getConsoleDB(): Database
|
2021-07-27 02:59:38 +12:00
|
|
|
{
|
|
|
|
return $this->getDB(self::DATABASE_CONSOLE);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get console database
|
|
|
|
* @param string $type One of (internal, external, console)
|
|
|
|
* @param string $projectId of internal or external DB
|
|
|
|
* @return Database
|
|
|
|
*/
|
2022-06-20 21:22:53 +12:00
|
|
|
private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database
|
2021-07-26 11:14:19 +12:00
|
|
|
{
|
|
|
|
global $register;
|
|
|
|
|
2021-07-27 02:59:38 +12:00
|
|
|
$namespace = '';
|
2022-01-22 06:32:27 +13:00
|
|
|
$sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary
|
2021-07-27 02:59:38 +12:00
|
|
|
|
|
|
|
switch ($type) {
|
2021-12-28 01:45:23 +13:00
|
|
|
case self::DATABASE_PROJECT:
|
2021-07-27 02:59:38 +12:00
|
|
|
if (!$projectId) {
|
|
|
|
throw new \Exception('ProjectID not provided - cannot get database');
|
|
|
|
}
|
2022-06-20 21:22:53 +12:00
|
|
|
$namespace = "_{$projectInternalId}";
|
2021-07-27 02:59:38 +12:00
|
|
|
break;
|
|
|
|
case self::DATABASE_CONSOLE:
|
2022-02-08 01:25:15 +13:00
|
|
|
$namespace = "_console";
|
2021-07-27 02:59:38 +12:00
|
|
|
$sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
throw new \Exception('Unknown database type: ' . $type);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2021-07-26 11:14:19 +12:00
|
|
|
$attempts = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
try {
|
|
|
|
$attempts++;
|
|
|
|
$cache = new Cache(new RedisCache($register->get('cache')));
|
2021-07-27 02:59:38 +12:00
|
|
|
$database = new Database(new MariaDB($register->get('db')), $cache);
|
2021-12-29 01:04:58 +13:00
|
|
|
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
|
2021-07-27 02:59:38 +12:00
|
|
|
$database->setNamespace($namespace); // Main DB
|
2022-01-22 06:32:27 +13:00
|
|
|
|
2021-12-23 14:51:49 +13:00
|
|
|
if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) {
|
|
|
|
throw new \Exception("Project does not exist: {$projectId}");
|
2021-07-26 11:14:19 +12:00
|
|
|
}
|
2022-01-22 06:32:27 +13:00
|
|
|
|
2022-05-13 06:05:11 +12:00
|
|
|
if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), '_metadata')) {
|
2022-01-22 06:32:27 +13:00
|
|
|
throw new \Exception('Console project not ready');
|
|
|
|
}
|
|
|
|
|
2021-07-26 11:14:19 +12:00
|
|
|
break; // leave loop if successful
|
2022-05-24 02:54:50 +12:00
|
|
|
} catch (\Exception $e) {
|
2021-07-26 11:14:19 +12:00
|
|
|
Console::warning("Database not ready. Retrying connection ({$attempts})...");
|
2022-01-22 06:32:27 +13:00
|
|
|
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
|
2022-05-24 02:54:50 +12:00
|
|
|
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
|
2021-07-26 11:14:19 +12:00
|
|
|
}
|
|
|
|
sleep($sleep);
|
|
|
|
}
|
2022-01-22 06:32:27 +13:00
|
|
|
} while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
|
2021-07-26 11:14:19 +12:00
|
|
|
|
2021-07-27 02:59:38 +12:00
|
|
|
return $database;
|
2021-07-26 11:14:19 +12:00
|
|
|
}
|
2022-02-17 22:13:39 +13:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Get Functions Storage Device
|
|
|
|
* @param string $projectId of the project
|
|
|
|
* @return Device
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
protected function getFunctionsDevice($projectId): Device
|
|
|
|
{
|
2022-02-17 22:13:39 +13:00
|
|
|
return $this->getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get Files Storage Device
|
|
|
|
* @param string $projectId of the project
|
|
|
|
* @return Device
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
protected function getFilesDevice($projectId): Device
|
|
|
|
{
|
2022-02-17 22:13:39 +13:00
|
|
|
return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get Builds Storage Device
|
|
|
|
* @param string $projectId of the project
|
|
|
|
* @return Device
|
|
|
|
*/
|
2022-05-24 02:54:50 +12:00
|
|
|
protected function getBuildsDevice($projectId): Device
|
|
|
|
{
|
2022-02-17 22:13:39 +13:00
|
|
|
return $this->getDevice(APP_STORAGE_BUILDS . '/app-' . $projectId);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get Device based on selected storage environment
|
|
|
|
* @param string $root path of the device
|
|
|
|
* @return Device
|
|
|
|
*/
|
2022-05-13 02:22:41 +12:00
|
|
|
public function getDevice($root): Device
|
2022-02-17 22:13:39 +13:00
|
|
|
{
|
|
|
|
switch (App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) {
|
2022-05-24 02:54:50 +12:00
|
|
|
case Storage::DEVICE_LOCAL:
|
|
|
|
default:
|
|
|
|
return new Local($root);
|
|
|
|
case Storage::DEVICE_S3:
|
|
|
|
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
|
|
|
|
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
|
|
|
|
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
|
|
|
|
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
|
|
|
|
$s3Acl = 'private';
|
|
|
|
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
|
|
|
|
case Storage::DEVICE_DO_SPACES:
|
|
|
|
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
|
|
|
|
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
|
|
|
|
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
|
|
|
|
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
|
|
|
|
$doSpacesAcl = 'private';
|
|
|
|
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
|
|
|
|
case Storage::DEVICE_BACKBLAZE:
|
|
|
|
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
|
|
|
|
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
|
|
|
|
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
|
|
|
|
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
|
|
|
|
$backblazeAcl = 'private';
|
|
|
|
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
|
|
|
|
case Storage::DEVICE_LINODE:
|
|
|
|
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
|
|
|
|
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
|
|
|
|
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
|
|
|
|
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
|
|
|
|
$linodeAcl = 'private';
|
|
|
|
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
|
|
|
|
case Storage::DEVICE_WASABI:
|
|
|
|
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
|
|
|
|
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
|
|
|
|
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
|
|
|
|
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
|
|
|
|
$wasabiAcl = 'private';
|
|
|
|
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
|
2022-02-17 22:13:39 +13:00
|
|
|
}
|
|
|
|
}
|
2021-03-10 21:08:17 +13:00
|
|
|
}
|