CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-01 18:39:57 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feat-db-pools-master' into feat-db-pools-sync

Browse source
This commit is contained in:
Matej Bačo 2023-03-14 12:58:54 +01:00
parent 70e4bd844e 46f07faa98
commit c01fcb130c
20 changed files with 1179 additions and 453 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env
View file

@ -41,6 +41,8 @@ _APP_SMTP_PORT=1025
_APP_SMTP_SECURE=
_APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_HAMSTER_RECIPIENTS=
_APP_HAMSTER_INTERVAL=86400
_APP_SMS_PROVIDER=sms://username:password@mock
_APP_SMS_FROM=+123456789
_APP_STORAGE_LIMIT=30000000

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = feat-cloud
branch = 2.2.2

1
Dockerfile
View file

@ -121,6 +121,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/sdks && \
chmod +x /usr/local/bin/specs && \
chmod +x /usr/local/bin/ssl && \
chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/test && \
chmod +x /usr/local/bin/vars && \
chmod +x /usr/local/bin/worker-audits && \

24
app/config/collections.php
View file

@ -2616,6 +2616,17 @@ $collections = [
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('endTime'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('duration'),
'type' => Database::VAR_INTEGER,
@ -2672,7 +2683,7 @@ $collections = [
'filters' => [],
],
[
'$id' => ID::custom('path'),
'$id' => ID::custom('outputPath'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 2048,
@ -2682,17 +2693,6 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('size'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('stderr'),
'type' => Database::VAR_STRING,

9
app/config/errors.php
View file

@ -88,6 +88,11 @@ return [
'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.',
'code' => 500,
],
Exception::GENERAL_CODES_DISABLED => [
'name' => Exception::GENERAL_CODES_DISABLED,
'description' => 'Invitation codes are disabled on this server. Please contact the server administrator.',
'code' => 500,
],
/** User Errors */
Exception::USER_COUNT_EXCEEDED => [
@ -125,8 +130,8 @@ return [
'description' => 'Console registration is restricted to specific emails. Contact your administrator for more information.',
'code' => 401,
],
Exception::USER_CODE_INVALID => [
'name' => Exception::USER_CODE_INVALID,
Exception::USER_INVALID_CODE => [
'name' => Exception::USER_INVALID_CODE,
'description' => 'The specified code is not valid. Contact your administrator for more information.',
'code' => 401,
],

8
app/controllers/api/account.php
View file

@ -81,8 +81,12 @@ App::post('/v1/account/invite')
$whitelistCodes = (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null)) : [];
if (empty($whitelistCodes)) {
throw new Exception(Exception::GENERAL_CODES_DISABLED);
}
if (!empty($whitelistCodes) && !\in_array($code, $whitelistCodes)) {
throw new Exception(Exception::USER_CODE_INVALID);
throw new Exception(Exception::USER_INVALID_CODE);
}
$limit = $project->getAttribute('auths', [])['limit'] ?? 0;
@ -168,7 +172,7 @@ App::post('/v1/account')
$whitelistEmails = $project->getAttribute('authWhitelistEmails');
$whitelistIPs = $project->getAttribute('authWhitelistIPs');
if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails)) {
if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails) && !\in_array(strtoupper($email), $whitelistEmails)) {
throw new Exception(Exception::USER_EMAIL_NOT_WHITELISTED);
}

2
app/controllers/api/functions.php
View file

@ -1163,7 +1163,7 @@ App::post('/v1/functions/:functionId/executions')
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $build->getAttribute('path', ''),
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);

14
app/workers/builds.php
View file

@ -104,13 +104,13 @@ class BuildsV1 extends Worker
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'status' => 'processing',
'path' => '',
'size' => 0,
'outputPath' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path'),
'sourceType' => $device,
'stdout' => '',
'stderr' => '',
'endTime' => null,
'duration' => 0
]));
$deployment->setAttribute('buildId', $build->getId());
@ -192,12 +192,14 @@ class BuildsV1 extends Worker
]
);
$endTime = new \DateTime();
$endTime->setTimestamp($response['endTimeUnix']);
/** Update the build document */
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
$build->setAttribute('endTime', DateTime::format($endTime));
$build->setAttribute('duration', \intval($response['duration']));
$build->setAttribute('status', $response['status']);
$build->setAttribute('path', $response['path']);
$build->setAttribute('size', $response['size']);
$build->setAttribute('outputPath', $response['outputPath']);
$build->setAttribute('stderr', $response['stderr']);
$build->setAttribute('stdout', $response['stdout']);
@ -229,7 +231,7 @@ class BuildsV1 extends Worker
} catch (\Throwable $th) {
$endTime = DateTime::now();
$interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', $interval->format('%s') + 0);
$build->setAttribute('status', 'failed');
$build->setAttribute('stderr', $th->getMessage());

12
app/workers/deletes.php
View file

@ -527,10 +527,10 @@ class DeletesV1 extends Worker
$this->deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) {
if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
} else {
Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
}
});
}
@ -576,10 +576,10 @@ class DeletesV1 extends Worker
$this->deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds) {
if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
} else {
Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
}
});

2
app/workers/functions.php
View file

@ -141,7 +141,7 @@ Server::setResource('execute', function () {
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $build->getAttribute('path', ''),
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);

3
bin/hamster Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php hamster $@

4
composer.json
View file

@ -72,7 +72,9 @@
"phpmailer/phpmailer": "6.6.0",
"chillerlan/php-qrcode": "4.3.3",
"adhocore/jwt": "1.1.2",
"webonyx/graphql-php": "14.11.*"
"webonyx/graphql-php": "14.11.*",
"slickdeals/statsd": "3.1.0",
"league/csv": "^9.0.0"
},
"repositories": [
{

1206
composer.lock generated
View file

File diff suppressed because it is too large Load diff

38
docker-compose.yml
View file

@ -54,7 +54,7 @@ services:
TESTING: true
VERSION: dev
VITE_CONSOLE_MODE: cloud
ports:
ports:
- 9501:80
networks:
- appwrite
@ -134,6 +134,7 @@ services:
- _APP_SMTP_SECURE
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_HAMSTER_RECIPIENTS
- _APP_USAGE_STATS
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
@ -542,6 +543,39 @@ services:
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-hamster:
entrypoint: hamster
<<: *x-logging
container_name: appwrite-hamster
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_HAMSTER_RECIPIENTS
- _APP_HAMSTER_INTERVAL
appwrite-maintenance:
entrypoint: maintenance
<<: *x-logging
@ -655,7 +689,7 @@ services:
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.2.0
image: openruntimes/executor:0.1.6
networks:
- appwrite
- runtimes

4
src/Appwrite/Extend/Exception.php
View file

@ -51,6 +51,7 @@ class Exception extends \Exception
public const GENERAL_CURSOR_NOT_FOUND = 'general_cursor_not_found';
public const GENERAL_SERVER_ERROR = 'general_server_error';
public const GENERAL_PROTOCOL_UNSUPPORTED = 'general_protocol_unsupported';
public const GENERAL_CODES_DISABLED = 'general_codes_disabled';
/** Users */
public const USER_COUNT_EXCEEDED = 'user_count_exceeded';
@ -61,7 +62,7 @@ class Exception extends \Exception
public const USER_PASSWORD_RESET_REQUIRED = 'user_password_reset_required';
public const USER_EMAIL_NOT_WHITELISTED = 'user_email_not_whitelisted';
public const USER_IP_NOT_WHITELISTED = 'user_ip_not_whitelisted';
public const USER_CODE_INVALID = 'user_code_invalid';
public const USER_INVALID_CODE = 'user_invalid_code';
public const USER_INVALID_CREDENTIALS = 'user_invalid_credentials';
public const USER_ANONYMOUS_CONSOLE_PROHIBITED = 'user_anonymous_console_prohibited';
public const USER_SESSION_ALREADY_EXISTS = 'user_session_already_exists';
@ -184,6 +185,7 @@ class Exception extends \Exception
public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries';
protected $type = '';
protected $errors = [];
public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
{

11
src/Appwrite/Migration/Version/V17.php
View file

@ -80,17 +80,8 @@ class V17 extends Migration
} catch (\Throwable $th) {
Console::warning("'mimeType' from {$id}: {$th->getMessage()}");
}
break;
case 'builds':
try {
/**
* Create 'size' attribute
*/
$this->createAttributeFromCollection($this->projectDB, $id, 'size');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'size' from {$id}: {$th->getMessage()}");
}
try {
/**

3
src/Appwrite/Platform/Services/Tasks.php
View file

@ -12,6 +12,8 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Hamster;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
@ -25,6 +27,7 @@ class Tasks extends Service
->addAction(Version::getName(), new Version())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
->addAction(Doctor::getName(), new Doctor())
->addAction(Install::getName(), new Install())
->addAction(Maintenance::getName(), new Maintenance())

271
src/Appwrite/Platform/Tasks/Hamster.php Normal file
View file

@ -0,0 +1,271 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Database\Document;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class Hamster extends Action
{
private array $columns = [
'Project ID',
'Project Name',
'Functions',
'Deployments',
'Members',
'Domains',
'Files',
'Buckets',
'Databases',
'Documents',
'Collections',
'Storage',
'Requests',
'Bandwidth',
'Users',
'Sessions',
'Executions'
];
private array $usageStats = [
'Files' => 'files.$all.count.total',
'Buckets' => 'buckets.$all.count.total',
'Databases' => 'databases.$all.count.total',
'Documents' => 'documents.$all.count.total',
'Collections' => 'collections.$all.count.total',
'Storage' => 'project.$all.storage.size',
'Requests' => 'project.$all.network.requests',
'Bandwidth' => 'project.$all.network.bandwidth',
'Users' => 'users.$all.count.total',
'Sessions' => 'sessions.$all.requests.create',
'Executions' => 'executions.$all.compute.total',
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'hamster';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('register')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(function (Registry $register, Group $pools, Cache $cache, Database $dbForConsole) {
$this->action($register, $pools, $cache, $dbForConsole);
});
}
private function getStats(Database $dbForConsole, Database $dbForProject, Document $project): array
{
$stats = [];
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Project Name */
$stats['Project Name'] = $project->getAttribute('name');
/** Get Total Functions */
$stats['Functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
/** Get Total Deployments */
$stats['Deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
/** Get Total Members */
$teamInternalId = $project->getAttribute('teamInternalId', null);
if ($teamInternalId) {
$stats['Members'] = $dbForConsole->count('memberships', [
Query::equal('teamInternalId', [$teamInternalId])
], APP_LIMIT_COUNT);
} else {
$stats['Members'] = 0;
}
/** Get Domains */
$stats['Domains'] = $dbForProject->count('domains', [], APP_LIMIT_COUNT);
/** Get Usage stats */
$range = '90d';
$periods = [
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = array_values($this->usageStats);
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
$stats[$metric] = array_reverse($stats[$metric]);
// Calculate aggregate of each metric
$stats[$metric] = array_sum(array_column($stats[$metric], 'value'));
}
});
return $stats;
}
public function action(Registry $register, Group $pools, Cache $cache, Database $dbForConsole): void
{
Console::title('Cloud Hamster V1');
Console::success(APP_NAME . ' cloud hamster process v1 has started');
$interval = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
Console::loop(function () use ($register, $pools, $cache, $dbForConsole, $interval) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Getting Cloud Usage Stats every {$interval} seconds");
$loopStart = microtime(true);
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects') + 1;
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$statsPerProject = $this->getStats($dbForConsole, $dbForProject, $project);
$csv->insertOne(array_values($statsPerProject));
} catch (\Throwable $th) {
throw $th;
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
Console::log('Iterated through ' . $count . '/' . $totalProjects . ' projects...');
}
$this->sendEmail($register);
$pools
->get('console')
->reclaim();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
}, $interval);
}
private function sendEmail(Registry $register)
{
/** @var \PHPMailer\PHPMailer\PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_HAMSTER_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}

12
src/Appwrite/Utopia/Response/Model/Build.php
View file

@ -51,18 +51,18 @@ class Build extends Model
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('endTime', [
'type' => self::TYPE_DATETIME,
'description' => 'The time the build was finished in ISO 8601 format.',
'default' => '',
'example' => self::TYPE_DATETIME_EXAMPLE,
])
->addRule('duration', [
'type' => self::TYPE_INTEGER,
'description' => 'The build duration in seconds.',
'default' => 0,
'example' => 0,
])
->addRule('size', [
'type' => self::TYPE_INTEGER,
'description' => 'The code size in bytes.',
'default' => 0,
'example' => 128,
])
;
}

4
tests/e2e/Services/Account/AccountConsoleClientTest.php
View file

@ -39,7 +39,7 @@ class AccountConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 401);
$this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID);
$this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE);
$response = $this->client->call(Client::METHOD_POST, '/account/invite', array_merge([
'origin' => 'http://localhost',
@ -53,7 +53,7 @@ class AccountConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 401);
$this->assertEquals($response['body']['type'], Exception::USER_CODE_INVALID);
$this->assertEquals($response['body']['type'], Exception::USER_INVALID_CODE);
/**
* Test for SUCCESS