CodeMirror/appwrite
1
0
Fork 0
mirror of synced 2024-06-20 19:50:30 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'cl-1.4.x' of https://github.com/appwrite/appwrite into 1.4.x

Browse source
This commit is contained in:
Christy Jacob 2023-08-08 20:51:31 +00:00
parent 88e1deefb5 c1167f7dc7
commit c009a12a25
204 changed files with 10540 additions and 9825 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
.env
View file

@ -1,9 +1,11 @@
_APP_ENV=development
_APP_LOCALE=en
_APP_WORKER_PER_CORE=6
_APP_WORKER_PER_CORE=2
_APP_CONSOLE_WHITELIST_ROOT=disabled
_APP_CONSOLE_WHITELIST_EMAILS=
_APP_CONSOLE_WHITELIST_CODES=code-zero,code-one
_APP_CONSOLE_WHITELIST_IPS=
_APP_CONSOLE_INVITES=enabled
_APP_CONSOLE_ROOT_SESSION=disabled
_APP_SYSTEM_EMAIL_NAME=Appwrite
_APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
@ -24,39 +26,26 @@ _APP_DB_SCHEMA=appwrite
_APP_DB_USER=user
_APP_DB_PASS=password
_APP_DB_ROOT_PASS=rootsecretpassword
_APP_STORAGE_DEVICE=Local
_APP_STORAGE_S3_ACCESS_KEY=
_APP_STORAGE_S3_SECRET=
_APP_STORAGE_S3_REGION=us-east-1
_APP_STORAGE_S3_BUCKET=
_APP_STORAGE_DO_SPACES_ACCESS_KEY=
_APP_STORAGE_DO_SPACES_SECRET=
_APP_STORAGE_DO_SPACES_REGION=us-east-1
_APP_STORAGE_DO_SPACES_BUCKET=
_APP_STORAGE_BACKBLAZE_ACCESS_KEY=
_APP_STORAGE_BACKBLAZE_SECRET=
_APP_STORAGE_BACKBLAZE_REGION=us-west-004
_APP_STORAGE_BACKBLAZE_BUCKET=
_APP_STORAGE_LINODE_ACCESS_KEY=
_APP_STORAGE_LINODE_SECRET=
_APP_STORAGE_LINODE_REGION=eu-central-1
_APP_STORAGE_LINODE_BUCKET=
_APP_STORAGE_WASABI_ACCESS_KEY=
_APP_STORAGE_WASABI_SECRET=
_APP_STORAGE_WASABI_REGION=eu-central-1
_APP_STORAGE_WASABI_BUCKET=
_APP_CONNECTIONS_MAX=3100
_APP_POOL_CLIENTS=14
_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
_APP_CONNECTIONS_STORAGE=local://localhost
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310
_APP_INFLUXDB_HOST=influxdb
_APP_INFLUXDB_PORT=8086
_APP_STATSD_HOST=telegraf
_APP_STATSD_PORT=8125
_APP_SMTP_HOST=maildev
_APP_SMTP_PORT=1025
_APP_SMTP_SECURE=
_APP_SMTP_USERNAME=
_APP_SMTP_PASSWORD=
_APP_USERS_STATS_RECIPIENTS=
_APP_HAMSTER_INTERVAL=86400
_APP_HAMSTER_TIME=21:00
_APP_MIXPANEL_TOKEN=
_APP_SMS_PROVIDER=sms://username:password@mock
_APP_SMS_FROM=+123456789
_APP_STORAGE_LIMIT=30000000
@ -64,28 +53,31 @@ _APP_STORAGE_PREVIEW_LIMIT=20000000
_APP_FUNCTIONS_SIZE_LIMIT=30000000
_APP_FUNCTIONS_TIMEOUT=900
_APP_FUNCTIONS_BUILD_TIMEOUT=900
_APP_FUNCTIONS_CONTAINERS=10
_APP_FUNCTIONS_CPUS=0
_APP_FUNCTIONS_MEMORY=0
_APP_FUNCTIONS_MEMORY_SWAP=0
_APP_FUNCTIONS_INACTIVE_THRESHOLD=60
OPEN_RUNTIMES_NETWORK=appwrite_runtimes
_APP_FUNCTIONS_CPUS=1
_APP_FUNCTIONS_MEMORY=512
_APP_FUNCTIONS_INACTIVE_THRESHOLD=600
_APP_FUNCTIONS_MAINTENANCE_INTERVAL=600
_APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes
_APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_HOST=http://appwrite-executor/v1
_APP_EXECUTOR_HOST=http://exc1/v1
_APP_FUNCTIONS_RUNTIMES=
_APP_MAINTENANCE_INTERVAL=86400
_APP_MAINTENANCE_RETENTION_CACHE=2592000
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_USAGE_AGGREGATION_INTERVAL=5
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_USAGE_STATS=enabled
_APP_USAGE_AGGREGATION_INTERVAL=30
_APP_LOGGING_PROVIDER=
_APP_LOGGING_CONFIG=
_APP_GRAPHQL_MAX_BATCH_SIZE=10
_APP_GRAPHQL_MAX_COMPLEXITY=250
_APP_GRAPHQL_MAX_DEPTH=3
DOCKERHUB_PULL_USERNAME=
DOCKERHUB_PULL_PASSWORD=
DOCKERHUB_PULL_EMAIL=
_APP_REGION=default
_APP_DOCKER_HUB_USERNAME=
_APP_DOCKER_HUB_PASSWORD=
_APP_CONSOLE_GITHUB_SECRET=
_APP_CONSOLE_GITHUB_APP_ID=
OPENAI_API_KEY=YOUR_OPENAI_API_KEY

46
.github/workflows/publish.yml vendored Normal file
View file

@ -0,0 +1,46 @@
name: "Build and Publish"
on:
push:
tags:
- appwrite-*
jobs:
build-publish:
name: Build and Publish
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 2
submodules: recursive
ref: feat-db-pools-master
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4
with:
images: appwrite/cloud
tags: |
type=ref,event=tag
- name: Build & Publish to DockerHub
uses: docker/build-push-action@v4
with:
context: .
platforms: linux/amd64
build-args: |
VERSION=${{ steps.meta.outputs.version }}
VITE_APPWRITE_GROWTH_ENDPOINT=https://growth.appwrite.io/v1
VITE_GA_PROJECT=G-L7G2B6PLDS
VITE_CONSOLE_MODE=cloud
push: true
tags: ${{ steps.meta.outputs.tags }}

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = 2.3.4
branch = disallow-personal-data

43
CHANGES.md
View file

@ -4,6 +4,7 @@
- Add error attribute to indexes and attributes [#4575](https://github.com/appwrite/appwrite/pull/4575)
- Add new index validation rules [#5710](https://github.com/appwrite/appwrite/pull/5710)
- Added support for disallowing passwords that contain personal data [#5371](https://github.com/appwrite/appwrite/pull/5371)
## Fixes
@ -17,8 +18,16 @@
- Improve permission indexes [DB #248](https://github.com/utopia-php/database/pull/248)
- Validators back-ported to Utopia [#5439](https://github.com/appwrite/appwrite/pull/5439)
# Version 1.3.8
## Changes
- Replace Appwrite executor with OpenRuntimes Executor [#4650](https://github.com/appwrite/appwrite/pull/4650)
- Add `_APP_CONNECTIONS_MAX` env var [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Increase Traefik TCP + file limits [#4673](https://github.com/appwrite/appwrite/pull/4673)
- Store build output file size [#4844](https://github.com/appwrite/appwrite/pull/4844)
## Bugs
- Fix audit user internal [#5809](https://github.com/appwrite/appwrite/pull/5809)
@ -112,6 +121,40 @@
- Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780)
- Fix french translation [#4782](https://github.com/appwrite/appwrite/pull/4782)
- Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
- New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770)
- Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
- Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
- Removed usage labels dependency.
- Dropped type attribute from stats collection.
- Usage metrics are processed via new usage worker.
- Metrics changes:
- Storage
- deprecated
- filesCreate, filesRead, filesUpdate, filesDelete, bucketsCreate, bucketsRead, bucketsUpdate, bucketsDelete.
- renamed
- filesCount to filesTotal, storage to filesStorage, bucketsCount to bucketsTotal.
- Auth
- deprecated
- usersCreate, usersRead, usersUpdate, usersDelete, sessionsCreate sessionsProviderCreate, sessionsDelete.
- renamed
- usersCount to usersTotal.
- added
- sessionsTotal.
- Databases
- deprecated
- databasesCreate, databasesRead, databasesDelete, documentsCreate, documentsRead, documentsUpdate, documentsDelete, collectionsCreate, collectionsRead, collectionsUpdate, collectionsDelete.
- renamed
- databasesCount to databasesTotal, collectionsCount to collectionsTotal, documentsCount to documentsTotal.
- Functions
- deprecated
- executionsFailure, executionsSuccess, buildsFailure, buildsSuccess, executionsFailure, executionsSuccess.
- renamed
- executionsTime to executionsCompute, buildsTime to buildsCompute, documentsCount to documentsTotal.
- added
- functionsTotal, buildsStorage, deploymentsTotal, deploymentsStorage.
- Project
- renamed
- executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal.
## Bugs
- Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776)

2
CONTRIBUTING.md
View file

@ -238,8 +238,6 @@ Appwrite stack is a combination of a variety of open-source technologies and too
- Redis - for managing cache and in-memory data (currently, we do not use Redis for persistent data).
- MariaDB - for database storage and queries.
- InfluxDB - for managing stats and time-series based data.
- Statsd - for sending data over UDP protocol (using Telegraf).
- ClamAV - for validating and scanning storage files.
- Imagemagick - for manipulating and managing image media files.
- Webp - for better compression of images on supporting clients.

58
Dockerfile
View file

@ -43,7 +43,6 @@ ENV DOCKER_COMPOSE_VERSION=v2.5.0
ENV _APP_SERVER=swoole \
_APP_ENV=production \
_APP_LOCALE=en \
_APP_WORKER_PER_CORE= \
_APP_DOMAIN=localhost \
_APP_DOMAIN_TARGET=localhost \
_APP_HOME=https://appwrite.io \
@ -63,27 +62,6 @@ ENV _APP_SERVER=swoole \
_APP_STORAGE_ANTIVIRUS=enabled \
_APP_STORAGE_ANTIVIRUS_HOST=clamav \
_APP_STORAGE_ANTIVIRUS_PORT=3310 \
_APP_STORAGE_DEVICE=Local \
_APP_STORAGE_S3_ACCESS_KEY= \
_APP_STORAGE_S3_SECRET= \
_APP_STORAGE_S3_REGION= \
_APP_STORAGE_S3_BUCKET= \
_APP_STORAGE_DO_SPACES_ACCESS_KEY= \
_APP_STORAGE_DO_SPACES_SECRET= \
_APP_STORAGE_DO_SPACES_REGION= \
_APP_STORAGE_DO_SPACES_BUCKET= \
_APP_STORAGE_BACKBLAZE_ACCESS_KEY= \
_APP_STORAGE_BACKBLAZE_SECRET= \
_APP_STORAGE_BACKBLAZE_REGION= \
_APP_STORAGE_BACKBLAZE_BUCKET= \
_APP_STORAGE_LINODE_ACCESS_KEY= \
_APP_STORAGE_LINODE_SECRET= \
_APP_STORAGE_LINODE_REGION= \
_APP_STORAGE_LINODE_BUCKET= \
_APP_STORAGE_WASABI_ACCESS_KEY= \
_APP_STORAGE_WASABI_SECRET= \
_APP_STORAGE_WASABI_REGION= \
_APP_STORAGE_WASABI_BUCKET= \
_APP_REDIS_HOST=redis \
_APP_REDIS_PORT=6379 \
_APP_DB_HOST=mariadb \
@ -91,39 +69,22 @@ ENV _APP_SERVER=swoole \
_APP_DB_USER=root \
_APP_DB_PASS=password \
_APP_DB_SCHEMA=appwrite \
_APP_INFLUXDB_HOST=influxdb \
_APP_INFLUXDB_PORT=8086 \
_APP_STATSD_HOST=telegraf \
_APP_STATSD_PORT=8125 \
_APP_SMTP_HOST= \
_APP_SMTP_PORT= \
_APP_SMTP_SECURE= \
_APP_SMTP_USERNAME= \
_APP_SMTP_PASSWORD= \
_APP_SMS_PROVIDER= \
_APP_SMS_FROM= \
_APP_FUNCTIONS_SIZE_LIMIT=30000000 \
_APP_FUNCTIONS_TIMEOUT=900 \
_APP_FUNCTIONS_CONTAINERS=10 \
_APP_FUNCTIONS_CPUS=1 \
_APP_FUNCTIONS_MEMORY=128 \
_APP_FUNCTIONS_MEMORY_SWAP=128 \
_APP_EXECUTOR_SECRET=a-random-secret \
_APP_EXECUTOR_HOST=http://appwrite-executor/v1 \
_APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes \
_APP_EXECUTOR_HOST=http://exc1/v1 \
_APP_SETUP=self-hosted \
_APP_VERSION=$VERSION \
_APP_USAGE_STATS=enabled \
_APP_USAGE_AGGREGATION_INTERVAL=30 \
# 14 Days = 1209600 s
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \
_APP_MAINTENANCE_RETENTION_AUDIT=1209600 \
# 1 Day = 86400 s
_APP_MAINTENANCE_RETENTION_ABUSE=86400 \
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \
_APP_MAINTENANCE_INTERVAL=86400 \
_APP_LOGGING_PROVIDER= \
_APP_LOGGING_CONFIG=
_APP_MAINTENANCE_INTERVAL=86400
RUN \
if [ "$DEBUG" == "true" ]; then \
@ -137,6 +98,7 @@ COPY --from=node /usr/local/src/console/build /usr/src/code/console
# Add Source Code
COPY ./app /usr/src/code/app
COPY ./public /usr/src/code/public
COPY ./bin /usr/local/bin
COPY ./docs /usr/src/code/docs
COPY ./src /usr/src/code/src
@ -157,16 +119,21 @@ RUN mkdir -p /storage/uploads && \
# Executables
RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/realtime && \
chmod +x /usr/local/bin/executor && \
chmod +x /usr/local/bin/schedule && \
chmod +x /usr/local/bin/sdks && \
chmod +x /usr/local/bin/specs && \
chmod +x /usr/local/bin/ssl && \
chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/test && \
chmod +x /usr/local/bin/vars && \
chmod +x /usr/local/bin/worker-audits && \
@ -177,7 +144,8 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-builds && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-usage
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

172
app/cli.php
View file

@ -3,30 +3,170 @@
require_once __DIR__ . '/init.php';
require_once __DIR__ . '/controllers/general.php';
use Utopia\App;
use Appwrite\Event\Func;
use Appwrite\Platform\Appwrite;
use Utopia\CLI\CLI;
use Utopia\CLI\Console;
use Utopia\Database\Validator\Authorization;
use Utopia\Platform\Service;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Logger\Log;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
Authorization::disable();
$cli = new CLI();
CLI::setResource('register', fn()=>$register);
include 'tasks/doctor.php';
include 'tasks/maintenance.php';
include 'tasks/install.php';
include 'tasks/migrate.php';
include 'tasks/sdks.php';
include 'tasks/specs.php';
include 'tasks/ssl.php';
include 'tasks/vars.php';
include 'tasks/usage.php';
CLI::setResource('cache', function ($pools) {
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}, ['pools']);
CLI::setResource('pools', function (Registry $register) {
return $register->get('pools');
}, ['register']);
CLI::setResource('dbForConsole', function ($pools, $cache) {
$sleep = 3;
$maxAttempts = 5;
$attempts = 0;
$ready = false;
do {
$attempts++;
try {
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setNamespace('console');
// Ensure tables exist
$collections = Config::getParam('collections', [])['console'];
$last = \array_key_last($collections);
if (!($dbForConsole->exists($dbForConsole->getDefaultDatabase(), $last))) { /** TODO cache ready variable using registry */
throw new Exception('Tables not ready yet.');
}
$ready = true;
} catch (\Exception $err) {
Console::warning($err->getMessage());
$pools->get('console')->reclaim();
sleep($sleep);
}
} while ($attempts < $maxAttempts);
if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");
}
return $dbForConsole;
}, ['pools', 'cache']);
CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
$getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) {
if ($project->isEmpty() || $project->getId() === 'console') {
return $dbForConsole;
}
$databaseName = $project->getAttribute('database');
if (isset($databases[$databaseName])) {
$database = $databases[$databaseName];
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get($databaseName)
->pop()
->getResource();
$database = new Database($dbAdapter, $cache);
$databases[$databaseName] = $database;
$database->setNamespace('_' . $project->getInternalId());
return $database;
};
return $getProjectDB;
}, ['pools', 'dbForConsole', 'cache']);
CLI::setResource('queueForFunctions', function (Group $pools) {
return new Func($pools->get('queue')->pop()->getResource());
}, ['pools']);
CLI::setResource('logError', function (Registry $register) {
return function (Throwable $error, string $namespace, string $action) use ($register) {
$logger = $register->get('logger');
if ($logger) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$log = new Log();
$log->setNamespace($namespace);
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('code', $error->getCode());
$log->addTag('verboseType', get_class($error));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Usage stats log pushed with status code: ' . $responseCode);
}
Console::warning("Failed: {$error->getMessage()}");
Console::warning($error->getTraceAsString());
};
}, ['register']);
$platform = new Appwrite();
$platform->init(Service::TYPE_CLI);
$cli = $platform->getCli();
$cli
->task('version')
->desc('Get the server version')
->action(function () {
Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN'));
->error()
->inject('error')
->action(function (Throwable $error) {
Console::error($error->getMessage());
});
$cli->run();

1
app/config/cloud/contributors.json Normal file
View file

@ -0,0 +1 @@
[1297371,1759475,6360216,20852629,5857008,19310830,9708641,26739219,23742426,22174310,77877486,1477010,29069505,62933155,45863583,42211,176163,58045728,7091609,31401437,1911066,66096031,22432834,43054051,79051850,22895284,100597998,91385411,49699333,7818620,11004008,54898623,27856297,33250853,49375670,30630364,50206,47822499,7481165,52557347,13681567,20492520,51369094,51821861,50256986,28431370,13692220,24373771,15938422,27148250,14805534,36137226,36071208,40193621,56179878,53281158,48085134,19358691,32528768,19733683,37348419,13719696,18309412,42106787,53618500,5306011,4408379,35486736,18586611,15062564,27011453,56090587,44623032,25107942,81188,33922418,16717633,32809211,69401139,25815659,4104127,20889958,3102249,12476526,2635185,28495651,50957556,2791280,31023616,20955511,835733,471907,69008866,44906587,45271396,76054330,45748739,41908747,23402178,47356149,8921,36632821,3668741,71702982,29725587,26303198,785830,51410502,60089135,2847349,43172716,48546075,8216525,41161981,51828039,4334997,80918302,38534289,47860497,80036766,41341387,49818988,58487637,29237374,46913894,5148229,4377199,29686102,26272249,75117692,4090256,27357868,33062368,38664231,46695441,743291,22633385,6368283,11593067,45097959,43381712,3284228,1972717,33012425,61755381,25405707,3144291,44156359,5497267,7423905,20716175,28586681,5975506,23518097,22187384,24191952,7768078,971530,51240166,55633427,34207400,77061285,11719476,35950229,66742927,34406802,802933,50047839,39148877,26602940,9693472,44273767,19362725,31209978,30521594,686298,6237394,35039730,42580581,36671793,8502129,8466918,81866614,54903252,28373606,13381361,72331432,30694270,5355510,8209163,86675510,9453522,42496309,56145786,2149381,393945,22084723,52621436,8872447,5575392,29619660,5547479,8852116,11151445,4717349,17725274,65615065,18537755,29292618,53044263,26597930,10313411,55998629,77529288,17404636,33729848,19422168,17916404,66111735,10329006,33502846,398230,81643826,105039167,47522632,91655303,9774614,10603631,284924,60857954,22885912,116552306,36103454,794606,27729549,1754457,36594527,13899668,78664749,47406531,27698189,5305654,53345517,6756412,29176704,77790497,47504894,37251540,52361778,52200375,1351177,66022861,73975409,25745396,31433638,37118134,43210805,20317665,11923975,47187468,16362381,36751163,14959876,32362757,65529384,52352285,74085816,3628535,43902034,75667593,26132902,466713,617558,96806061,33605526,11290524,43621940,12446314,17146935,55018955,56096559,79797000,40014186,34449936,58387964,23368207,42414965,44056349,33743031,12294525,58251592,33755729,9021747,932084,11428067,97121933,80122730,60894542,58583793,56051809,32243289,9934371,90936802,74638775,65399526,77604,64524822,47782249,43633955,42793632,55969597,72334601,82395440,92818577,60866204,65016769,23725091,45892107,55308895,86314140,82756460,47685349,63562160,73419211,1613216,50882624,91469717,46166258,60927324,41763158,83607556,2171717,50497814,39427312,61322830,40076195,39419448,29397545,55090719,53259730,20885012,64558515,69677883,55741087,72426535,46033036,68477507,30376878,73700530,25518600,29922887,36229969,47573417,40424087,49054503,16880385,22801227,72848513,64347914,814402,49149679,55017867,49481876,67067955,31439735,63878173,80322286,43746210,17332970,22702905,62476876,89888292,75736952,54059881,90782137,63588969,57111920,63330165,70258211,46371923,17837758,59364507,52203828,60147326,18481195,74822422,9803078,67309607,60410049,47360939,19922556,90848252,24698014,58886915,63579762,96648934,68523530,60518745,37345795,3929651,54993657,52061363,43019989,5787917,94674993,71593494,17143469,10288548,1830380,71510505,59124772,2335145,70798495,46474346,49263351,52062536,63151043,65248303,26071571,53626355,43992469,60785452,63467479,71837281,19490891,58628586,38250310,7271718,1110414,57227290,11625672,85063520,88965873,70096901,42029519,85363195,64471630,69353350,66922161,2221746,100430077,12299813,62690310,68282006,99184676,2450,22989561,22212661,59973863,11232940,76688923,22321353,77732479,84286404,32268377,34828782,23068019,57074509,24620969,20735983,26173690,75809937,49760818,86646105,52617262]

33
app/config/cloud/employees.json Normal file
View file

@ -0,0 +1,33 @@
{
"eldad@appwrite.io": { "memberSince": "2020-10-15", "spot": "0", "gitHub": "eldadfux" },
"christy@appwrite.io": { "memberSince": "2020-12-01", "spot": "1", "gitHub": "christyjacob4" },
"torsten@appwrite.io": { "memberSince": "2020-12-28", "spot": "2", "gitHub": "torstendittmann" },
"damodar@appwrite.io": { "memberSince": "2021-01-02", "spot": "3", "gitHub": "lohanidamodar" },
"bradley@appwrite.io": { "memberSince": "2021-05-21", "spot": "5", "gitHub": "PineappleIOnic" },
"jake@appwrite.io": { "memberSince": "2021-06-28", "spot": "6", "gitHub": "abnegate" },
"sara@appwrite.io": { "memberSince": "2021-08-16", "spot": "7", "gitHub": "sarakaandorp" },
"matej@appwrite.io": { "memberSince": "2021-08-23", "spot": "8", "gitHub": "meldiron" },
"aditya@appwrite.io": { "memberSince": "2021-09-01", "spot": "9", "gitHub": "adityaoberai" },
"wess@appwrite.io": { "memberSince": "2021-11-08", "spot": "12", "gitHub": "wess" },
"may@appwrite.io": { "memberSince": "2021-11-28", "spot": "14", "gitHub": "MayEnder" },
"elad@appwrite.io": { "memberSince": "2021-12-19", "spot": "15", "gitHub": "elad2412" },
"vincent@appwrite.io": { "memberSince": "2022-01-01", "spot": "16", "gitHub": "gewenyu99" },
"haimantika@appwrite.io": { "memberSince": "2022-04-01", "spot": "18", "gitHub": "Haimantika" },
"chen@appwrite.io": { "memberSince": "2022-01-24", "spot": "19", "gitHub": "chenparnasa" },
"tessa@appwrite.io": { "memberSince": "2022-04-21", "spot": "20", "gitHub": "tessamero" },
"shimon@appwrite.io": { "memberSince": "2022-05-01", "spot": "23", "gitHub": "shimonewman" },
"shmuel@appwrite.io": { "memberSince": "2022-03-20", "spot": "24", "gitHub": "fogelito" },
"arman@appwrite.io": { "memberSince": "2022-04-04", "spot": "25", "gitHub": "ArmanNik" },
"carla@appwrite.io": { "memberSince": "2022-04-04", "spot": "26", "gitHub": "heyCarla" },
"emma@appwrite.io": { "memberSince": "2022-05-08", "spot": "27", "gitHub": "emmacarpagnano1" },
"dylan@appwrite.io": { "memberSince": "2022-05-09", "spot": "28", "gitHub": "DylanG-64" },
"steven@appwrite.io": { "memberSince": "2022-07-01", "spot": "30", "gitHub": "stnguyen90" },
"jyoti@appwrite.io": { "memberSince": "2022-10-24", "spot": "31", "gitHub": "joeyouss" },
"jade@appwrite.io": { "memberSince": "2022-10-31", "spot": "32", "gitHub": "dajebp" },
"khushboo@appwrite.io": { "memberSince": "2021-11-08", "spot": "13", "gitHub": "vermakhushboo" },
"thomas@appwrite.io": { "memberSince": "2022-11-03", "spot": "34", "gitHub": "TGlide" },
"holly@appwrite.io": { "memberSince": "2022-12-05", "spot": "35", "gitHub": "HollyBarclay" },
"laura@appwrite.io": { "memberSince": "2023-01-25", "spot": "36", "gitHub": "LauraDuRy" },
"caio@appwrite.io": { "memberSince": "2023-03-27", "spot": "37", "gitHub": "ariascaio" },
"luke@appwrite.io": { "memberSince": "2023-05-04", "spot": "38", "gitHub": "loks0n" }
}

9
app/config/cloud/heroes.json Normal file
View file

@ -0,0 +1,9 @@
{
"bishwajeet.techmaster@gmail.com": { "memberSince": "2023-02-07" },
"lucasaudart@gmail.com": { "memberSince": "2023-02-07" },
"tkarmakar27112000@gmail.com": { "memberSince": "2023-02-07" },
"alves.mckl@gmail.com": { "memberSince": "2023-02-07" },
"dpns_nampula@rnlay.com": { "memberSince": "2023-02-07" },
"a.stephensimon@outlook.com": { "memberSince": "2023-02-07" },
"hidianapham@gmail.com": { "memberSince": "2023-02-07" }
}

3530
app/config/collections.php
View file

File diff suppressed because it is too large Load diff

27
app/config/errors.php
View file

@ -88,6 +88,11 @@ return [
'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.',
'code' => 500,
],
Exception::GENERAL_CODES_DISABLED => [
'name' => Exception::GENERAL_CODES_DISABLED,
'description' => 'Invitation codes are disabled on this server. Please contact the server administrator.',
'code' => 500,
],
Exception::GENERAL_USAGE_DISABLED => [
'name' => Exception::GENERAL_USAGE_DISABLED,
'description' => 'Usage stats is not configured. Please check the value of the _APP_USAGE_STATS environment variable of your Appwrite server.',
@ -130,6 +135,11 @@ return [
'description' => 'Console registration is restricted to specific emails. Contact your administrator for more information.',
'code' => 401,
],
Exception::USER_INVALID_CODE => [
'name' => Exception::USER_INVALID_CODE,
'description' => 'The specified code is not valid. Contact your administrator for more information.',
'code' => 401,
],
Exception::USER_IP_NOT_WHITELISTED => [
'name' => Exception::USER_IP_NOT_WHITELISTED,
'description' => 'Console registration is restricted to specific IPs. Contact your administrator for more information.',
@ -165,6 +175,16 @@ return [
'description' => 'Passwords do not match. Please check the password and confirm password.',
'code' => 400,
],
Exception::USER_PASSWORD_RECENTLY_USED => [
'name' => Exception::USER_PASSWORD_RECENTLY_USED,
'description' => 'The password you are trying to use is similar to your previous password. Please choose a stronger password.',
'code' => 400,
],
Exception::USER_PASSWORD_PERSONAL_DATA => [
'name' => Exception::USER_PASSWORD_PERSONAL_DATA,
'description' => 'The password you are trying to use contains references to your name, email, phone or userID. Please choose a different password and try again.',
'code' => 400,
],
Exception::USER_SESSION_NOT_FOUND => [
'name' => Exception::USER_SESSION_NOT_FOUND,
'description' => 'The current user session could not be found.',
@ -596,9 +616,14 @@ return [
],
Exception::DOMAIN_ALREADY_EXISTS => [
'name' => Exception::DOMAIN_ALREADY_EXISTS,
'description' => 'A Domain with the requested ID already exists.',
'description' => 'The requested domain is currently in use by a project.',
'code' => 409,
],
Exception::DOMAIN_FORBIDDEN => [
'name' => Exception::DOMAIN_FORBIDDEN,
'description' => 'The requested domain cannot be used as a custom domain.',
'code' => 403,
],
Exception::VARIABLE_NOT_FOUND => [
'name' => Exception::VARIABLE_NOT_FOUND,
'description' => 'Variable with the requested ID could not be found.',

13
app/config/services.php
View file

@ -108,6 +108,19 @@ return [
'optional' => false,
'icon' => '',
],
'project' => [
'key' => 'project',
'name' => 'Project',
'subtitle' => 'The Project service allows you to manage all the projects in your Appwrite server.',
'description' => '',
'controller' => 'api/project.php',
'sdk' => true,
'docs' => true,
'docsUrl' => '',
'tests' => false,
'optional' => false,
'icon' => '',
],
'storage' => [
'key' => 'storage',
'name' => 'Storage',

1
app/config/specs/open-api3-1.4.x-client.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/open-api3-1.4.x-console.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/open-api3-1.4.x-server.json Normal file
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/open-api3-latest-server.json
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/swagger2-1.4.x-client.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/swagger2-1.4.x-console.json Normal file
View file

File diff suppressed because one or more lines are too long

1
app/config/specs/swagger2-1.4.x-server.json Normal file
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-client.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-console.json
View file

File diff suppressed because one or more lines are too long

2
app/config/specs/swagger2-latest-server.json
View file

File diff suppressed because one or more lines are too long

24
app/config/usage.php Normal file
View file

@ -0,0 +1,24 @@
<?php
return [
'24h' => [
'period' => '1h',
'limit' => 24,
'factor' => 3600,
],
'7d' => [
'period' => '1d',
'limit' => 7,
'factor' => 86400,
],
'30d' => [
'period' => '1d',
'limit' => 30,
'factor' => 86400,
],
'90d' => [
'period' => '1d',
'limit' => 90,
'factor' => 86400,
],
];

185
app/config/variables.php
View file

@ -152,7 +152,7 @@ return [
],
[
'name' => '_APP_USAGE_STATS',
'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage, Influxdb and Telegraf containers for better resource usage.',
'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage container for better resource usage.',
'introduction' => '0.7.0',
'default' => 'enabled',
'required' => false,
@ -212,7 +212,16 @@ return [
'required' => false,
'question' => '',
'filter' => ''
]
],
[
'name' => '_APP_CONSOLE_INVITES',
'description' => 'This option allows you to disable the invitation of new users to the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.',
'introduction' => '1.2.0',
'default' => 'enabled',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
@ -315,54 +324,33 @@ return [
'question' => '',
'filter' => 'password'
],
],
],
[
'category' => 'InfluxDB',
'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
'variables' => [
[
'name' => '_APP_INFLUXDB_HOST',
'description' => 'InfluxDB server host name address. Default value is: \'influxdb\'.',
'introduction' => '',
'default' => 'influxdb',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_INFLUXDB_PORT',
'description' => 'InfluxDB server TCP port. Default value is: \'8086\'.',
'introduction' => '',
'default' => '8086',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
'category' => 'StatsD',
'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
'variables' => [
[
'name' => '_APP_STATSD_HOST',
'description' => 'StatsD server host name address. Default value is: \'telegraf\'.',
'introduction' => '',
'default' => 'telegraf',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_STATSD_PORT',
'description' => 'StatsD server TCP port. Default value is: \'8125\'.',
'introduction' => '',
'default' => '8125',
'name' => '_APP_CONNECTIONS_MAX',
'description' => 'MariaDB server maximum connections.',
'introduction' => 'TBD',
'default' => 251,
'required' => false,
'question' => '',
'filter' => ''
],
// [
// 'name' => '_APP_CONNECTIONS_DB_PROJECT',
// 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ],
// [
// 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
// 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
// 'introduction' => 'TBD',
// 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
// 'required' => true,
// 'question' => '',
// 'filter' => ''
// ]
],
],
[
@ -489,9 +477,17 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_CONNECTIONS_STORAGE',
'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format <device>://<access_key>:<access_secret>@<host>:<port>/<bucket>?region=<region>. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are local, s3, dospaces, linode, backblaze and wasabi.',
'introduction' => '1.1.0',
'default' => '',
'required' => false,
'question' => '',
],
[
'name' => '_APP_STORAGE_DEVICE',
'description' => 'Select default storage device. The default value is \'local\'. List of supported adapters are \'local\', \'s3\', \'dospaces\', \'backblaze\', \'linode\' and \'wasabi\'.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => 'local',
'required' => false,
@ -499,7 +495,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_ACCESS_KEY',
'description' => 'AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -507,7 +503,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_SECRET',
'description' => 'AWS S3 storage secret key. Required when the storage adapter is set to S3. You can get your secret key from your AWS console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -515,7 +511,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_REGION',
'description' => 'AWS S3 storage region. Required when storage adapter is set to S3. You can find your region info for your bucket from AWS console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => 'us-east-1',
'required' => false,
@ -523,7 +519,7 @@ return [
],
[
'name' => '_APP_STORAGE_S3_BUCKET',
'description' => 'AWS S3 storage bucket. Required when storage adapter is set to S3. You can create buckets in your AWS console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -531,7 +527,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_ACCESS_KEY',
'description' => 'DigitalOcean spaces access key. Required when the storage adapter is set to DOSpaces. You can get your access key from your DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -539,7 +535,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_SECRET',
'description' => 'DigitalOcean spaces secret key. Required when the storage adapter is set to DOSpaces. You can get your secret key from your DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -547,7 +543,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_REGION',
'description' => 'DigitalOcean spaces region. Required when storage adapter is set to DOSpaces. You can find your region info for your space from DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => 'us-east-1',
'required' => false,
@ -555,7 +551,7 @@ return [
],
[
'name' => '_APP_STORAGE_DO_SPACES_BUCKET',
'description' => 'DigitalOcean spaces bucket. Required when storage adapter is set to DOSpaces. You can create spaces in your DigitalOcean console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.13.0',
'default' => '',
'required' => false,
@ -563,7 +559,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY',
'description' => 'Backblaze access key. Required when the storage adapter is set to Backblaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -571,7 +567,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_SECRET',
'description' => 'Backblaze secret key. Required when the storage adapter is set to Backblaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -579,7 +575,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_REGION',
'description' => 'Backblaze region. Required when storage adapter is set to Backblaze. You can find your region info from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => 'us-west-004',
'required' => false,
@ -587,7 +583,7 @@ return [
],
[
'name' => '_APP_STORAGE_BACKBLAZE_BUCKET',
'description' => 'Backblaze bucket. Required when storage adapter is set to Backblaze. You can create your bucket from your Backblaze console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -595,7 +591,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_ACCESS_KEY',
'description' => 'Linode object storage access key. Required when the storage adapter is set to Linode. You can get your access key from your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -603,7 +599,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_SECRET',
'description' => 'Linode object storage secret key. Required when the storage adapter is set to Linode. You can get your secret key from your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -611,7 +607,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_REGION',
'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => 'eu-central-1',
'required' => false,
@ -619,7 +615,7 @@ return [
],
[
'name' => '_APP_STORAGE_LINODE_BUCKET',
'description' => 'Linode object storage bucket. Required when storage adapter is set to Linode. You can create buckets in your Linode console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -627,7 +623,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_ACCESS_KEY',
'description' => 'Wasabi access key. Required when the storage adapter is set to Wasabi. You can get your access key from your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -635,7 +631,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_SECRET',
'description' => 'Wasabi secret key. Required when the storage adapter is set to Wasabi. You can get your secret key from your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -643,7 +639,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_REGION',
'description' => 'Wasabi region. Required when storage adapter is set to Wasabi. You can find your region info from your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => 'eu-central-1',
'required' => false,
@ -651,7 +647,7 @@ return [
],
[
'name' => '_APP_STORAGE_WASABI_BUCKET',
'description' => 'Wasabi bucket. Required when storage adapter is set to Wasabi. You can create buckets in your Wasabi console.',
'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
'introduction' => '0.14.2',
'default' => '',
'required' => false,
@ -692,7 +688,7 @@ return [
],
[
'name' => '_APP_FUNCTIONS_CONTAINERS',
'description' => 'The maximum number of containers Appwrite is allowed to keep alive in the background for function environments. Running containers allow faster execution time as there is no need to recreate each container every time a function gets executed. The default value is 10.',
'description' => 'Deprecated since 1.2.0. Runtimes now timeout by inactivity using \'_APP_FUNCTIONS_INACTIVE_THRESHOLD\'.',
'introduction' => '0.7.0',
'default' => '10',
'required' => false,
@ -719,7 +715,7 @@ return [
],
[
'name' => '_APP_FUNCTIONS_MEMORY_SWAP',
'description' => 'The maximum amount of swap memory a single cloud function is allowed to use in megabytes. The default value is empty. When it\'s empty, swap memory limit will be disabled.',
'description' => 'Deprecated since 1.2.0. High use of swap memory is not recommended to preserve harddrive health.',
'introduction' => '0.7.0',
'default' => '0',
'required' => false,
@ -782,7 +778,7 @@ return [
],
[
'name' => 'DOCKERHUB_PULL_USERNAME',
'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'description' => 'Deprecated with 1.2.0, use \'_APP_DOCKER_HUB_USERNAME\' instead!',
'introduction' => '0.10.0',
'default' => '',
'required' => false,
@ -791,7 +787,7 @@ return [
],
[
'name' => 'DOCKERHUB_PULL_PASSWORD',
'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'description' => 'Deprecated with 1.2.0, use \'_APP_DOCKER_HUB_PASSWORD\' instead!',
'introduction' => '0.10.0',
'default' => '',
'required' => false,
@ -800,7 +796,7 @@ return [
],
[
'name' => 'DOCKERHUB_PULL_EMAIL',
'description' => 'The email for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'description' => 'Deprecated since 1.2.0. Email is no longer needed.',
'introduction' => '0.10.0',
'default' => '',
'required' => false,
@ -809,13 +805,49 @@ return [
],
[
'name' => 'OPEN_RUNTIMES_NETWORK',
'description' => 'The docker network used for communication between the executor and runtimes. Change this if you have altered the default network names.',
'description' => 'Deprecated with 1.2.0, use \'_APP_FUNCTIONS_RUNTIMES_NETWORK\' instead!',
'introduction' => '0.13.0',
'default' => 'appwrite_runtimes',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_FUNCTIONS_RUNTIMES_NETWORK',
'description' => 'The docker network used for communication between the executor and runtimes. Change this if you have altered the default network names.',
'introduction' => '1.2.0',
'default' => 'runtimes',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_DOCKER_HUB_USERNAME',
'description' => 'The username for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'introduction' => '1.2.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_DOCKER_HUB_PASSWORD',
'description' => 'The password for hub.docker.com. This variable is used to pull images from hub.docker.com.',
'introduction' => '1.2.0',
'default' => '',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_FUNCTIONS_MAINTENANCE_INTERVAL',
'description' => 'Interval how often executor checks for inactive runimes. The default value is 60 seconds.',
'introduction' => '1.2.0',
'default' => '60',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
@ -876,6 +908,15 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_MAINTENANCE_RETENTION_SCHEDULES',
'description' => 'Schedules deletion interval ( in seconds ) ',
'introduction' => 'TBD',
'default' => '86400',
'required' => false,
'question' => '',
'filter' => ''
]
],
],
[

188
app/controllers/api/account.php
View file

@ -43,10 +43,106 @@ use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Appwrite\Auth\Validator\PasswordHistory;
use Appwrite\Auth\Validator\PasswordDictionary;
use Appwrite\Auth\Validator\PersonalData;
$oauthDefaultSuccess = '/auth/oauth2/success';
$oauthDefaultFailure = '/auth/oauth2/failure';
App::post('/v1/account/invite')
->desc('Create account using an invite code')
->groups(['api', 'account', 'auth'])
->label('event', 'users.[userId].create')
->label('scope', 'public')
->label('auth.type', 'emailPassword')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createWithInviteCode')
->label('sdk.description', '/docs/references/account/create.md')
->label('sdk.response.code', Response::STATUS_CODE_CREATED)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_ACCOUNT)
->label('abuse-limit', 10)
->param('userId', '', new CustomId(), 'Unique Id. Choose your own unique ID or pass the string `ID.unique()` to auto generate it. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('email', '', new Email(), 'User email.')
->param('password', '', new Password(), 'User password. Must be at least 8 chars.')
->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true)
->param('code', '', new Text(128), 'An invite code to restrict user signups on the Appwrite console. Users with an invite code will be able to create accounts irrespective of email and IP whitelists.', true)
->inject('request')
->inject('response')
->inject('project')
->inject('dbForProject')
->inject('events')
->action(function (string $userId, string $email, string $password, string $name, string $code, Request $request, Response $response, Document $project, Database $dbForProject, Event $events) {
if ($project->getId() !== 'console') {
throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN);
}
$email = \strtolower($email);
$whitelistCodes = (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null)) : [];
if (empty($whitelistCodes)) {
throw new Exception(Exception::GENERAL_CODES_DISABLED);
}
if (!empty($whitelistCodes) && !\in_array($code, $whitelistCodes)) {
throw new Exception(Exception::USER_INVALID_CODE);
}
$limit = $project->getAttribute('auths', [])['limit'] ?? 0;
if ($limit !== 0) {
$total = $dbForProject->count('users', max: APP_LIMIT_USERS);
if ($total >= $limit) {
throw new Exception(Exception::USER_COUNT_EXCEEDED);
}
}
try {
$userId = $userId == 'unique()' ? ID::unique() : $userId;
$user = Authorization::skip(fn() => $dbForProject->createDocument('users', new Document([
'$id' => $userId,
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::user($userId)),
Permission::delete(Role::user($userId)),
],
'email' => $email,
'emailVerification' => false,
'status' => true,
'password' => Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS),
'hash' => Auth::DEFAULT_ALGO,
'hashOptions' => Auth::DEFAULT_ALGO_OPTIONS,
'passwordUpdate' => DateTime::now(),
'registration' => DateTime::now(),
'reset' => false,
'name' => $name,
'prefs' => new \stdClass(),
'sessions' => null,
'tokens' => null,
'memberships' => null,
'search' => implode(' ', [$userId, $email, $name])
])));
} catch (Duplicate $th) {
throw new Exception(Exception::USER_ALREADY_EXISTS);
}
Authorization::unsetRole(Role::guests()->toString());
Authorization::setRole(Role::user($user->getId())->toString());
Authorization::setRole(Role::users()->toString());
$events->setParam('userId', $user->getId());
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($user, Response::MODEL_ACCOUNT);
});
App::post('/v1/account')
->desc('Create Account')
->groups(['api', 'account', 'auth'])
@ -56,7 +152,6 @@ App::post('/v1/account')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'create')
@ -81,7 +176,7 @@ App::post('/v1/account')
$whitelistEmails = $project->getAttribute('authWhitelistEmails');
$whitelistIPs = $project->getAttribute('authWhitelistIPs');
if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails)) {
if (!empty($whitelistEmails) && !\in_array($email, $whitelistEmails) && !\in_array(strtoupper($email), $whitelistEmails)) {
throw new Exception(Exception::USER_EMAIL_NOT_WHITELISTED);
}
@ -100,6 +195,13 @@ App::post('/v1/account')
}
}
if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
$personalDataValidator = new PersonalData($userId, $email, $name, null);
if (!$personalDataValidator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
}
}
$passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
try {
@ -155,8 +257,6 @@ App::post('/v1/account/sessions/email')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:email'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createEmailSession')
@ -290,6 +390,7 @@ App::get('/v1/account/sessions/oauth2/:provider')
->action(function (string $provider, string $success, string $failure, array $scopes, Request $request, Response $response, Document $project) use ($oauthDefaultSuccess, $oauthDefaultFailure) {
$protocol = $request->getProtocol();
$callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId();
$providerEnabled = $project->getAttribute('authProviders', [])[$provider . 'Enabled'] ?? false;
@ -408,8 +509,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->label('docs', false)
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:{request.provider}'])
->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
->param('code', '', new Text(2048, 0), 'OAuth2 code.', true)
->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@ -549,6 +648,10 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
$name = $oauth2->getUserName($accessToken);
$email = $oauth2->getUserEmail($accessToken);
if (empty($email)) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'OAuth provider failed to return email.');
}
/**
* Is verified is not used yet, since we don't know after an accout is created anymore if it was verified or not.
*/
@ -743,7 +846,7 @@ App::post('/v1/account/sessions/magic-url')
}
}
$userId = $userId == 'unique()' ? ID::unique() : $userId;
$userId = $userId === 'unique()' ? ID::unique() : $userId;
$user->setAttributes([
'$id' => $userId,
@ -865,8 +968,6 @@ App::put('/v1/account/sessions/magic-url')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:magic-url'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateMagicURLSession')
@ -1121,8 +1222,6 @@ App::put('/v1/account/sessions/phone')
->groups(['api', 'account', 'session'])
->label('scope', 'public')
->label('event', 'users.[userId].sessions.[sessionId].create')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:phone'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneSession')
@ -1243,8 +1342,6 @@ App::post('/v1/account/sessions/anonymous')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:anonymous'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createAnonymousSession')
@ -1420,7 +1517,6 @@ App::get('/v1/account')
->desc('Get Account')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'get')
@ -1441,7 +1537,6 @@ App::get('/v1/account/prefs')
->desc('Get Account Preferences')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getPrefs')
@ -1464,7 +1559,6 @@ App::get('/v1/account/sessions')
->desc('List Sessions')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listSessions')
@ -1503,7 +1597,6 @@ App::get('/v1/account/logs')
->desc('List Logs')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listLogs')
@ -1564,7 +1657,6 @@ App::get('/v1/account/sessions/:sessionId')
->desc('Get Session')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getSession')
@ -1612,8 +1704,6 @@ App::patch('/v1/account/name')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateName')
@ -1648,7 +1738,6 @@ App::patch('/v1/account/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePassword')
@ -1675,16 +1764,22 @@ App::patch('/v1/account/password')
$newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
$historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$history = [];
$history = $user->getAttribute('passwordHistory', []);
if ($historyLimit > 0) {
$history = $user->getAttribute('passwordHistory', []);
$validator = new PasswordHistory($history, $user->getAttribute('hash'), $user->getAttribute('hashOptions'));
if (!$validator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED, 'The password was recently used', 409);
throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED);
}
$history[] = $newPassword;
array_slice($history, (count($history) - $historyLimit), $historyLimit);
$history = array_slice($history, (count($history) - $historyLimit), $historyLimit);
}
if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
$personalDataValidator = new PersonalData($user->getId(), $user->getAttribute('email'), $user->getAttribute('name'), $user->getAttribute('phone'));
if (!$personalDataValidator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
}
}
$user
@ -1708,8 +1803,6 @@ App::patch('/v1/account/email')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateEmail')
@ -1770,8 +1863,6 @@ App::patch('/v1/account/phone')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhone')
@ -1830,8 +1921,6 @@ App::patch('/v1/account/prefs')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePrefs')
@ -1865,8 +1954,6 @@ App::patch('/v1/account/status')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateStatus')
@ -1910,8 +1997,6 @@ App::delete('/v1/account/sessions/:sessionId')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('audits.userId', '{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSession')
@ -1988,7 +2073,6 @@ App::patch('/v1/account/sessions/:sessionId')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateSession')
@ -2073,8 +2157,6 @@ App::delete('/v1/account/sessions')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('audits.userId', '{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSessions')
@ -2136,7 +2218,6 @@ App::post('/v1/account/recovery')
->label('audits.event', 'recovery.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createRecovery')
@ -2277,7 +2358,6 @@ App::put('/v1/account/recovery')
->label('audits.event', 'recovery.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateRecovery')
@ -2294,8 +2374,9 @@ App::put('/v1/account/recovery')
->inject('response')
->inject('user')
->inject('dbForProject')
->inject('project')
->inject('events')
->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Event $events) {
->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $events) {
if ($password !== $passwordAgain) {
throw new Exception(Exception::USER_PASSWORD_MISMATCH);
}
@ -2315,8 +2396,23 @@ App::put('/v1/account/recovery')
Authorization::setRole(Role::user($profile->getId())->toString());
$newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
$historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$history = $profile->getAttribute('passwordHistory', []);
if ($historyLimit > 0) {
$validator = new PasswordHistory($history, $profile->getAttribute('hash'), $profile->getAttribute('hashOptions'));
if (!$validator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED);
}
$history[] = $newPassword;
$history = array_slice($history, (count($history) - $historyLimit), $historyLimit);
}
$profile = $dbForProject->updateDocument('users', $profile->getId(), $profile
->setAttribute('password', Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS))
->setAttribute('password', $newPassword)
->setAttribute('passwordHistory', $history)
->setAttribute('passwordUpdate', DateTime::now())
->setAttribute('hash', Auth::DEFAULT_ALGO)
->setAttribute('hashOptions', Auth::DEFAULT_ALGO_OPTIONS)
@ -2348,8 +2444,6 @@ App::post('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createVerification')
@ -2469,8 +2563,6 @@ App::put('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateVerification')
@ -2531,8 +2623,6 @@ App::post('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createPhoneVerification')
@ -2628,8 +2718,6 @@ App::put('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneVerification')

829
app/controllers/api/avatars.php
View file

@ -7,9 +7,16 @@ use Appwrite\Utopia\Response;
use chillerlan\QRCode\QRCode;
use chillerlan\QRCode\QROptions;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Image\Image;
use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Validator\Boolean;
use Utopia\Validator\HexColor;
use Utopia\Validator\Range;
@ -49,11 +56,139 @@ $avatarCallback = function (string $type, string $code, int $width, int $height,
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT')
->setContentType('image/png')
->file($data)
;
->file($data);
unset($image);
};
$getUserGitHub = function (string $userId, Document $project, Database $dbForProject, Database $dbForConsole, ?Logger $logger) {
try {
$user = Authorization::skip(fn () => $dbForConsole->getDocument('users', $userId));
$sessions = $user->getAttribute('sessions', []);
$gitHubSession = null;
foreach ($sessions as $session) {
if ($session->getAttribute('provider', '') === 'github') {
$gitHubSession = $session;
break;
}
}
if (empty($gitHubSession)) {
throw new Exception(Exception::GENERAL_UNKNOWN, 'GitHub session not found.');
}
$provider = $gitHubSession->getAttribute('provider', '');
$accessToken = $gitHubSession->getAttribute('providerAccessToken');
$accessTokenExpiry = $gitHubSession->getAttribute('providerAccessTokenExpiry');
$refreshToken = $gitHubSession->getAttribute('providerRefreshToken');
$appId = $project->getAttribute('authProviders', [])[$provider . 'Appid'] ?? '';
$appSecret = $project->getAttribute('authProviders', [])[$provider . 'Secret'] ?? '{}';
$className = 'Appwrite\\Auth\\OAuth2\\' . \ucfirst($provider);
if (!\class_exists($className)) {
throw new Exception(Exception::PROJECT_PROVIDER_UNSUPPORTED);
}
$oauth2 = new $className($appId, $appSecret, '', [], []);
$isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
if ($isExpired) {
try {
$oauth2->refreshTokens($refreshToken);
$accessToken = $oauth2->getAccessToken('');
$refreshToken = $oauth2->getRefreshToken('');
$verificationId = $oauth2->getUserID($accessToken);
if (empty($verificationId)) {
throw new \Exception("Locked tokens."); // Race codition, handeled in catch
}
$gitHubSession
->setAttribute('providerAccessToken', $accessToken)
->setAttribute('providerRefreshToken', $refreshToken)
->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$oauth2->getAccessTokenExpiry('')));
Authorization::skip(fn () => $dbForProject->updateDocument('sessions', $gitHubSession->getId(), $gitHubSession));
$dbForProject->deleteCachedDocument('users', $user->getId());
} catch (Throwable $err) {
$index = 0;
do {
$previousAccessToken = $gitHubSession->getAttribute('providerAccessToken');
$user = Authorization::skip(fn () => $dbForConsole->getDocument('users', $userId));
$sessions = $user->getAttribute('sessions', []);
$gitHubSession = new Document();
foreach ($sessions as $session) {
if ($session->getAttribute('provider', '') === 'github') {
$gitHubSession = $session;
break;
}
}
$accessToken = $gitHubSession->getAttribute('providerAccessToken');
if ($accessToken !== $previousAccessToken) {
break;
}
$index++;
\usleep(500000);
} while ($index < 10);
}
}
$oauth2 = new $className($appId, $appSecret, '', [], []);
$githubUser = $oauth2->getUserSlug($accessToken);
$githubId = $oauth2->getUserID($accessToken);
return [
'name' => $githubUser,
'id' => $githubId
];
} catch (Exception $error) {
if ($logger) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$log = new Log();
$log->setNamespace('console');
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('code', $error->getCode());
$log->addTag('verboseType', get_class($error));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->setAction('avatarsGetGitHub');
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('GitHub error log pushed with status code: ' . $responseCode);
}
Console::warning("Failed: {$error->getMessage()}");
Console::warning($error->getTraceAsString());
return [];
}
return [];
};
App::get('/v1/avatars/credit-cards/:code')
->desc('Get Credit Card Icon')
->groups(['api', 'avatars'])
@ -160,8 +295,7 @@ App::get('/v1/avatars/image')
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT')
->setContentType('image/png')
->file($data)
;
->file($data);
unset($image);
});
@ -274,8 +408,7 @@ App::get('/v1/avatars/favicon')
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT')
->setContentType('image/x-icon')
->file($data)
;
->file($data);
}
$fetch = @\file_get_contents($outputHref, false);
@ -292,8 +425,7 @@ App::get('/v1/avatars/favicon')
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + 60 * 60 * 24 * 30) . ' GMT')
->setContentType('image/png')
->file($data)
;
->file($data);
unset($image);
});
@ -334,8 +466,7 @@ App::get('/v1/avatars/qr')
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->setContentType('image/png')
->send($image->output('png', 9))
;
->send($image->output('png', 9));
});
App::get('/v1/avatars/initials')
@ -419,6 +550,680 @@ App::get('/v1/avatars/initials')
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->setContentType('image/png')
->file($image->getImageBlob())
;
->file($image->getImageBlob());
});
App::get('/v1/cards/cloud')
->desc('Get Front Of Cloud Card')
->groups(['api', 'avatars'])
->label('scope', 'avatars.read')
->label('cache', true)
->label('cache.resourceType', 'cards/cloud')
->label('cache.resource', 'card/{request.userId}')
->label('docs', false)
->label('origin', '*')
->param('userId', '', new UID(), 'User ID.', true)
->param('mock', '', new WhiteList(['employee', 'employee-2digit', 'hero', 'contributor', 'normal', 'platinum', 'normal-no-github', 'normal-long']), 'Mocking behaviour.', true)
->param('width', 0, new Range(0, 512), 'Resize image width, Pass an integer between 0 to 512.', true)
->param('height', 0, new Range(0, 320), 'Resize image height, Pass an integer between 0 to 320.', true)
->inject('user')
->inject('project')
->inject('dbForProject')
->inject('dbForConsole')
->inject('response')
->inject('heroes')
->inject('contributors')
->inject('employees')
->inject('logger')
->action(function (string $userId, string $mock, int $width, int $height, Document $user, Document $project, Database $dbForProject, Database $dbForConsole, Response $response, array $heroes, array $contributors, array $employees, ?Logger $logger) use ($getUserGitHub) {
$user = Authorization::skip(fn () => $dbForConsole->getDocument('users', $userId));
if ($user->isEmpty() && empty($mock)) {
throw new Exception(Exception::USER_NOT_FOUND);
}
if (!$mock) {
$name = $user->getAttribute('name', 'Anonymous');
$email = $user->getAttribute('email', '');
$createdAt = new \DateTime($user->getCreatedAt());
$gitHub = $getUserGitHub($user->getId(), $project, $dbForProject, $dbForConsole, $logger);
$githubName = $gitHub['name'] ?? '';
$githubId = $gitHub['id'] ?? '';
$isHero = \array_key_exists($email, $heroes);
$isContributor = \in_array($githubId, $contributors);
$isEmployee = \array_key_exists($email, $employees);
$employeeNumber = $isEmployee ? $employees[$email]['spot'] : '';
if ($isHero) {
$createdAt = new \DateTime($heroes[$email]['memberSince'] ?? '');
} elseif ($isEmployee) {
$createdAt = new \DateTime($employees[$email]['memberSince'] ?? '');
}
if (!$isEmployee && !empty($githubName)) {
$employeeGitHub = \array_search(\strtolower($githubName), \array_map(fn ($employee) => \strtolower($employee['gitHub']) ?? '', $employees));
if (!empty($employeeGitHub)) {
$isEmployee = true;
$employeeNumber = $isEmployee ? $employees[$employeeGitHub]['spot'] : '';
$createdAt = new \DateTime($employees[$employeeGitHub]['memberSince'] ?? '');
}
}
$isPlatinum = $user->getInternalId() % 100 === 0;
} else {
$name = $mock === 'normal-long' ? 'Sir First Walter O\'Brian Junior' : 'Walter O\'Brian';
$createdAt = new \DateTime('now');
$githubName = $mock === 'normal-no-github' ? '' : ($mock === 'normal-long' ? 'sir-first-walterobrian-junior' : 'walterobrian');
$isHero = $mock === 'hero';
$isContributor = $mock === 'contributor';
$isEmployee = \str_starts_with($mock, 'employee');
$employeeNumber = match ($mock) {
'employee' => '1',
'employee-2digit' => '18',
default => ''
};
$isPlatinum = $mock === 'platinum';
}
if ($isEmployee) {
$isContributor = false;
$isHero = false;
}
if ($isHero) {
$isContributor = false;
$isEmployee = false;
}
if ($isContributor) {
$isHero = false;
$isEmployee = false;
}
$isGolden = $isEmployee || $isHero || $isContributor;
$isPlatinum = $isGolden ? false : $isPlatinum;
$memberSince = \strtoupper('Member since ' . $createdAt->format('M') . ' ' . $createdAt->format('d') . ', ' . $createdAt->format('o'));
$imagePath = $isGolden ? 'front-golden.png' : ($isPlatinum ? 'front-platinum.png' : 'front.png');
$baseImage = new \Imagick(__DIR__ . '/../../../public/images/cards/cloud/' . $imagePath);
if ($isEmployee) {
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/employee.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 793, 35);
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_CENTER);
$text->setFont(__DIR__ . '/../../../public/fonts/Inter-Bold.ttf');
$text->setFillColor(new \ImagickPixel('#FFFADF'));
$text->setFontSize(\strlen($employeeNumber) <= 2 ? 54 : 48);
$text->setFontWeight(700);
$metricsText = $baseImage->queryFontMetrics($text, $employeeNumber);
$hashtag = new \ImagickDraw();
$hashtag->setTextAlignment(Imagick::ALIGN_CENTER);
$hashtag->setFont(__DIR__ . '/../../../public/fonts/Inter-Bold.ttf');
$hashtag->setFillColor(new \ImagickPixel('#FFFADF'));
$hashtag->setFontSize(28);
$hashtag->setFontWeight(700);
$metricsHashtag = $baseImage->queryFontMetrics($hashtag, '#');
$startX = 898;
$totalWidth = $metricsHashtag['textWidth'] + 12 + $metricsText['textWidth'];
$hashtagX = ($metricsHashtag['textWidth'] / 2);
$textX = $hashtagX + 12 + ($metricsText['textWidth'] / 2);
$hashtagX -= $totalWidth / 2;
$textX -= $totalWidth / 2;
$hashtagX += $startX;
$textX += $startX;
$baseImage->annotateImage($hashtag, $hashtagX, 150, 0, '#');
$baseImage->annotateImage($text, $textX, 150, 0, $employeeNumber);
}
if ($isContributor) {
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/contributor.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 793, 34);
}
if ($isHero) {
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/hero.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 793, 34);
}
setlocale(LC_ALL, "en_US.utf8");
// $name = \iconv("utf-8", "ascii//TRANSLIT", $name);
// $memberSince = \iconv("utf-8", "ascii//TRANSLIT", $memberSince);
// $githubName = \iconv("utf-8", "ascii//TRANSLIT", $githubName);
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_CENTER);
$text->setFont(__DIR__ . '/../../../public/fonts/Poppins-Bold.ttf');
$text->setFillColor(new \ImagickPixel('#FFFFFF'));
if (\strlen($name) > 32) {
$name = \substr($name, 0, 32);
}
if (\strlen($name) <= 23) {
$text->setFontSize(80);
$scalingDown = false;
} else {
$text->setFontSize(54);
$scalingDown = true;
}
$text->setFontWeight(700);
$baseImage->annotateImage($text, 512, 477, 0, $name);
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_CENTER);
$text->setFont(__DIR__ . '/../../../public/fonts/Inter-SemiBold.ttf');
$text->setFillColor(new \ImagickPixel($isGolden || $isPlatinum ? '#FFFFFF' : '#FFB9CC'));
$text->setFontSize(27);
$text->setFontWeight(600);
$text->setTextKerning(1.08);
$baseImage->annotateImage($text, 512, 541, 0, \strtoupper($memberSince));
if (!empty($githubName)) {
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_CENTER);
$text->setFont(__DIR__ . '/../../../public/fonts/Inter-Regular.ttf');
$text->setFillColor(new \ImagickPixel('#FFFFFF'));
$text->setFontSize($scalingDown ? 28 : 32);
$text->setFontWeight(400);
$metrics = $baseImage->queryFontMetrics($text, $githubName);
$baseImage->annotateImage($text, 512 + 20 + 4, 373 + ($scalingDown ? 2 : 0), 0, $githubName);
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/github.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$precisionFix = 5;
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 512 - ($metrics['textWidth'] / 2) - 20 - 4, 373 - ($metrics['textHeight'] - $precisionFix));
}
if (!empty($width) || !empty($height)) {
$baseImage->resizeImage($width, $height, Imagick::FILTER_LANCZOS, 1);
}
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->setContentType('image/png')
->file($baseImage->getImageBlob());
});
App::get('/v1/cards/cloud-back')
->desc('Get Back Of Cloud Card')
->groups(['api', 'avatars'])
->label('scope', 'avatars.read')
->label('cache', true)
->label('cache.resourceType', 'cards/cloud-back')
->label('cache.resource', 'card-back/{request.userId}')
->label('docs', false)
->label('origin', '*')
->param('userId', '', new UID(), 'User ID.', true)
->param('mock', '', new WhiteList(['golden', 'normal', 'platinum']), 'Mocking behaviour.', true)
->param('width', 0, new Range(0, 512), 'Resize image width, Pass an integer between 0 to 512.', true)
->param('height', 0, new Range(0, 320), 'Resize image height, Pass an integer between 0 to 320.', true)
->inject('user')
->inject('project')
->inject('dbForProject')
->inject('dbForConsole')
->inject('response')
->inject('heroes')
->inject('contributors')
->inject('employees')
->inject('logger')
->action(function (string $userId, string $mock, int $width, int $height, Document $user, Document $project, Database $dbForProject, Database $dbForConsole, Response $response, array $heroes, array $contributors, array $employees, ?Logger $logger) use ($getUserGitHub) {
$user = Authorization::skip(fn () => $dbForConsole->getDocument('users', $userId));
if ($user->isEmpty() && empty($mock)) {
throw new Exception(Exception::USER_NOT_FOUND);
}
if (!$mock) {
$userId = $user->getId();
$email = $user->getAttribute('email', '');
$gitHub = $getUserGitHub($user->getId(), $project, $dbForProject, $dbForConsole, $logger);
$githubId = $gitHub['id'] ?? '';
$isHero = \array_key_exists($email, $heroes);
$isContributor = \in_array($githubId, $contributors);
$isEmployee = \array_key_exists($email, $employees);
$isGolden = $isEmployee || $isHero || $isContributor;
$isPlatinum = $user->getInternalId() % 100 === 0;
} else {
$userId = '63e0bcf3c3eb803ba530';
$isGolden = $mock === 'golden';
$isPlatinum = $mock === 'platinum';
}
$userId = 'UID ' . $userId;
$isPlatinum = $isGolden ? false : $isPlatinum;
$imagePath = $isGolden ? 'back-golden.png' : ($isPlatinum ? 'back-platinum.png' : 'back.png');
$baseImage = new \Imagick(__DIR__ . '/../../../public/images/cards/cloud/' . $imagePath);
setlocale(LC_ALL, "en_US.utf8");
// $userId = \iconv("utf-8", "ascii//TRANSLIT", $userId);
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_CENTER);
$text->setFont(__DIR__ . '/../../../public/fonts/SourceCodePro-Regular.ttf');
$text->setFillColor(new \ImagickPixel($isGolden ? '#664A1E' : ($isPlatinum ? '#555555' : '#E8E9F0')));
$text->setFontSize(28);
$text->setFontWeight(400);
$baseImage->annotateImage($text, 512, 596, 0, $userId);
if (!empty($width) || !empty($height)) {
$baseImage->resizeImage($width, $height, Imagick::FILTER_LANCZOS, 1);
}
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->setContentType('image/png')
->file($baseImage->getImageBlob());
});
App::get('/v1/cards/cloud-og')
->desc('Get OG Image From Cloud Card')
->groups(['api', 'avatars'])
->label('scope', 'avatars.read')
->label('cache', true)
->label('cache.resourceType', 'cards/cloud-og')
->label('cache.resource', 'card-og/{request.userId}')
->label('docs', false)
->label('origin', '*')
->param('userId', '', new UID(), 'User ID.', true)
->param('mock', '', new WhiteList(['employee', 'employee-2digit', 'hero', 'contributor', 'normal', 'platinum', 'normal-no-github', 'normal-long', 'normal-long-right', 'normal-long-middle', 'normal-bg2', 'normal-bg3', 'normal-right', 'normal-middle', 'platinum-right', 'platinum-middle', 'hero-middle', 'hero-right', 'contributor-right', 'employee-right', 'contributor-middle', 'employee-middle', 'employee-2digit-middle', 'employee-2digit-right']), 'Mocking behaviour.', true)
->param('width', 0, new Range(0, 1024), 'Resize image card width, Pass an integer between 0 to 1024.', true)
->param('height', 0, new Range(0, 1024), 'Resize image card height, Pass an integer between 0 to 1024.', true)
->inject('user')
->inject('project')
->inject('dbForProject')
->inject('dbForConsole')
->inject('response')
->inject('heroes')
->inject('contributors')
->inject('employees')
->inject('logger')
->action(function (string $userId, string $mock, int $width, int $height, Document $user, Document $project, Database $dbForProject, Database $dbForConsole, Response $response, array $heroes, array $contributors, array $employees, ?Logger $logger) use ($getUserGitHub) {
$user = Authorization::skip(fn () => $dbForConsole->getDocument('users', $userId));
if ($user->isEmpty() && empty($mock)) {
throw new Exception(Exception::USER_NOT_FOUND);
}
if (!$mock) {
$internalId = $user->getInternalId();
$bgVariation = $internalId % 3 === 0 ? '1' : ($internalId % 3 === 1 ? '2' : '3');
$cardVariation = $internalId % 3 === 0 ? '1' : ($internalId % 3 === 1 ? '2' : '3');
$name = $user->getAttribute('name', 'Anonymous');
$email = $user->getAttribute('email', '');
$createdAt = new \DateTime($user->getCreatedAt());
$gitHub = $getUserGitHub($user->getId(), $project, $dbForProject, $dbForConsole, $logger);
$githubName = $gitHub['name'] ?? '';
$githubId = $gitHub['id'] ?? '';
$isHero = \array_key_exists($email, $heroes);
$isContributor = \in_array($githubId, $contributors);
$isEmployee = \array_key_exists($email, $employees);
$employeeNumber = $isEmployee ? $employees[$email]['spot'] : '';
if ($isHero) {
$createdAt = new \DateTime($heroes[$email]['memberSince'] ?? '');
} elseif ($isEmployee) {
$createdAt = new \DateTime($employees[$email]['memberSince'] ?? '');
}
if (!$isEmployee && !empty($githubName)) {
$employeeGitHub = \array_search(\strtolower($githubName), \array_map(fn ($employee) => \strtolower($employee['gitHub']) ?? '', $employees));
if (!empty($employeeGitHub)) {
$isEmployee = true;
$employeeNumber = $isEmployee ? $employees[$employeeGitHub]['spot'] : '';
$createdAt = new \DateTime($employees[$employeeGitHub]['memberSince'] ?? '');
}
}
$isPlatinum = $user->getInternalId() % 100 === 0;
} else {
$bgVariation = \str_ends_with($mock, '-bg2') ? '2' : (\str_ends_with($mock, '-bg3') ? '3' : '1');
$cardVariation = \str_ends_with($mock, '-right') ? '2' : (\str_ends_with($mock, '-middle') ? '3' : '1');
$name = \str_starts_with($mock, 'normal-long') ? 'Sir First Walter O\'Brian Junior' : 'Walter O\'Brian';
$createdAt = new \DateTime('now');
$githubName = $mock === 'normal-no-github' ? '' : (\str_starts_with($mock, 'normal-long') ? 'sir-first-walterobrian-junior' : 'walterobrian');
$isHero = \str_starts_with($mock, 'hero');
$isContributor = \str_starts_with($mock, 'contributor');
$isEmployee = \str_starts_with($mock, 'employee');
$employeeNumber = match ($mock) {
'employee' => '1',
'employee-right' => '1',
'employee-middle' => '1',
'employee-2digit' => '18',
'employee-2digit-right' => '18',
'employee-2digit-middle' => '18',
default => ''
};
$isPlatinum = \str_starts_with($mock, 'platinum');
}
if ($isEmployee) {
$isContributor = false;
$isHero = false;
}
if ($isHero) {
$isContributor = false;
$isEmployee = false;
}
if ($isContributor) {
$isHero = false;
$isEmployee = false;
}
$isGolden = $isEmployee || $isHero || $isContributor;
$isPlatinum = $isGolden ? false : $isPlatinum;
$memberSince = \strtoupper('Member since ' . $createdAt->format('M') . ' ' . $createdAt->format('d') . ', ' . $createdAt->format('o'));
$baseImage = new \Imagick(__DIR__ . "/../../../public/images/cards/cloud/og-background{$bgVariation}.png");
$cardType = $isGolden ? '-golden' : ($isPlatinum ? '-platinum' : '');
$image = new Imagick(__DIR__ . "/../../../public/images/cards/cloud/og-card{$cardType}{$cardVariation}.png");
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 1008 / 2 - $image->getImageWidth() / 2, 1008 / 2 - $image->getImageHeight() / 2);
$imageLogo = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/og-background-logo.png');
$imageShadow = new Imagick(__DIR__ . "/../../../public/images/cards/cloud/og-shadow{$cardType}.png");
if ($cardVariation === '1') {
$baseImage->compositeImage($imageLogo, Imagick::COMPOSITE_OVER, 32, 1008 - $imageLogo->getImageHeight() - 32);
$baseImage->compositeImage($imageShadow, Imagick::COMPOSITE_OVER, -450, 700);
} elseif ($cardVariation === '2') {
$baseImage->compositeImage($imageLogo, Imagick::COMPOSITE_OVER, 1008 - $imageLogo->getImageWidth() - 32, 1008 - $imageLogo->getImageHeight() - 32);
$baseImage->compositeImage($imageShadow, Imagick::COMPOSITE_OVER, -20, 710);
} else {
$baseImage->compositeImage($imageLogo, Imagick::COMPOSITE_OVER, 1008 - $imageLogo->getImageWidth() - 32, 1008 - $imageLogo->getImageHeight() - 32);
$baseImage->compositeImage($imageShadow, Imagick::COMPOSITE_OVER, -135, 710);
}
if ($isEmployee) {
$file = $cardVariation === '3' ? 'employee-skew.png' : 'employee.png';
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/' . $file);
$image->setGravity(Imagick::GRAVITY_CENTER);
$hashtag = new \ImagickDraw();
$hashtag->setTextAlignment(Imagick::ALIGN_LEFT);
$hashtag->setFont(__DIR__ . '/../../../public/fonts/Inter-Bold.ttf');
$hashtag->setFillColor(new \ImagickPixel('#FFFADF'));
$hashtag->setFontSize(20);
$hashtag->setFontWeight(700);
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_LEFT);
$text->setFont(__DIR__ . '/../../../public/fonts/Inter-Bold.ttf');
$text->setFillColor(new \ImagickPixel('#FFFADF'));
$text->setFontSize(\strlen($employeeNumber) <= 1 ? 36 : 28);
$text->setFontWeight(700);
if ($cardVariation === '3') {
$hashtag->setFontSize(16);
$text->setFontSize(\strlen($employeeNumber) <= 1 ? 30 : 26);
$hashtag->skewY(20);
$hashtag->skewX(20);
$text->skewY(20);
$text->skewX(20);
}
$metricsHashtag = $baseImage->queryFontMetrics($hashtag, '#');
$metricsText = $baseImage->queryFontMetrics($text, $employeeNumber);
$group = new Imagick();
$groupWidth = $metricsHashtag['textWidth'] + 6 + $metricsText['textWidth'];
if ($cardVariation === '1') {
$group->newImage($groupWidth, $metricsText['textHeight'], '#00000000');
$group->annotateImage($hashtag, 0, $metricsText['textHeight'], 0, '#');
$group->annotateImage($text, $metricsHashtag['textWidth'] + 6, $metricsText['textHeight'], 0, $employeeNumber);
$image->resizeImage(120, 120, Imagick::FILTER_LANCZOS, 1);
$image->rotateImage(new ImagickPixel('#00000000'), -20);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 612, 203);
$group->rotateImage(new ImagickPixel('#00000000'), -22);
if (\strlen($employeeNumber) <= 1) {
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, 660, 245);
} else {
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, 655, 247);
}
} elseif ($cardVariation === '2') {
$group->newImage($groupWidth, $metricsText['textHeight'], '#00000000');
$group->annotateImage($hashtag, 0, $metricsText['textHeight'], 0, '#');
$group->annotateImage($text, $metricsHashtag['textWidth'] + 6, $metricsText['textHeight'], 0, $employeeNumber);
$image->resizeImage(120, 120, Imagick::FILTER_LANCZOS, 1);
$image->rotateImage(new ImagickPixel('#00000000'), 30);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 715, 425);
$group->rotateImage(new ImagickPixel('#00000000'), 32);
if (\strlen($employeeNumber) <= 1) {
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, 775, 465);
} else {
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, 767, 470);
}
} else {
$group->newImage(300, 300, '#00000000');
$hashtag->annotation(0, $metricsText['textHeight'], '#');
$text->annotation($metricsHashtag['textWidth'] + 2, $metricsText['textHeight'], $employeeNumber);
$group->drawImage($hashtag);
$group->drawImage($text);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 640, 293);
if (\strlen($employeeNumber) <= 1) {
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, 670, 317);
} else {
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, 663, 322);
}
}
}
if ($isContributor) {
$file = $cardVariation === '3' ? 'contributor-skew.png' : 'contributor.png';
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/' . $file);
$image->setGravity(Imagick::GRAVITY_CENTER);
if ($cardVariation === '1') {
$image->resizeImage(120, 120, Imagick::FILTER_LANCZOS, 1);
$image->rotateImage(new ImagickPixel('#00000000'), -20);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 612, 203);
} elseif ($cardVariation === '2') {
$image->resizeImage(120, 120, Imagick::FILTER_LANCZOS, 1);
$image->rotateImage(new ImagickPixel('#00000000'), 30);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 715, 425);
} else {
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 640, 293);
}
}
if ($isHero) {
$file = $cardVariation === '3' ? 'hero-skew.png' : 'hero.png';
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/' . $file);
$image->setGravity(Imagick::GRAVITY_CENTER);
if ($cardVariation === '1') {
$image->resizeImage(120, 120, Imagick::FILTER_LANCZOS, 1);
$image->rotateImage(new ImagickPixel('#00000000'), -20);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 612, 203);
} elseif ($cardVariation === '2') {
$image->resizeImage(120, 120, Imagick::FILTER_LANCZOS, 1);
$image->rotateImage(new ImagickPixel('#00000000'), 30);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 715, 425);
} else {
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 640, 293);
}
}
setlocale(LC_ALL, "en_US.utf8");
// $name = \iconv("utf-8", "ascii//TRANSLIT", $name);
// $memberSince = \iconv("utf-8", "ascii//TRANSLIT", $memberSince);
// $githubName = \iconv("utf-8", "ascii//TRANSLIT", $githubName);
$textName = new \ImagickDraw();
$textName->setTextAlignment(Imagick::ALIGN_CENTER);
$textName->setFont(__DIR__ . '/../../../public/fonts/Poppins-Bold.ttf');
$textName->setFillColor(new \ImagickPixel('#FFFFFF'));
if (\strlen($name) > 32) {
$name = \substr($name, 0, 32);
}
if ($cardVariation === '1') {
if (\strlen($name) <= 23) {
$scalingDown = false;
$textName->setFontSize(54);
} else {
$scalingDown = true;
$textName->setFontSize(36);
}
} elseif ($cardVariation === '2') {
if (\strlen($name) <= 23) {
$scalingDown = false;
$textName->setFontSize(50);
} else {
$scalingDown = true;
$textName->setFontSize(34);
}
} else {
if (\strlen($name) <= 23) {
$scalingDown = false;
$textName->setFontSize(44);
} else {
$scalingDown = true;
$textName->setFontSize(32);
}
}
$textName->setFontWeight(700);
$textMember = new \ImagickDraw();
$textMember->setTextAlignment(Imagick::ALIGN_CENTER);
$textMember->setFont(__DIR__ . '/../../../public/fonts/Inter-Medium.ttf');
$textMember->setFillColor(new \ImagickPixel($isGolden || $isPlatinum ? '#FFFFFF' : '#FFB9CC'));
$textMember->setFontWeight(500);
$textMember->setTextKerning(1.12);
if ($cardVariation === '1') {
$textMember->setFontSize(21);
$baseImage->annotateImage($textName, 550, 600, -22, $name);
$baseImage->annotateImage($textMember, 585, 635, -22, $memberSince);
} elseif ($cardVariation === '2') {
$textMember->setFontSize(20);
$baseImage->annotateImage($textName, 435, 590, 31.37, $name);
$baseImage->annotateImage($textMember, 412, 628, 31.37, $memberSince);
} else {
$textMember->setFontSize(16);
$textName->skewY(20);
$textName->skewX(20);
$textName->annotation(320, 700, $name);
$textMember->skewY(20);
$textMember->skewX(20);
$textMember->annotation(330, 735, $memberSince);
$baseImage->drawImage($textName);
$baseImage->drawImage($textMember);
}
if (!empty($githubName)) {
$text = new \ImagickDraw();
$text->setTextAlignment(Imagick::ALIGN_LEFT);
$text->setFont(__DIR__ . '/../../../public/fonts/Inter-Regular.ttf');
$text->setFillColor(new \ImagickPixel('#FFFFFF'));
$text->setFontSize($scalingDown ? 16 : 20);
$text->setFontWeight(400);
if ($cardVariation === '1') {
$metrics = $baseImage->queryFontMetrics($text, $githubName);
$group = new Imagick();
$groupWidth = $metrics['textWidth'] + 32 + 4;
$group->newImage($groupWidth, $metrics['textHeight'] + 10, '#00000000');
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/github.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$image->resizeImage(32, 32, Imagick::FILTER_LANCZOS, 1);
$precisionFix = -1;
$group->compositeImage($image, Imagick::COMPOSITE_OVER, 0, 0);
$group->annotateImage($text, 32 + 4, $metrics['textHeight'] - $precisionFix, 0, $githubName);
$group->rotateImage(new ImagickPixel('#00000000'), -22);
$x = 510 - $group->getImageWidth() / 2;
$y = 530 - $group->getImageHeight() / 2;
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, $x, $y);
} elseif ($cardVariation === '2') {
$metrics = $baseImage->queryFontMetrics($text, $githubName);
$group = new Imagick();
$groupWidth = $metrics['textWidth'] + 32 + 4;
$group->newImage($groupWidth, $metrics['textHeight'] + 10, '#00000000');
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/github.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$image->resizeImage(32, 32, Imagick::FILTER_LANCZOS, 1);
$precisionFix = -1;
$group->compositeImage($image, Imagick::COMPOSITE_OVER, 0, 0);
$group->annotateImage($text, 32 + 4, $metrics['textHeight'] - $precisionFix, 0, $githubName);
$group->rotateImage(new ImagickPixel('#00000000'), 31.11);
$x = 485 - $group->getImageWidth() / 2;
$y = 530 - $group->getImageHeight() / 2;
$baseImage->compositeImage($group, Imagick::COMPOSITE_OVER, $x, $y);
} else {
$text->skewY(20);
$text->skewX(20);
$text->setTextAlignment(\Imagick::ALIGN_CENTER);
$text->annotation(320 + 15 + 2, 640, $githubName);
$metrics = $baseImage->queryFontMetrics($text, $githubName);
$image = new Imagick(__DIR__ . '/../../../public/images/cards/cloud/github-skew.png');
$image->setGravity(Imagick::GRAVITY_CENTER);
$baseImage->compositeImage($image, Imagick::COMPOSITE_OVER, 512 - ($metrics['textWidth'] / 2), 518 + \strlen($githubName) * 1.3);
$baseImage->drawImage($text);
}
}
if (!empty($width) || !empty($height)) {
$baseImage->resizeImage($width, $height, Imagick::FILTER_LANCZOS, 1);
}
$response
->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
->setContentType('image/png')
->file($baseImage->getImageBlob());
});

497
app/controllers/api/databases.php
View file

@ -376,7 +376,6 @@ App::post('/v1/databases')
->label('scope', 'databases.write')
->label('audits.event', 'database.create')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'create')
@ -403,7 +402,7 @@ App::post('/v1/databases')
]));
$database = $dbForProject->getDocument('databases', $databaseId);
$collections = Config::getParam('collections', [])['collections'] ?? [];
$collections = (Config::getParam('collections', [])['databases'] ?? [])['collections'] ?? [];
if (empty($collections)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'The "collections" collection is not configured.');
}
@ -450,7 +449,6 @@ App::get('/v1/databases')
->desc('List Databases')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'list')
@ -496,7 +494,6 @@ App::get('/v1/databases/:databaseId')
->desc('Get Database')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'get')
@ -611,7 +608,6 @@ App::put('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].update')
->label('audits.event', 'database.update')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'update')
@ -656,7 +652,6 @@ App::delete('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].delete')
->label('audits.event', 'database.delete')
->label('audits.resource', 'database/{request.databaseId}')
->label('usage.metric', 'databases.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'delete')
@ -704,8 +699,6 @@ App::post('/v1/databases/:databaseId/collections')
->label('scope', 'collections.write')
->label('audits.event', 'collection.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
->label('usage.metric', 'collections.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createCollection')
@ -771,8 +764,6 @@ App::get('/v1/databases/:databaseId/collections')
->desc('List Collections')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollections')
@ -828,8 +819,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
->desc('Get Collection')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getCollection')
@ -864,8 +853,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
->desc('List Collection Logs')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollectionLogs')
@ -963,8 +950,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].update')
->label('audits.event', 'collection.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateCollection')
@ -1033,8 +1018,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].delete')
->label('audits.event', 'collection.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteCollection')
@ -1089,8 +1072,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createStringAttribute')
@ -1139,8 +1120,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEmailAttribute')
@ -1183,8 +1162,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEnumAttribute')
@ -1243,8 +1220,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIpAttribute')
@ -1287,8 +1262,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createUrlAttribute')
@ -1331,8 +1304,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIntegerAttribute')
@ -1404,8 +1375,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createFloatAttribute')
@ -1480,8 +1449,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createBooleanAttribute')
@ -1523,8 +1490,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createDatetimeAttribute')
@ -1567,8 +1532,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createRelationshipAttribute')
@ -1646,8 +1609,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
->desc('List Attributes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listAttributes')
@ -1686,8 +1647,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
->desc('Get Attribute')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getAttribute')
@ -1765,8 +1724,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateStringAttribute')
@ -1805,8 +1762,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateEmailAttribute')
@ -1846,8 +1801,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateEnumAttribute')
@ -1889,8 +1842,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateIpAttribute')
@ -1930,8 +1881,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/:
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateUrlAttribute')
@ -1971,8 +1920,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateIntegerAttribute')
@ -2022,8 +1969,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateFloatAttribute')
@ -2073,8 +2018,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateBooleanAttribute')
@ -2113,8 +2056,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateDatetimeAttribute')
@ -2153,8 +2094,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
->label('audits.event', 'attribute.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateRelationshipAttribute')
@ -2209,8 +2148,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete')
->label('audits.event', 'attribute.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteAttribute')
@ -2320,8 +2257,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
->label('scope', 'collections.write')
->label('audits.event', 'index.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createIndex')
@ -2358,7 +2293,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
Query::equal('databaseInternalId', [$db->getInternalId()])
], 61);
$limit = 64 - MariaDB::getCountOfDefaultIndexes();
$limit = $dbForProject->getLimitForIndexes();
if ($count >= $limit) {
throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index limit exceeded');
@ -2477,8 +2412,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
->desc('List Indexes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listIndexes')
@ -2516,8 +2449,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->desc('Get Index')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getIndex')
@ -2560,8 +2491,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete')
->label('audits.event', 'index.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteIndex')
@ -2626,8 +2555,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
->label('scope', 'documents.write')
->label('audits.event', 'document.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
->label('usage.metric', 'documents.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -2863,8 +2790,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->desc('List Documents')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocuments')
@ -2995,8 +2920,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('Get Document')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getDocument')
@ -3091,8 +3014,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('List Document Logs')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocumentLogs')
@ -3195,8 +3116,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
->label('scope', 'documents.write')
->label('audits.event', 'document.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
->label('usage.metric', 'documents.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -3441,8 +3360,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
->label('audits.event', 'document.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
->label('usage.metric', 'documents.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -3614,107 +3531,57 @@ App::get('/v1/databases/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_DATABASES,
METRIC_COLLECTIONS,
METRIC_DOCUMENTS,
];
$metrics = [
'databases.$all.count.total',
'documents.$all.count.total',
'collections.$all.count.total',
'databases.$all.requests.create',
'databases.$all.requests.read',
'databases.$all.requests.update',
'databases.$all.requests.delete',
'collections.$all.requests.create',
'collections.$all.requests.read',
'collections.$all.requests.update',
'collections.$all.requests.delete',
'documents.$all.requests.create',
'documents.$all.requests.read',
'documents.$all.requests.update',
'documents.$all.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// Added 3'rd level to Index [period, metric, time] because of order by.
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'databasesCount' => $stats['databases.$all.count.total'] ?? [],
'documentsCount' => $stats['documents.$all.count.total'] ?? [],
'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
'documentsCreate' => $stats['documents.$all.requests.create'] ?? [],
'documentsRead' => $stats['documents.$all.requests.read'] ?? [],
'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
'collectionsRead' => $stats['collections.$all.requests.read'] ?? [],
'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
'databasesRead' => $stats['databases.$all.requests.read'] ?? [],
'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
}
$response->dynamic(new Document([
'range' => $range,
'databasesTotal' => $usage[$metrics[0]],
'collectionsTotal' => $usage[$metrics[1]],
'documentsTotal' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_DATABASES);
});
App::get('/v1/databases/:databaseId/usage')
@ -3733,97 +3600,62 @@ App::get('/v1/databases/:databaseId/usage')
->inject('dbForProject')
->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$database = $dbForProject->getDocument('databases', $databaseId);
$metrics = [
'collections.' . $databaseId . '.count.total',
'collections.' . $databaseId . '.requests.create',
'collections.' . $databaseId . '.requests.read',
'collections.' . $databaseId . '.requests.update',
'collections.' . $databaseId . '.requests.delete',
'documents.' . $databaseId . '.count.total',
'documents.' . $databaseId . '.requests.create',
'documents.' . $databaseId . '.requests.read',
'documents.' . $databaseId . '.requests.update',
'documents.' . $databaseId . '.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// TODO@kodumbeats explore performance if query is ordered by time ASC
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
'collectionsRead' => $stats["collections.{$databaseId}.requests.read"] ?? [],
'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
]);
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'collectionsTotal' => $usage[$metrics[0]],
'documentsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_DATABASE);
});
App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
@ -3844,93 +3676,60 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
->inject('dbForProject')
->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject) {
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
];
$metrics = [
"documents.{$databaseId}/{$collectionId}.count.total",
"documents.{$databaseId}/{$collectionId}.requests.create",
"documents.{$databaseId}/{$collectionId}.requests.read",
"documents.{$databaseId}/{$collectionId}.requests.update",
"documents.{$databaseId}/{$collectionId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
$response->dynamic(new Document([
'range' => $range,
'documentsTotal' => $usage[$metrics[0]],
]), Response::MODEL_USAGE_COLLECTION);
});

467
app/controllers/api/functions.php
View file

@ -6,6 +6,7 @@ use Appwrite\Event\Build;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\Validator\Event as ValidatorEvent;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Validator\CustomId;
@ -13,7 +14,6 @@ use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\UID;
use Appwrite\Usage\Stats;
use Utopia\Storage\Device;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
@ -36,7 +36,6 @@ use Utopia\Validator\Text;
use Utopia\Validator\Range;
use Utopia\Validator\WhiteList;
use Utopia\Config\Config;
use Cron\CronExpression;
use Executor\Executor;
use Utopia\CLI\Console;
use Utopia\Database\Validator\Roles;
@ -72,10 +71,8 @@ App::post('/v1/functions')
->inject('project')
->inject('user')
->inject('events')
->action(function (string $functionId, string $name, array $execute, string $runtime, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance) {
$cron = !empty($schedule) ? new CronExpression($schedule) : null;
$next = !empty($schedule) ? DateTime::format($cron->getNextRunDate()) : null;
->inject('dbForConsole')
->action(function (string $functionId, string $name, array $execute, string $runtime, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) {
$functionId = ($functionId == 'unique()') ? ID::unique() : $functionId;
$function = $dbForProject->createDocument('functions', new Document([
@ -84,26 +81,31 @@ App::post('/v1/functions')
'enabled' => $enabled,
'name' => $name,
'runtime' => $runtime,
'deploymentInternalId' => '',
'deployment' => '',
'events' => $events,
'schedule' => $schedule,
'scheduleUpdatedAt' => DateTime::now(),
'schedulePrevious' => null,
'scheduleNext' => $next,
'scheduleInternalId' => '',
'timeout' => $timeout,
'search' => implode(' ', [$functionId, $name, $runtime])
]));
if ($next) {
// Async task reschedule
$functionEvent = new Func();
$functionEvent
->setFunction($function)
->setType('schedule')
->setUser($user)
->setProject($project)
->schedule(new \DateTime($next));
}
$schedule = Authorization::skip(
fn() => $dbForConsole->createDocument('schedules', new Document([
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
'active' => false,
]))
);
$function->setAttribute('scheduleId', $schedule->getId());
$function->setAttribute('scheduleInternalId', $schedule->getInternalId());
$dbForProject->updateDocument('functions', $function->getId(), $function);
$eventsInstance->setParam('functionId', $function->getId());
@ -231,92 +233,66 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
$metrics = [
"executions.$functionId.compute.total",
"executions.$functionId.compute.success",
"executions.$functionId.compute.failure",
"executions.$functionId.compute.time",
"builds.$functionId.compute.total",
"builds.$functionId.compute.success",
"builds.$functionId.compute.failure",
"builds.$functionId.compute.time",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [],
'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'buildsTotal' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsTime' => $usage[$metrics[4]],
'executionsTotal' => $usage[$metrics[5]],
'executionsTime' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
@ -334,92 +310,67 @@ App::get('/v1/functions/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
$metrics = [
'executions.$all.compute.total',
'executions.$all.compute.failure',
'executions.$all.compute.success',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.failure',
'builds.$all.compute.success',
'builds.$all.compute.time',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats[$metrics[0]] ?? [],
'executionsFailure' => $stats[$metrics[1]] ?? [],
'executionsSuccess' => $stats[$metrics[2]] ?? [],
'executionsTime' => $stats[$metrics[3]] ?? [],
'buildsTotal' => $stats[$metrics[4]] ?? [],
'buildsFailure' => $stats[$metrics[5]] ?? [],
'buildsSuccess' => $stats[$metrics[6]] ?? [],
'buildsTime' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
}
$response->dynamic(new Document([
'range' => $range,
'functionsTotal' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'buildsTotal' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsTime' => $usage[$metrics[5]],
'executionsTotal' => $usage[$metrics[6]],
'executionsTime' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
@ -448,7 +399,8 @@ App::put('/v1/functions/:functionId')
->inject('project')
->inject('user')
->inject('events')
->action(function (string $functionId, string $name, array $execute, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance) {
->inject('dbForConsole')
->action(function (string $functionId, string $name, array $execute, array $events, string $schedule, int $timeout, bool $enabled, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventsInstance, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -456,9 +408,6 @@ App::put('/v1/functions/:functionId')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$cron = !empty($schedule) ? new CronExpression($schedule) : null;
$next = !empty($schedule) ? DateTime::format($cron->getNextRunDate()) : null;
$enabled ??= $function->getAttribute('enabled', true);
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [
@ -466,23 +415,17 @@ App::put('/v1/functions/:functionId')
'name' => $name,
'events' => $events,
'schedule' => $schedule,
'scheduleUpdatedAt' => DateTime::now(),
'scheduleNext' => $next,
'timeout' => $timeout,
'enabled' => $enabled,
'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]),
])));
if ($next) {
// Async task reschedule
$functionEvent = new Func();
$functionEvent
->setFunction($function)
->setType('schedule')
->setUser($user)
->setProject($project)
->schedule(new \DateTime($next));
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$eventsInstance->setParam('functionId', $function->getId());
@ -509,7 +452,8 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId')
->inject('dbForProject')
->inject('project')
->inject('events')
->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $events) {
->inject('dbForConsole')
->action(function (string $functionId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $events, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
@ -532,9 +476,17 @@ App::patch('/v1/functions/:functionId/deployments/:deploymentId')
}
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [
'deployment' => $deployment->getId()
'deploymentInternalId' => $deployment->getInternalId(),
'deployment' => $deployment->getId(),
])));
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$events
->setParam('functionId', $function->getId())
->setParam('deploymentId', $deployment->getId());
@ -560,7 +512,9 @@ App::delete('/v1/functions/:functionId')
->inject('dbForProject')
->inject('deletes')
->inject('events')
->action(function (string $functionId, Response $response, Database $dbForProject, Delete $deletes, Event $events) {
->inject('project')
->inject('dbForConsole')
->action(function (string $functionId, Response $response, Database $dbForProject, Delete $deletes, Event $events, Document $project, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -572,6 +526,12 @@ App::delete('/v1/functions/:functionId')
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove function from DB');
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('active', false);
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$deletes
->setType(DELETE_TYPE_DOCUMENT)
->setDocument($function);
@ -608,7 +568,8 @@ App::post('/v1/functions/:functionId/deployments')
->inject('project')
->inject('deviceFunctions')
->inject('deviceLocal')
->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) {
->inject('dbForConsole')
->action(function (string $functionId, string $entrypoint, mixed $code, bool $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -677,7 +638,6 @@ App::post('/v1/functions/:functionId/deployments')
// Save to storage
$fileSize ??= $deviceLocal->getFileSize($fileTmpName);
$path = $deviceFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)];
@ -722,8 +682,10 @@ App::post('/v1/functions/:functionId/deployments')
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@ -752,8 +714,10 @@ App::post('/v1/functions/:functionId/deployments')
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@ -933,6 +897,7 @@ App::delete('/v1/functions/:functionId/deployments/:deploymentId')
if ($function->getAttribute('deployment') === $deployment->getId()) { // Reset function deployment
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document(array_merge($function->getArrayCopy(), [
'deployment' => '',
'deploymentInternalId' => '',
])));
}
@ -1006,8 +971,6 @@ App::post('/v1/functions/:functionId/deployments/:deploymentId/builds/:buildId')
$response->noContent();
});
App::post('/v1/functions/:functionId/executions')
->groups(['api', 'functions'])
->desc('Create Execution')
@ -1031,9 +994,10 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('user')
->inject('events')
->inject('usage')
->inject('mode')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode) {
->inject('queueForFunctions')
->inject('queueForUsage')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, string $mode, Func $queueForFunctions, Usage $queueForUsage) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
@ -1083,7 +1047,9 @@ App::post('/v1/functions/:functionId/executions')
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', new Document([
'$id' => $executionId,
'$permissions' => !$user->isEmpty() ? [Permission::read(Role::user($user->getId()))] : [],
'functionInternalId' => $function->getInternalId(),
'functionId' => $function->getId(),
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'trigger' => 'http', // http / schedule / event
'status' => $async ? 'waiting' : 'processing', // waiting / processing / completed / failed
@ -1121,24 +1087,22 @@ App::post('/v1/functions/:functionId/executions')
->setContext('function', $function);
if ($async) {
$event = new Func();
$event
$queueForFunctions
->setType('http')
->setExecution($execution)
->setFunction($function)
->setData($data)
->setJWT($jwt)
->setProject($project)
->setUser($user);
$event->trigger();
->setUser($user)
->trigger();
return $response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
->dynamic($execution, Response::MODEL_EXECUTION);
}
$vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) {
$vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
$carry[$var->getAttribute('key')] = $var->getAttribute('value') ?? '';
return $carry;
}, []);
@ -1162,13 +1126,12 @@ App::post('/v1/functions/:functionId/executions')
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
path: $build->getAttribute('outputPath', ''),
vars: $vars,
data: $data,
entrypoint: $deployment->getAttribute('entrypoint', ''),
runtime: $function->getAttribute('runtime', ''),
payload: $data,
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
baseImage: $runtime['image']
image: $runtime['image'],
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);
/** Update execution status */
@ -1178,6 +1141,13 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('duration', $executionResponse['duration']);
/**
* Sync execution compute usage from
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
;
} catch (\Throwable $th) {
$interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
$execution
@ -1190,13 +1160,6 @@ App::post('/v1/functions/:functionId/executions')
Authorization::skip(fn () => $dbForProject->updateDocument('executions', $executionId, $execution));
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);
@ -1351,7 +1314,8 @@ App::post('/v1/functions/:functionId/variables')
->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@ -1380,8 +1344,22 @@ App::post('/v1/functions/:functionId/variables')
throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($variable, Response::MODEL_VARIABLE);
@ -1467,7 +1445,8 @@ App::put('/v1/functions/:functionId/variables/:variableId')
->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
@ -1496,8 +1475,22 @@ App::put('/v1/functions/:functionId/variables/:variableId')
throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
}
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response->dynamic($variable, Response::MODEL_VARIABLE);
});
@ -1517,7 +1510,8 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
->param('variableId', '', new UID(), 'Variable unique ID.', false)
->inject('response')
->inject('dbForProject')
->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject) {
->inject('dbForConsole')
->action(function (string $functionId, string $variableId, Response $response, Database $dbForProject, Database $dbForConsole) {
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
@ -1534,7 +1528,22 @@ App::delete('/v1/functions/:functionId/variables/:variableId')
}
$dbForProject->deleteDocument('variables', $variable->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$dbForProject->deleteCachedDocument('functions', $function->getId());
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
$response->noContent();
});

239
app/controllers/api/health.php
View file

@ -5,7 +5,11 @@ use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\Pools\Group;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
@ -26,6 +30,7 @@ App::get('/v1/health')
->action(function (Response $response) {
$output = [
'name' => 'http',
'status' => 'pass',
'ping' => 0
];
@ -42,7 +47,6 @@ App::get('/v1/health/version')
->label('sdk.response.model', Response::MODEL_HEALTH_VERSION)
->inject('response')
->action(function (Response $response) {
$response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION);
});
@ -58,30 +62,50 @@ App::get('/v1/health/db')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('utopia')
->action(function (Response $response, App $utopia) {
->inject('pools')
->action(function (Response $response, Group $pools) {
$checkStart = \microtime(true);
$output = [];
try {
$db = $utopia->getResource('db'); /* @var $db PDO */
// Run a small test to check the connection
$statement = $db->prepare("SELECT 1;");
$statement->closeCursor();
$statement->execute();
} catch (Exception $_e) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available');
}
$output = [
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
$configs = [
'Console.DB' => Config::getParam('pools-console'),
'Projects.DB' => Config::getParam('pools-database'),
];
$response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/cache')
@ -96,23 +120,163 @@ App::get('/v1/health/cache')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('utopia')
->action(function (Response $response, App $utopia) {
->inject('pools')
->action(function (Response $response, Group $pools) {
$checkStart = \microtime(true);
$output = [];
$redis = $utopia->getResource('cache');
if (!$redis->ping(true)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available');
}
$output = [
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
$configs = [
'Cache' => Config::getParam('pools-cache'),
];
$response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS);
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/queue')
->desc('Get Queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getQueue')
->label('sdk.description', '/docs/references/health/get-queue.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('pools')
->action(function (Response $response, Group $pools) {
$output = [];
$configs = [
'Queue' => Config::getParam('pools-queue'),
];
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/pubsub')
->desc('Get PubSub')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'health')
->label('sdk.method', 'getPubSub')
->label('sdk.description', '/docs/references/health/get-pubsub.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_STATUS)
->inject('response')
->inject('pools')
->action(function (Response $response, Group $pools) {
$output = [];
$configs = [
'PubSub' => Config::getParam('pools-pubsub'),
];
foreach ($configs as $key => $config) {
foreach ($config as $database) {
try {
$adapter = $pools->get($database)->pop()->getResource();
$checkStart = \microtime(true);
if ($adapter->ping()) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'pass',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
} else {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
} catch (\Throwable $th) {
$output[] = new Document([
'name' => $key . " ($database)",
'status' => 'fail',
'ping' => \round((\microtime(true) - $checkStart) / 1000)
]);
}
}
}
$response->dynamic(new Document([
'statuses' => $output,
'total' => count($output),
]), Response::MODEL_HEALTH_STATUS_LIST);
});
App::get('/v1/health/time')
@ -234,10 +398,11 @@ App::get('/v1/health/queue/functions')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->inject('queue')
->inject('response')
->action(function (Response $response) {
$response->dynamic(new Document([ 'size' => Resque::size(Event::FUNCTIONS_QUEUE_NAME) ]), Response::MODEL_HEALTH_QUEUE);
->action(function (Connection $queue, Response $response) {
$client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/storage/local')

94
app/controllers/api/project.php Normal file
View file

@ -0,0 +1,94 @@
<?php
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Validator\WhiteList;
App::get('/v1/project/usage')
->desc('Get usage stats for a project')
->groups(['api', 'usage'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'project')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_NETWORK_REQUESTS,
METRIC_NETWORK_INBOUND,
METRIC_NETWORK_OUTBOUND,
METRIC_EXECUTIONS,
METRIC_DOCUMENTS,
METRIC_DATABASES,
METRIC_USERS,
METRIC_BUCKETS,
METRIC_FILES_STORAGE
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'requestsTotal' => ($usage[$metrics[0]]),
'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]),
'executionsTotal' => $usage[$metrics[3]],
'documentsTotal' => $usage[$metrics[4]],
'databasesTotal' => $usage[$metrics[5]],
'usersTotal' => $usage[$metrics[6]],
'bucketsTotal' => $usage[$metrics[7]],
'filesStorage' => $usage[$metrics[8]],
]), Response::MODEL_USAGE_PROJECT);
});

217
app/controllers/api/projects.php
View file

@ -16,21 +16,21 @@ use Utopia\App;
use Utopia\Audit\Audit;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Datetime as DatetimeValidator;
use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
use Utopia\Registry\Registry;
use Appwrite\Extend\Exception;
use Appwrite\Network\Validator\Email;
use Appwrite\Utopia\Database\Validator\Queries\Projects;
use Utopia\Database\Exception\Duplicate;
use Utopia\Cache\Cache;
use Utopia\Pools\Group;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Boolean;
use Utopia\Validator\Hostname;
@ -76,8 +76,10 @@ App::post('/v1/projects')
->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true)
->inject('response')
->inject('dbForConsole')
->inject('dbForProject')
->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Database $dbForProject) {
->inject('cache')
->inject('pools')
->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) {
$team = $dbForConsole->getDocument('teams', $teamId);
@ -86,13 +88,46 @@ App::post('/v1/projects')
}
$auth = Config::getParam('auth', []);
$auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG];
$auths = ['limit' => 0, 'maxSessions' => APP_LIMIT_USER_SESSIONS_DEFAULT, 'passwordHistory' => 0, 'passwordDictionary' => false, 'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, 'personalDataCheck' => false];
foreach ($auth as $index => $method) {
$auths[$method['key'] ?? ''] = true;
}
$projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
$backups['database_db_fra1_02'] = ['from' => '7:30', 'to' => '8:15'];
$backups['database_db_fra1_03'] = ['from' => '10:30', 'to' => '11:15'];
$backups['database_db_fra1_04'] = ['from' => '13:30', 'to' => '14:15'];
$backups['database_db_fra1_05'] = ['from' => '4:30', 'to' => '5:15'];
$databases = Config::getParam('pools-database', []);
/**
* Extract db from list while backing
*/
if (count($databases) > 1) {
$now = new \DateTime();
foreach ($databases as $index => $database) {
if (empty($backups[$database])) {
continue;
}
$backup = $backups[$database];
$from = \DateTime::createFromFormat('H:i', $backup['from']);
$to = \DateTime::createFromFormat('H:i', $backup['to']);
if ($now >= $from && $now <= $to) {
unset($databases[$index]);
break;
}
}
}
if ($index = array_search('database_db_fra1_05', $databases)) {
$database = $databases[$index];
} else {
$database = $databases[array_rand($databases)];
}
if ($projectId === 'console') {
throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project.");
}
@ -129,14 +164,13 @@ App::post('/v1/projects')
'domains' => null,
'auths' => $auths,
'search' => implode(' ', [$projectId, $name]),
'database' => $database
]));
} catch (Duplicate $th) {
throw new Exception(Exception::PROJECT_ALREADY_EXISTS);
}
/** @var array $collections */
$collections = Config::getParam('collections', []);
$dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache);
$dbForProject->setNamespace("_{$project->getInternalId()}");
$dbForProject->create();
@ -146,6 +180,9 @@ App::post('/v1/projects')
$adapter = new TimeLimit('', 0, 1, $dbForProject);
$adapter->setup();
/** @var array $collections */
$collections = Config::getParam('collections', [])['projects'] ?? [];
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
@ -254,120 +291,6 @@ App::get('/v1/projects/:projectId')
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::get('/v1/projects/:projectId/usage')
->desc('Get usage stats for a project')
->groups(['api', 'projects', 'usage'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'projects')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForConsole')
->inject('dbForProject')
->inject('register')
->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) {
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
throw new Exception(Exception::PROJECT_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$dbForProject->setNamespace("_{$project->getInternalId()}");
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
});
App::patch('/v1/projects/:projectId')
->desc('Update Project')
->groups(['api', 'projects'])
@ -692,7 +615,7 @@ App::patch('/v1/projects/:projectId/auth/password-history')
});
App::patch('/v1/projects/:projectId/auth/password-dictionary')
->desc('Update authentication password disctionary status. Use this endpoint to enable or disable the dicitonary check for user password')
->desc('Update authentication password dictionary status. Use this endpoint to enable or disable the dicitonary check for user password')
->groups(['api', 'projects'])
->label('scope', 'projects.write')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
@ -722,6 +645,37 @@ App::patch('/v1/projects/:projectId/auth/password-dictionary')
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::patch('/v1/projects/:projectId/auth/personal-data')
->desc('Enable or disable checking user passwords for similarity with their personal data.')
->groups(['api', 'projects'])
->label('scope', 'projects.write')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'projects')
->label('sdk.method', 'updatePersonalDataCheck')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('enabled', false, new Boolean(false), 'Set whether or not to check a password for similarity with personal data. Default is false.')
->inject('response')
->inject('dbForConsole')
->action(function (string $projectId, bool $enabled, Response $response, Database $dbForConsole) {
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
throw new Exception(Exception::PROJECT_NOT_FOUND);
}
$auths = $project->getAttribute('auths', []);
$auths['personalDataCheck'] = $enabled;
$dbForConsole->updateDocument('projects', $project->getId(), $project
->setAttribute('auths', $auths));
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::patch('/v1/projects/:projectId/auth/max-sessions')
->desc('Update Project user sessions limit')
->groups(['api', 'projects'])
@ -1469,9 +1423,12 @@ App::post('/v1/projects/:projectId/domains')
throw new Exception(Exception::PROJECT_NOT_FOUND);
}
if ($domain === App::getEnv('_APP_DOMAIN', '') || $domain === App::getEnv('_APP_DOMAIN_TARGET', '')) {
throw new Exception(Exception::DOMAIN_FORBIDDEN);
}
$document = $dbForConsole->findOne('domains', [
Query::equal('domain', [$domain]),
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('domain', [$domain])
]);
if ($document && !$document->isEmpty()) {
@ -1669,6 +1626,10 @@ App::delete('/v1/projects/:projectId/domains/:domainId')
throw new Exception(Exception::DOMAIN_NOT_FOUND);
}
if ($domain->getAttribute('domain') === App::getEnv('_APP_DOMAIN', '') || $domain->getAttribute('domain') === App::getEnv('_APP_DOMAIN_TARGET', '')) {
throw new Exception(Exception::DOMAIN_FORBIDDEN);
}
$dbForConsole->deleteDocument('domains', $domain->getId());
$dbForConsole->deleteCachedDocument('projects', $project->getId());

296
app/controllers/api/storage.php
View file

@ -41,6 +41,7 @@ use Utopia\Validator\HexColor;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\DSN\DSN;
use Utopia\Swoole\Request;
App::post('/v1/storage/buckets')
@ -50,7 +51,6 @@ App::post('/v1/storage/buckets')
->label('event', 'buckets.[bucketId].create')
->label('audits.event', 'bucket.create')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createBucket')
@ -79,7 +79,7 @@ App::post('/v1/storage/buckets')
$permissions = Permission::aggregate($permissions);
try {
$files = Config::getParam('collections', [])['files'] ?? [];
$files = (Config::getParam('collections', [])['buckets'] ?? [])['files'] ?? [];
if (empty($files)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Files collection is not configured.');
}
@ -146,7 +146,6 @@ App::get('/v1/storage/buckets')
->desc('List buckets')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listBuckets')
@ -193,7 +192,6 @@ App::get('/v1/storage/buckets/:bucketId')
->desc('Get Bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucket')
@ -222,7 +220,6 @@ App::put('/v1/storage/buckets/:bucketId')
->label('event', 'buckets.[bucketId].update')
->label('audits.event', 'bucket.update')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'updateBucket')
@ -290,7 +287,6 @@ App::delete('/v1/storage/buckets/:bucketId')
->label('audits.event', 'bucket.delete')
->label('event', 'buckets.[bucketId].delete')
->label('audits.resource', 'bucket/{request.bucketId}')
->label('usage.metric', 'buckets.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'deleteBucket')
@ -333,8 +329,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('audits.event', 'file.create')
->label('event', 'buckets.[bucketId].files.[fileId].create')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -359,8 +353,7 @@ App::post('/v1/storage/buckets/:bucketId/files')
->inject('mode')
->inject('deviceFiles')
->inject('deviceLocal')
->inject('deletes')
->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal, Delete $deletes) {
->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
@ -504,7 +497,7 @@ App::post('/v1/storage/buckets/:bucketId/files')
}
if ($chunksUploaded === $chunks) {
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)) === Storage::DEVICE_LOCAL) {
if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled' && $bucket->getAttribute('antivirus', true) && $fileSize <= APP_LIMIT_ANTIVIRUS && $deviceFiles->getType() === Storage::DEVICE_LOCAL) {
$antivirus = new Network(
App::getEnv('_APP_STORAGE_ANTIVIRUS_HOST', 'clamav'),
(int) App::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310)
@ -570,6 +563,7 @@ App::post('/v1/storage/buckets/:bucketId/files')
'$id' => $fileId,
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getInternalId(),
'name' => $fileName,
'path' => $path,
'signature' => $fileHash,
@ -619,6 +613,7 @@ App::post('/v1/storage/buckets/:bucketId/files')
'$id' => ID::custom($fileId),
'$permissions' => $permissions,
'bucketId' => $bucket->getId(),
'bucketInternalId' => $bucket->getInternalId(),
'name' => $fileName,
'path' => $path,
'signature' => '',
@ -656,11 +651,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->setContext('bucket', $bucket)
;
$deletes
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
->setResource('file/' . $file->getId())
;
$metadata = null; // was causing leaks as it was passed by reference
$response
@ -674,8 +664,6 @@ App::get('/v1/storage/buckets/:bucketId/files')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listFiles')
->label('sdk.description', '/docs/references/storage/list-files.md')
@ -751,8 +739,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFile')
->label('sdk.description', '/docs/references/storage/get-file.md')
@ -800,8 +786,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->label('cache', true)
->label('cache.resourceType', 'bucket/{request.bucketId}')
->label('cache.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFilePreview')
@ -963,8 +947,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->desc('Get File for Download')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileDownload')
@ -1103,8 +1085,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->desc('Get File for View')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileView')
@ -1257,8 +1237,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].update')
->label('audits.event', 'file.update')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.update')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1365,8 +1343,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].delete')
->label('audits.event', 'file.delete')
->label('audits.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.delete')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@ -1468,103 +1444,62 @@ App::get('/v1/storage/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_BUCKETS,
METRIC_FILES,
METRIC_FILES_STORAGE,
];
$metrics = [
'project.$all.storage.size',
'buckets.$all.count.total',
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
'files.$all.storage.size',
'files.$all.count.total',
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'bucketsCount' => $stats['buckets.$all.count.total'],
'bucketsCreate' => $stats['buckets.$all.requests.create'],
'bucketsRead' => $stats['buckets.$all.requests.read'],
'bucketsUpdate' => $stats['buckets.$all.requests.update'],
'bucketsDelete' => $stats['buckets.$all.requests.delete'],
'storage' => $stats['project.$all.storage.size'],
'filesCount' => $stats['files.$all.count.total'],
'filesCreate' => $stats['files.$all.requests.create'],
'filesRead' => $stats['files.$all.requests.read'],
'filesUpdate' => $stats['files.$all.requests.update'],
'filesDelete' => $stats['files.$all.requests.delete'],
]);
$format = (match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
});
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
$response->dynamic(new Document([
'range' => $range,
'bucketsTotal' => $usage[$metrics[0]],
'filesTotal' => $usage[$metrics[1]],
'filesStorage' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for a storage bucket')
->desc('Get usage stats for storage bucket')
->groups(['api', 'storage', 'usage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
@ -1585,86 +1520,55 @@ App::get('/v1/storage/:bucketId/usage')
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
];
$metrics = [
"files.{$bucketId}.count.total",
"files.{$bucketId}.storage.size",
"files.{$bucketId}.requests.create",
"files.{$bucketId}.requests.read",
"files.{$bucketId}.requests.update",
"files.{$bucketId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'filesCount' => $stats[$metrics[0]],
'filesStorage' => $stats[$metrics[1]],
'filesCreate' => $stats[$metrics[2]],
'filesRead' => $stats[$metrics[3]],
'filesUpdate' => $stats[$metrics[4]],
'filesDelete' => $stats[$metrics[5]],
]);
$format = (match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
});
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
$response->dynamic(new Document([
'range' => $range,
'filesTotal' => $usage[$metrics[0]],
'filesStorage' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_BUCKETS);
});

12
app/controllers/api/teams.php
View file

@ -338,18 +338,6 @@ App::delete('/v1/teams/:teamId')
throw new Exception(Exception::TEAM_NOT_FOUND);
}
$memberships = $dbForProject->find('memberships', [
Query::equal('teamId', [$teamId]),
Query::limit(2000), // TODO fix members limit
]);
// TODO delete all members individually from the user object
foreach ($memberships as $membership) {
if (!$dbForProject->deleteDocument('memberships', $membership->getId())) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove membership for team from DB');
}
}
if (!$dbForProject->deleteDocument('teams', $teamId)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove team from DB');
}

211
app/controllers/api/users.php
View file

@ -37,6 +37,7 @@ use MaxMind\Db\Reader;
use Utopia\Validator\Integer;
use Appwrite\Auth\Validator\PasswordHistory;
use Appwrite\Auth\Validator\PasswordDictionary;
use Appwrite\Auth\Validator\PersonalData;
/** TODO: Remove function when we move to using utopia/platform */
function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $events): Document
@ -53,6 +54,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
? ID::unique()
: ID::custom($userId);
if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
$personalDataValidator = new PersonalData($userId, $email, $name, $phone);
if (!$personalDataValidator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
}
}
$password = (!empty($password)) ? ($hash === 'plaintext' ? Auth::passwordHash($password, $hash, $hashOptionsObject) : $password) : null;
$user = $dbForProject->createDocument('users', new Document([
'$id' => $userId,
@ -97,7 +105,6 @@ App::post('/v1/users')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'create')
@ -130,7 +137,6 @@ App::post('/v1/users/bcrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createBcryptUser')
@ -161,7 +167,6 @@ App::post('/v1/users/md5')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createMD5User')
@ -192,7 +197,6 @@ App::post('/v1/users/argon2')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createArgon2User')
@ -223,7 +227,6 @@ App::post('/v1/users/sha')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createSHAUser')
@ -261,7 +264,6 @@ App::post('/v1/users/phpass')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createPHPassUser')
@ -292,7 +294,6 @@ App::post('/v1/users/scrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptUser')
@ -336,7 +337,6 @@ App::post('/v1/users/scrypt-modified')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptModifiedUser')
@ -367,7 +367,6 @@ App::get('/v1/users')
->desc('List Users')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'list')
@ -414,7 +413,6 @@ App::get('/v1/users/:userId')
->desc('Get User')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'get')
@ -440,7 +438,6 @@ App::get('/v1/users/:userId/prefs')
->desc('Get User Preferences')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'getPrefs')
@ -468,7 +465,6 @@ App::get('/v1/users/:userId/sessions')
->desc('List User Sessions')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listSessions')
@ -510,7 +506,6 @@ App::get('/v1/users/:userId/memberships')
->desc('List User Memberships')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listMemberships')
@ -550,7 +545,6 @@ App::get('/v1/users/:userId/logs')
->desc('List User Logs')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listLogs')
@ -636,7 +630,6 @@ App::patch('/v1/users/:userId/status')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateStatus')
@ -704,6 +697,41 @@ App::put('/v1/users/:userId/labels')
$response->dynamic($user, Response::MODEL_USER);
});
App::patch('/v1/users/:userId/verification')
->desc('Update Email Verification')
->groups(['api', 'users'])
->label('event', 'users.[userId].update.verification')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
->label('sdk.description', '/docs/references/users/update-user-email-verification.md')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USER)
->param('userId', '', new UID(), 'User ID.')
->param('emailVerification', false, new Boolean(), 'User email verification status.')
->inject('response')
->inject('dbForProject')
->inject('events')
->action(function (string $userId, bool $emailVerification, Response $response, Database $dbForProject, Event $events) {
$user = $dbForProject->getDocument('users', $userId);
if ($user->isEmpty()) {
throw new Exception(Exception::USER_NOT_FOUND);
}
$user = $dbForProject->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification));
$events
->setParam('userId', $user->getId());
$response->dynamic($user, Response::MODEL_USER);
});
App::patch('/v1/users/:userId/verification/phone')
->desc('Update Phone Verification')
->groups(['api', 'users'])
@ -711,7 +739,6 @@ App::patch('/v1/users/:userId/verification/phone')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhoneVerification')
@ -748,7 +775,6 @@ App::patch('/v1/users/:userId/name')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateName')
@ -786,7 +812,6 @@ App::patch('/v1/users/:userId/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePassword')
@ -808,19 +833,25 @@ App::patch('/v1/users/:userId/password')
throw new Exception(Exception::USER_NOT_FOUND);
}
if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
$personalDataValidator = new PersonalData($userId, $user->getAttribute('email'), $user->getAttribute('name'), $user->getAttribute('phone'));
if (!$personalDataValidator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
}
}
$newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
$historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
$history = [];
$history = $user->getAttribute('passwordHistory', []);
if ($historyLimit > 0) {
$history = $user->getAttribute('passwordHistory', []);
$validator = new PasswordHistory($history, $user->getAttribute('hash'), $user->getAttribute('hashOptions'));
if (!$validator->isValid($password)) {
throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED, 'The password was recently used', 409);
throw new Exception(Exception::USER_PASSWORD_RECENTLY_USED);
}
$history[] = $newPassword;
array_slice($history, (count($history) - $historyLimit), $historyLimit);
$history = array_slice($history, (count($history) - $historyLimit), $historyLimit);
}
$user
@ -845,7 +876,6 @@ App::patch('/v1/users/:userId/email')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmail')
@ -892,7 +922,6 @@ App::patch('/v1/users/:userId/phone')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhone')
@ -937,7 +966,6 @@ App::patch('/v1/users/:userId/verification')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{request.userId}')
->label('audits.userId', '{request.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@ -970,7 +998,6 @@ App::patch('/v1/users/:userId/prefs')
->groups(['api', 'users'])
->label('event', 'users.[userId].update.prefs')
->label('scope', 'users.write')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePrefs')
@ -1006,7 +1033,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSession')
@ -1050,7 +1076,6 @@ App::delete('/v1/users/:userId/sessions')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSessions')
@ -1093,7 +1118,6 @@ App::delete('/v1/users/:userId')
->label('scope', 'users.write')
->label('audits.event', 'user.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'delete')
@ -1140,96 +1164,59 @@ App::get('/v1/users/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_USERS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
->inject('response')
->inject('dbForProject')
->inject('register')
->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_USERS,
METRIC_SESSIONS,
];
$metrics = [
'users.$all.count.total',
'users.$all.requests.create',
'users.$all.requests.read',
'users.$all.requests.update',
'users.$all.requests.delete',
'sessions.$all.requests.create',
'sessions.$all.requests.delete',
"sessions.$provider.requests.create",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'usersCount' => $stats['users.$all.count.total'] ?? [],
'usersCreate' => $stats['users.$all.requests.create'] ?? [],
'usersRead' => $stats['users.$all.requests.read'] ?? [],
'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_USERS);
$response->dynamic(new Document([
'range' => $range,
'usersTotal' => $usage[$metrics[0]],
'sessionsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_USERS);
});

2
app/controllers/general.php
View file

@ -435,6 +435,7 @@ App::error()
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('database', $project->getAttribute('database', 'console'));
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
$log->addTag('verboseType', get_class($error));
@ -646,6 +647,7 @@ App::get('/.well-known/acme-challenge/*')
});
include_once __DIR__ . '/shared/api.php';
include_once __DIR__ . '/shared/api/auth.php';
foreach (Config::getParam('services', []) as $service) {
include_once $service['controller'];

1
app/controllers/mock.php
View file

@ -7,7 +7,6 @@ use Utopia\Database\Document;
use Utopia\Validator\Host;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\App;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Integer;

211
app/controllers/shared/api.php
View file

@ -5,10 +5,11 @@ use Appwrite\Event\Audit;
use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Extend\Exception;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Request;
use Utopia\App;
@ -47,43 +48,99 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
return $label;
};
$databaseListener = function (string $event, Document $document, Stats $usage) {
$multiplier = 1;
$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
$value = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$multiplier = -1;
$value = -1;
}
$collection = $document->getCollection();
switch ($collection) {
case 'users':
$usage->setParam('users.{scope}.count.total', 1 * $multiplier);
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage
->addMetric(METRIC_TEAMS, $value); // per project
break;
case 'databases':
$usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'users':
$queueForUsage
->addMetric(METRIC_USERS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case 'buckets':
$usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'sessions': // sessions
$queueForUsage
->addMetric(METRIC_SESSIONS, $value); //per project
break;
case 'deployments':
$usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
case $document->getCollection() === 'databases': // databases
$queueForUsage
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$collectionInternalId = $parts[3] ?? 0;
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$queueForUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'deployments':
$queueForUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);// per function
break;
case $document->getCollection() === 'executions':
$queueForUsage
->addMetric(METRIC_EXECUTIONS, $value) // per project
->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
break;
default:
if (strpos($collection, 'bucket_') === 0) {
$usage
->setParam('bucketId', $document->getAttribute('bucketId'))
->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
->setParam('files.{scope}.count.total', 1 * $multiplier);
} elseif (strpos($collection, 'database_') === 0) {
$usage
->setParam('databaseId', $document->getAttribute('databaseId'));
if (strpos($collection, '_collection_') !== false) {
$usage
->setParam('collectionId', $document->getAttribute('$collectionId'))
->setParam('documents.{scope}.count.total', 1 * $multiplier);
} else {
$usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
}
}
break;
}
};
@ -97,13 +154,13 @@ App::init()
->inject('user')
->inject('events')
->inject('audits')
->inject('usage')
->inject('deletes')
->inject('database')
->inject('dbForProject')
->inject('queueForUsage')
->inject('mode')
->inject('mails')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode, Mail $mails) use ($databaseListener) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Usage $queueForUsage, string $mode, Mail $mails) use ($databaseListener) {
$route = $utopia->match($request);
@ -122,11 +179,11 @@ App::init()
foreach ($abuseKeyLabel as $abuseKey) {
$timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
$timeLimit
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
$timeLimitArray[] = $timeLimit;
}
@ -185,14 +242,6 @@ App::init()
->setProject($project)
->setUser($user);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('project.{scope}.network.requests', 1)
->setParam('httpMethod', $request->getMethod())
->setParam('project.{scope}.network.inbound', 0)
->setParam('project.{scope}.network.outbound', 0);
$smtp = $project->getAttribute('smtp', []);
if (!empty($smtp) && ($smtp['enabled'] ?? false)) {
$mails
@ -207,9 +256,10 @@ App::init()
$deletes->setProject($project);
$database->setProject($project);
$dbForProject->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
;
$useCache = $route->getLabel('cache', false);
@ -372,25 +422,27 @@ App::shutdown()
->inject('user')
->inject('events')
->inject('audits')
->inject('usage')
->inject('deletes')
->inject('database')
->inject('mode')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('mode')
->inject('dbForConsole')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Database $dbForConsole) use ($parseLabel) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, string $mode, Database $dbForConsole) use ($parseLabel) {
$responsePayload = $response->getPayload();
if (!empty($events->getEvent())) {
if (empty($events->getPayload())) {
$events->setPayload($responsePayload);
}
/**
* Trigger functions.
*/
$events
->setClass(Event::FUNCTIONS_CLASS_NAME)
->setQueue(Event::FUNCTIONS_QUEUE_NAME)
$queueForFunctions
->from($events)
->trigger();
/**
@ -499,14 +551,14 @@ App::shutdown()
$key = md5($request->getURI() . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$data = json_encode([
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
$signature = md5($data);
$cacheLog = $dbForProject->getDocument('cache', $key);
$cacheLog = Authorization::skip(fn () => $dbForProject->getDocument('cache', $key));
$accessedAt = $cacheLog->getAttribute('accessedAt', '');
$now = DateTime::now();
if ($cacheLog->isEmpty()) {
@ -530,36 +582,25 @@ App::shutdown()
}
}
if (
App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
&& $project->getId()
&& !empty($route->getLabel('sdk.namespace', null))
) { // Don't calculate console usage on admin mode
$metric = $route->getLabel('usage.metric', '');
$usageParams = $route->getLabel('usage.params', []);
if (!empty($metric)) {
$usage->setParam($metric, 1);
foreach ($usageParams as $param) {
$param = $parseLabel($param, $responsePayload, $requestParams, $user);
$parts = explode(':', $param);
if (count($parts) != 2) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
}
$usage->setParam($parts[0], $parts[1]);
if ($project->getId() !== 'console') {
if ($mode !== APP_MODE_ADMIN) {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$usage
->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
->setParam('project.{scope}.network.outbound', $response->getSize())
->submit();
$queueForUsage
->setProject($project)
->trigger();
}
/**

62
app/controllers/shared/api/auth.php Normal file
View file

@ -0,0 +1,62 @@
<?php
use Appwrite\Auth\Auth;
use Appwrite\Utopia\Request;
use Utopia\App;
use Appwrite\Extend\Exception;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
App::init()
->groups(['auth'])
->inject('utopia')
->inject('request')
->inject('project')
->action(function (App $utopia, Request $request, Document $project) {
$route = $utopia->match($request);
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
$isAppUser = Auth::isAppUser(Authorization::getRoles());
if ($isAppUser || $isPrivilegedUser) { // Skip limits for app and console devs
return;
}
$auths = $project->getAttribute('auths', []);
switch ($route->getLabel('auth.type', '')) {
case 'emailPassword':
if (($auths['emailPassword'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Email / Password authentication is disabled for this project');
}
break;
case 'magic-url':
if ($project->getAttribute('usersAuthMagicURL', true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
}
break;
case 'anonymous':
if (($auths['anonymous'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Anonymous authentication is disabled for this project');
}
break;
case 'invites':
if (($auths['invites'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
}
break;
case 'jwt':
if (($auths['JWT'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'JWT authentication is disabled for this project');
}
break;
default:
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
break;
}
});

43
app/controllers/web/console.php
View file

@ -21,13 +21,54 @@ App::get('/console/*')
->alias('auth/*')
->alias('/invite')
->alias('/login')
->alias('/card/*')
->alias('/recover')
->alias('/register/*')
->groups(['web'])
->label('permission', 'public')
->label('scope', 'home')
->inject('request')
->inject('response')
->action(function (Response $response) {
->action(function (Request $request, Response $response) {
$fallback = file_get_contents(__DIR__ . '/../../../console/index.html');
// Card SSR
if (\str_starts_with($request->getURI(), '/card')) {
$urlCunks = \explode('/', $request->getURI());
$userId = $urlCunks[\count($urlCunks) - 1] ?? '';
$domain = $request->getProtocol() . '://' . $request->getHostname();
if (!empty($userId)) {
$ogImageUrl = $domain . '/v1/cards/cloud-og?userId=' . $userId;
} else {
$ogImageUrl = $domain . '/v1/cards/cloud-og?mock=normal';
}
$ogTags = [
'<title>Appwrite Cloud Card</title>',
'<meta name="description" content="Appwrite Cloud is now LIVE! Share your Cloud card for a chance to win an exclusive Cloud hoodie!">',
'<meta property="og:url" content="' . $domain . $request->getURI() . '">',
'<meta name="og:image:type" content="image/png">',
'<meta name="og:image:width" content="1008">',
'<meta name="og:image:height" content="1008">',
'<meta property="og:type" content="website">',
'<meta property="og:title" content="Appwrite Cloud Card">',
'<meta property="og:description" content="Appwrite Cloud is now LIVE! Share your Cloud card for a chance to win an exclusive Cloud hoodie!">',
'<meta property="og:image" content="' . $ogImageUrl . '">',
'<meta name="twitter:card" content="summary_large_image">',
'<meta property="twitter:domain" content="' . $request->getHostname() . '">',
'<meta property="twitter:url" content="' . $domain . $request->getURI() . '">',
'<meta name="twitter:title" content="Appwrite Cloud Card">',
'<meta name="twitter:image:type" content="image/png">',
'<meta name="twitter:image:width" content="1008">',
'<meta name="twitter:image:height" content="1008">',
'<meta name="twitter:description" content="Appwrite Cloud is now LIVE! Share your Cloud card for a chance to win an exclusive Cloud hoodie!">',
'<meta name="twitter:image" content="' . $ogImageUrl . '">',
];
$fallback = \str_replace('<!-- {{CLOUD_OG}} -->', \implode('', $ogTags), $fallback);
}
$response->html($fallback);
});

806
app/executor.php
View file

@ -1,806 +0,0 @@
<?php
require_once __DIR__ . '/../vendor/autoload.php';
use Appwrite\Runtimes\Runtimes;
use Swoole\ConnectionPool;
use Swoole\Http\Request as SwooleRequest;
use Swoole\Http\Response as SwooleResponse;
use Swoole\Http\Server;
use Swoole\Process;
use Swoole\Runtime;
use Swoole\Timer;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Orchestration\Adapter\DockerCLI;
use Utopia\Orchestration\Orchestration;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\Backblaze;
use Utopia\Storage\Device\DOSpaces;
use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Wasabi;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Storage;
use Utopia\Swoole\Request;
use Utopia\Swoole\Response;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Boolean;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
Runtime::enableCoroutine(true, SWOOLE_HOOK_ALL);
/** Constants */
const MAINTENANCE_INTERVAL = 3600; // 3600 seconds = 1 hour
/**
* Create a Swoole table to store runtime information
*/
$activeRuntimes = new Swoole\Table(1024);
$activeRuntimes->column('id', Swoole\Table::TYPE_STRING, 256);
$activeRuntimes->column('created', Swoole\Table::TYPE_INT, 8);
$activeRuntimes->column('updated', Swoole\Table::TYPE_INT, 8);
$activeRuntimes->column('name', Swoole\Table::TYPE_STRING, 128);
$activeRuntimes->column('status', Swoole\Table::TYPE_STRING, 128);
$activeRuntimes->column('key', Swoole\Table::TYPE_STRING, 256);
$activeRuntimes->create();
/**
* Create orchestration pool
*/
$orchestrationPool = new ConnectionPool(function () {
$dockerUser = App::getEnv('DOCKERHUB_PULL_USERNAME', null);
$dockerPass = App::getEnv('DOCKERHUB_PULL_PASSWORD', null);
$orchestration = new Orchestration(new DockerCLI($dockerUser, $dockerPass));
return $orchestration;
}, 10);
/**
* Create logger instance
*/
$providerName = App::getEnv('_APP_LOGGING_PROVIDER', '');
$providerConfig = App::getEnv('_APP_LOGGING_CONFIG', '');
$logger = null;
if (!empty($providerName) && !empty($providerConfig) && Logger::hasProvider($providerName)) {
$classname = '\\Utopia\\Logger\\Adapter\\' . \ucfirst($providerName);
$adapter = new $classname($providerConfig);
$logger = new Logger($adapter);
}
function logError(Throwable $error, string $action, Utopia\Route $route = null)
{
global $logger;
if ($logger) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$log = new Log();
$log->setNamespace("executor");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
if ($route) {
$log->addTag('method', $route->getMethod());
$log->addTag('url', $route->getPath());
}
$log->addTag('code', $error->getCode());
$log->addTag('verboseType', get_class($error));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Executor log pushed with status code: ' . $responseCode);
}
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $error->getMessage());
Console::error('[Error] File: ' . $error->getFile());
Console::error('[Error] Line: ' . $error->getLine());
}
function getStorageDevice($root): Device
{
switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL))) {
case Storage::DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
App::post('/v1/runtimes')
->desc("Create a new runtime server")
->param('runtimeId', '', new Text(64), 'Unique runtime ID.')
->param('source', '', new Text(0), 'Path to source files.')
->param('destination', '', new Text(0, 0), 'Destination folder to store build files into.', true)
->param('vars', [], new Assoc(), 'Environment Variables required for the build.')
->param('commands', [], new ArrayList(new Text(1024), 100), 'Commands required to build the container. Maximum of 100 commands are allowed, each 1024 characters long.')
->param('runtime', '', new Text(128), 'Runtime for the cloud function.')
->param('baseImage', '', new Text(128), 'Base image name of the runtime.')
->param('entrypoint', '', new Text(256), 'Entrypoint of the code file.', true)
->param('remove', false, new Boolean(), 'Remove a runtime after execution.')
->param('workdir', '', new Text(256, 0), 'Working directory.', true)
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) {
if ($activeRuntimes->exists($runtimeId)) {
if ($activeRuntimes->get($runtimeId)['status'] == 'pending') {
throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500);
}
throw new Exception('Runtime already exists.', 409);
}
$container = [];
$containerId = '';
$stdout = '';
$stderr = '';
$startTime = DateTime::now();
$startTimeUnix = (new \DateTime($startTime))->getTimestamp();
$endTimeUnix = 0;
$orchestration = $orchestrationPool->get();
$secret = \bin2hex(\random_bytes(16));
if (!$remove) {
$activeRuntimes->set($runtimeId, [
'id' => $containerId,
'name' => $runtimeId,
'created' => $startTimeUnix,
'updated' => $endTimeUnix,
'status' => 'pending',
'key' => $secret,
]);
}
try {
Console::info('Building container : ' . $runtimeId);
/**
* Temporary file paths in the executor
*/
$tmpSource = "/tmp/$runtimeId/src/code.tar.gz";
$tmpBuild = "/tmp/$runtimeId/builds/code.tar.gz";
/**
* Copy code files from source to a temporary location on the executor
*/
$sourceDevice = getStorageDevice("/");
$localDevice = new Local();
$buffer = $sourceDevice->read($source);
if (!$localDevice->write($tmpSource, $buffer)) {
throw new Exception('Failed to copy source code to temporary directory', 500);
};
/**
* Create the mount folder
*/
if (!\file_exists(\dirname($tmpBuild))) {
if (!@\mkdir(\dirname($tmpBuild), 0755, true)) {
throw new Exception("Failed to create temporary directory", 500);
}
}
/**
* Create container
*/
$vars = \array_merge($vars, [
'INTERNAL_RUNTIME_KEY' => $secret,
'INTERNAL_RUNTIME_ENTRYPOINT' => $entrypoint,
]);
$vars = array_map(fn ($v) => strval($v), $vars);
$orchestration
->setCpus((int) App::getEnv('_APP_FUNCTIONS_CPUS', 0))
->setMemory((int) App::getEnv('_APP_FUNCTIONS_MEMORY', 0))
->setSwap((int) App::getEnv('_APP_FUNCTIONS_MEMORY_SWAP', 0));
/** Keep the container alive if we have commands to be executed */
$entrypoint = !empty($commands) ? [
'tail',
'-f',
'/dev/null'
] : [];
$containerId = $orchestration->run(
image: $baseImage,
name: $runtimeId,
hostname: $runtimeId,
vars: $vars,
command: $entrypoint,
labels: [
'openruntimes-id' => $runtimeId,
'openruntimes-type' => 'runtime',
'openruntimes-created' => strval($startTimeUnix),
'openruntimes-runtime' => $runtime,
],
workdir: $workdir,
volumes: [
\dirname($tmpSource) . ':/tmp:rw',
\dirname($tmpBuild) . ':/usr/code:rw'
]
);
if (empty($containerId)) {
throw new Exception('Failed to create build container', 500);
}
$orchestration->networkConnect($runtimeId, App::getEnv('OPEN_RUNTIMES_NETWORK', 'appwrite_runtimes'));
/**
* Execute any commands if they were provided
*/
if (!empty($commands)) {
$status = $orchestration->execute(
name: $runtimeId,
command: $commands,
stdout: $stdout,
stderr: $stderr,
timeout: App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900)
);
if (!$status) {
throw new Exception('Failed to build dependenices ' . $stderr, 500);
}
}
/**
* Move built code to expected build directory
*/
if (!empty($destination)) {
// Check if the build was successful by checking if file exists
if (!\file_exists($tmpBuild)) {
throw new Exception('Something went wrong during the build process', 500);
}
$destinationDevice = getStorageDevice($destination);
$outputPath = $destinationDevice->getPath(\uniqid() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$buffer = $localDevice->read($tmpBuild);
if (!$destinationDevice->write($outputPath, $buffer, $localDevice->getFileMimeType($tmpBuild))) {
throw new Exception('Failed to move built code to storage', 500);
};
$container['outputPath'] = $outputPath;
}
if (empty($stdout)) {
$stdout = 'Build Successful!';
}
$endTime = DateTime::now();
$endTimeUnix = (new \DateTime($endTime))->getTimestamp();
$duration = $endTimeUnix - $startTimeUnix;
$container = array_merge($container, [
'status' => 'ready',
'response' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB
'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB
'startTime' => $startTime,
'endTime' => $endTime,
'duration' => $duration,
]);
if (!$remove) {
$activeRuntimes->set($runtimeId, [
'id' => $containerId,
'name' => $runtimeId,
'created' => $startTimeUnix,
'updated' => $endTimeUnix,
'status' => 'Up ' . \round($duration, 2) . 's',
'key' => $secret,
]);
}
Console::success('Build Stage completed in ' . ($duration) . ' seconds');
} catch (Throwable $th) {
Console::error('Build failed: ' . $th->getMessage() . $stdout);
throw new Exception($th->getMessage() . $stdout, 500);
} finally {
// Container cleanup
if ($remove) {
if (!empty($containerId)) {
// If container properly created
$orchestration->remove($containerId, true);
$activeRuntimes->del($runtimeId);
} else {
// If whole creation failed, but container might have been initialized
try {
// Try to remove with contaier name instead of ID
$orchestration->remove($runtimeId, true);
$activeRuntimes->del($runtimeId);
} catch (Throwable $th) {
// If fails, means initialization also failed.
// Contianer is not there, no need to remove
}
}
}
// Release orchestration back to pool, we are done with it
$orchestrationPool->put($orchestration);
}
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->json($container);
});
App::get('/v1/runtimes')
->desc("List currently active runtimes")
->inject('activeRuntimes')
->inject('response')
->action(function ($activeRuntimes, Response $response) {
$runtimes = [];
foreach ($activeRuntimes as $runtime) {
$runtimes[] = $runtime;
}
$response
->setStatusCode(Response::STATUS_CODE_OK)
->json($runtimes);
});
App::get('/v1/runtimes/:runtimeId')
->desc("Get a runtime by its ID")
->param('runtimeId', '', new Text(64), 'Runtime unique ID.')
->inject('activeRuntimes')
->inject('response')
->action(function ($runtimeId, $activeRuntimes, Response $response) {
if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found', 404);
}
$runtime = $activeRuntimes->get($runtimeId);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->json($runtime);
});
App::delete('/v1/runtimes/:runtimeId')
->desc('Delete a runtime')
->param('runtimeId', '', new Text(64), 'Runtime unique ID.', false)
->inject('orchestrationPool')
->inject('activeRuntimes')
->inject('response')
->action(function (string $runtimeId, $orchestrationPool, $activeRuntimes, Response $response) {
if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found', 404);
}
Console::info('Deleting runtime: ' . $runtimeId);
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($runtimeId, true);
$activeRuntimes->del($runtimeId);
Console::success('Removed runtime container: ' . $runtimeId);
} finally {
$orchestrationPool->put($orchestration);
}
// Remove all the build containers with that same ID
// TODO:: Delete build containers
// foreach ($buildIds as $buildId) {
// try {
// Console::info('Deleting build container : ' . $buildId);
// $status = $orchestration->remove('build-' . $buildId, true);
// } catch (Throwable $th) {
// Console::error($th->getMessage());
// }
// }
$response
->setStatusCode(Response::STATUS_CODE_OK)
->send();
});
App::post('/v1/execution')
->desc('Create an execution')
->param('runtimeId', '', new Text(64), 'The runtimeID to execute.')
->param('vars', [], new Assoc(), 'Environment variables required for the build.')
->param('data', '', new Text(8192, 0), 'Data to be forwarded to the function, this is user specified.', true)
->param('timeout', 15, new Range(1, (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900)), 'Function maximum execution time in seconds.')
->inject('activeRuntimes')
->inject('response')
->action(
function (string $runtimeId, array $vars, string $data, $timeout, $activeRuntimes, Response $response) {
if (!$activeRuntimes->exists($runtimeId)) {
throw new Exception('Runtime not found. Please create the runtime.', 404);
}
for ($i = 0; $i < 5; $i++) {
if ($activeRuntimes->get($runtimeId)['status'] === 'pending') {
Console::info('Waiting for runtime to be ready...');
sleep(1);
} else {
break;
}
if ($i === 4) {
throw new Exception('Runtime failed to launch in allocated time.', 500);
}
}
$runtime = $activeRuntimes->get($runtimeId);
$secret = $runtime['key'];
if (empty($secret)) {
throw new Exception('Runtime secret not found. Please re-create the runtime.', 500);
}
Console::info('Executing Runtime: ' . $runtimeId);
$execution = [];
$executionStart = \microtime(true);
$stdout = '';
$stderr = '';
$res = '';
$statusCode = 0;
$errNo = -1;
$executorResponse = '';
$timeout ??= (int) App::getEnv('_APP_FUNCTIONS_TIMEOUT', 900);
$ch = \curl_init();
$body = \json_encode([
'variables' => $vars,
'payload' => $data,
'timeout' => $timeout
]);
\curl_setopt($ch, CURLOPT_URL, "http://" . $runtimeId . ":3000/");
\curl_setopt($ch, CURLOPT_POST, true);
\curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
\curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
\curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
\curl_setopt($ch, CURLOPT_HTTPHEADER, [
'Content-Type: application/json',
'Content-Length: ' . \strlen($body),
'x-internal-challenge: ' . $secret,
'host: null'
]);
$executorResponse = \curl_exec($ch);
$executorResponse = json_decode($executorResponse, true);
$statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
$error = \curl_error($ch);
$errNo = \curl_errno($ch);
\curl_close($ch);
switch (true) {
/** No Error. */
case $errNo === 0:
break;
/** Runtime not ready for requests yet. 111 is the swoole error code for Connection Refused - see https://openswoole.com/docs/swoole-error-code */
case $errNo === 111:
throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 406);
/** Any other CURL error */
default:
throw new Exception('An internal curl error has occurred within the executor! Error Msg: ' . $error, 500);
}
switch (true) {
case $statusCode >= 500:
$stderr = ($executorResponse ?? [])['stderr'] ?? 'Internal Runtime error.';
$stdout = ($executorResponse ?? [])['stdout'] ?? 'Internal Runtime error.';
$res = ($executorResponse ?? [])['response'] ?? '';
if (is_array($res)) {
$res = json_encode($res, JSON_UNESCAPED_UNICODE);
}
break;
case $statusCode >= 100:
$stdout = $executorResponse['stdout'];
$res = $executorResponse['response'];
if (is_array($res)) {
$res = json_encode($res, JSON_UNESCAPED_UNICODE);
}
break;
default:
$stderr = ($executorResponse ?? [])['stderr'] ?? 'Execution failed.';
$stdout = ($executorResponse ?? [])['stdout'] ?? '';
break;
}
$executionEnd = \microtime(true);
$executionTime = ($executionEnd - $executionStart);
$functionStatus = ($statusCode >= 500) ? 'failed' : 'completed';
Console::success('Function executed in ' . $executionTime . ' seconds, status: ' . $functionStatus);
$execution = [
'status' => $functionStatus,
'statusCode' => $statusCode,
'response' => \mb_strcut($res, 0, 1000000), // Limit to 1MB
'stdout' => \mb_strcut($stdout, 0, 1000000), // Limit to 1MB
'stderr' => \mb_strcut($stderr, 0, 1000000), // Limit to 1MB
'duration' => $executionTime,
];
/** Update swoole table */
$runtime['updated'] = \time();
$activeRuntimes->set($runtimeId, $runtime);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->json($execution);
}
);
App::setMode(App::MODE_TYPE_PRODUCTION); // Define Mode
$http = new Server("0.0.0.0", 80);
/** Set Resources */
App::setResource('orchestrationPool', fn() => $orchestrationPool);
App::setResource('activeRuntimes', fn() => $activeRuntimes);
/** Set callbacks */
App::error()
->inject('utopia')
->inject('error')
->inject('request')
->inject('response')
->action(function (App $utopia, throwable $error, Request $request, Response $response) {
$route = $utopia->match($request);
logError($error, "httpError", $route);
switch ($error->getCode()) {
case 400: // Error allowed publicly
case 401: // Error allowed publicly
case 402: // Error allowed publicly
case 403: // Error allowed publicly
case 404: // Error allowed publicly
case 406: // Error allowed publicly
case 409: // Error allowed publicly
case 412: // Error allowed publicly
case 425: // Error allowed publicly
case 429: // Error allowed publicly
case 501: // Error allowed publicly
case 503: // Error allowed publicly
$code = $error->getCode();
break;
default:
$code = 500; // All other errors get the generic 500 server error status code
}
$output = [
'message' => $error->getMessage(),
'code' => $error->getCode(),
'file' => $error->getFile(),
'line' => $error->getLine(),
'trace' => $error->getTrace(),
'version' => App::getEnv('_APP_VERSION', 'UNKNOWN')
];
$response
->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
->addHeader('Expires', '0')
->addHeader('Pragma', 'no-cache')
->setStatusCode($code);
$response->json($output);
});
App::init()
->inject('request')
->action(function (Request $request) {
$secretKey = $request->getHeader('x-appwrite-executor-key', '');
if (empty($secretKey)) {
throw new Exception('Missing executor key', 401);
}
if ($secretKey !== App::getEnv('_APP_EXECUTOR_SECRET', '')) {
throw new Exception('Missing executor key', 401);
}
});
$http->on('start', function ($http) {
global $orchestrationPool;
global $activeRuntimes;
/**
* Warmup: make sure images are ready to run fast 🚀
*/
$runtimes = new Runtimes('v2');
$allowList = empty(App::getEnv('_APP_FUNCTIONS_RUNTIMES')) ? [] : \explode(',', App::getEnv('_APP_FUNCTIONS_RUNTIMES'));
$runtimes = $runtimes->getAll(true, $allowList);
foreach ($runtimes as $runtime) {
go(function () use ($runtime, $orchestrationPool) {
try {
$orchestration = $orchestrationPool->get();
Console::info('Warming up ' . $runtime['name'] . ' ' . $runtime['version'] . ' environment...');
$response = $orchestration->pull($runtime['image']);
if ($response) {
Console::success("Successfully Warmed up {$runtime['name']} {$runtime['version']}!");
} else {
Console::warning("Failed to Warmup {$runtime['name']} {$runtime['version']}!");
}
} catch (\Throwable $th) {
} finally {
$orchestrationPool->put($orchestration);
}
});
}
/**
* Remove residual runtimes
*/
Console::info('Removing orphan runtimes...');
try {
$orchestration = $orchestrationPool->get();
$orphans = $orchestration->list(['label' => 'openruntimes-type=runtime']);
} finally {
$orchestrationPool->put($orchestration);
}
foreach ($orphans as $runtime) {
go(function () use ($runtime, $orchestrationPool) {
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($runtime->getName(), true);
Console::success("Successfully removed {$runtime->getName()}");
} catch (\Throwable $th) {
Console::error('Orphan runtime deletion failed: ' . $th->getMessage());
} finally {
$orchestrationPool->put($orchestration);
}
});
}
/**
* Register handlers for shutdown
*/
@Process::signal(SIGINT, function () use ($http) {
$http->shutdown();
});
@Process::signal(SIGQUIT, function () use ($http) {
$http->shutdown();
});
@Process::signal(SIGKILL, function () use ($http) {
$http->shutdown();
});
@Process::signal(SIGTERM, function () use ($http) {
$http->shutdown();
});
/**
* Run a maintenance worker every MAINTENANCE_INTERVAL seconds to remove inactive runtimes
*/
Timer::tick(MAINTENANCE_INTERVAL * 1000, function () use ($orchestrationPool, $activeRuntimes) {
Console::warning("Running maintenance task ...");
foreach ($activeRuntimes as $runtime) {
$inactiveThreshold = \time() - App::getEnv('_APP_FUNCTIONS_INACTIVE_THRESHOLD', 60);
if ($runtime['updated'] < $inactiveThreshold) {
go(function () use ($runtime, $orchestrationPool, $activeRuntimes) {
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($runtime['name'], true);
$activeRuntimes->del($runtime['name']);
Console::success("Successfully removed {$runtime['name']}");
} catch (\Throwable $th) {
Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());
} finally {
$orchestrationPool->put($orchestration);
}
});
}
}
});
});
$http->on('beforeShutdown', function () {
global $orchestrationPool;
Console::info('Cleaning up containers before shutdown...');
$orchestration = $orchestrationPool->get();
$functionsToRemove = $orchestration->list(['label' => 'openruntimes-type=runtime']);
$orchestrationPool->put($orchestration);
foreach ($functionsToRemove as $container) {
go(function () use ($orchestrationPool, $container) {
try {
$orchestration = $orchestrationPool->get();
$orchestration->remove($container->getId(), true);
Console::info('Removed container ' . $container->getName());
} catch (\Throwable $th) {
Console::error('Failed to remove container: ' . $container->getName());
} finally {
$orchestrationPool->put($orchestration);
}
});
}
});
$http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) {
$request = new Request($swooleRequest);
$response = new Response($swooleResponse);
$app = new App('UTC');
try {
$app->run($request, $response);
} catch (\Throwable $th) {
logError($th, "serverError");
$swooleResponse->setStatusCode(500);
$output = [
'message' => 'Error: ' . $th->getMessage(),
'code' => 500,
'file' => $th->getFile(),
'line' => $th->getLine(),
'trace' => $th->getTrace()
];
$swooleResponse->end(\json_encode($output));
}
});
$http->start();

42
app/http.php
View file

@ -22,6 +22,7 @@ use Utopia\Swoole\Files;
use Appwrite\Utopia\Request;
use Utopia\Logger\Log;
use Utopia\Logger\Log\User;
use Utopia\Pools\Group;
$http = new Server("0.0.0.0", App::getEnv('PORT', 80));
@ -60,6 +61,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$app = new App('UTC');
go(function () use ($register, $app) {
$pools = $register->get('pools'); /** @var Group $pools */
App::setResource('pools', fn() => $pools);
// wait for database to be ready
$attempts = 0;
$max = 10;
@ -68,8 +72,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
do {
try {
$attempts++;
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */
break; // leave the do-while if successful
} catch (\Exception $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
@ -80,17 +83,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
}
} while ($attempts < $max);
App::setResource('db', fn () => $db);
App::setResource('cache', fn () => $redis);
/** @var Utopia\Database\Database $dbForConsole */
$dbForConsole = $app->getResource('dbForConsole');
Console::success('[Setup] - Server database init started...');
/** @var array $collections */
$collections = Config::getParam('collections', []);
try {
Console::success('[Setup] - Creating database: appwrite...');
$dbForConsole->create();
@ -108,7 +102,10 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$adapter->setup();
}
foreach ($collections as $key => $collection) {
/** @var array $collections */
$collections = Config::getParam('collections', []);
$consoleCollections = $collections['console'];
foreach ($consoleCollections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
@ -148,7 +145,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$dbForConsole->createCollection($key, $attributes, $indexes);
}
if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) {
if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) {
Console::success('[Setup] - Creating default bucket...');
$dbForConsole->createDocument('buckets', new Document([
'$id' => ID::custom('default'),
@ -173,7 +170,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$bucket = $dbForConsole->getDocument('buckets', 'default');
Console::success('[Setup] - Creating files collection for default bucket...');
$files = $collections['files'] ?? [];
$files = $collections['buckets']['files'] ?? [];
if (empty($files)) {
throw new Exception('Files collection is not configured.');
}
@ -208,6 +205,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
$dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
$pools->reclaim();
Console::success('[Setup] - Server database init completed...');
});
@ -239,11 +238,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$app = new App('UTC');
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
App::setResource('db', fn () => $db);
App::setResource('cache', fn () => $redis);
$pools = $register->get('pools');
App::setResource('pools', fn() => $pools);
try {
Authorization::cleanRoles();
@ -327,13 +323,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
$swooleResponse->end(\json_encode($output));
} finally {
/** @var PDOPool $dbPool */
$dbPool = $register->get('dbPool');
$dbPool->put($db);
/** @var RedisPool $redisPool */
$redisPool = $register->get('redisPool');
$redisPool->put($redis);
$pools->reclaim();
}
});

496
app/init.php
View file

@ -18,38 +18,26 @@ ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Event\Usage;
use Appwrite\Extend\Exception;
use Appwrite\Auth\Auth;
use Appwrite\DSN\DSN;
use Appwrite\Event\Audit;
use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Event\Phone;
use Appwrite\Extend\Exception;
use Appwrite\Extend\PDO;
use Appwrite\GraphQL\Promises\Adapter\Swoole;
use Appwrite\Event\Delete;
use Appwrite\GraphQL\Schema;
use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Origin;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\Usage\Stats;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOConfig;
use Swoole\Database\PDOPool;
use Swoole\Database\RedisConfig;
use Swoole\Database\RedisPool;
use Appwrite\URL\URL as AppwriteURL;
use Utopia\App;
use Utopia\Database\Helpers\ID;
use Utopia\Logger\Logger;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
@ -57,7 +45,9 @@ use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Datetime as DatetimeValidator;
use Utopia\Database\Validator\Structure;
use Utopia\Locale\Locale;
use Utopia\DSN\DSN;
use Utopia\Messaging\Adapters\SMS\Mock;
use Appwrite\GraphQL\Promises\Adapter\Swoole;
use Utopia\Messaging\Adapters\SMS\Msg91;
use Utopia\Messaging\Adapters\SMS\Telesign;
use Utopia\Messaging\Adapters\SMS\TextMagic;
@ -71,6 +61,19 @@ use Utopia\Storage\Device\Linode;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\S3;
use Utopia\Storage\Device\Wasabi;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Adapter\MySQL;
use Utopia\Pools\Group;
use Utopia\Pools\Pool;
use Ahc\Jwt\JWT;
use Ahc\Jwt\JWTException;
use Appwrite\Event\Func;
use MaxMind\Db\Reader;
use PHPMailer\PHPMailer\PHPMailer;
use Swoole\Database\PDOProxy;
use Utopia\Queue;
use Utopia\Queue\Connection;
use Utopia\Storage\Storage;
use Utopia\Validator\Range;
use Utopia\Validator\IP;
@ -160,6 +163,7 @@ const DELETE_TYPE_BUCKETS = 'buckets';
const DELETE_TYPE_SESSIONS = 'sessions';
const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource';
const DELETE_TYPE_SCHEDULES = 'schedules';
// Compression type
const COMPRESSION_TYPE_NONE = 'none';
const COMPRESSION_TYPE_GZIP = 'gzip';
@ -177,6 +181,39 @@ const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';
const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections';
const METRIC_DOCUMENTS = 'documents';
const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents';
const METRIC_BUCKETS = 'buckets';
const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';
const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files';
const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage';
const METRIC_FUNCTIONS = 'functions';
const METRIC_DEPLOYMENTS = 'deployments';
const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage';
const METRIC_BUILDS = 'builds';
const METRIC_BUILDS_STORAGE = 'builds.storage';
const METRIC_BUILDS_COMPUTE = 'builds.compute';
const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds';
const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute';
const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
const METRIC_EXECUTIONS = 'executions';
const METRIC_EXECUTIONS_COMPUTE = 'executions.compute';
const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions';
const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute';
const METRIC_NETWORK_REQUESTS = 'network.requests';
const METRIC_NETWORK_INBOUND = 'network.inbound';
const METRIC_NETWORK_OUTBOUND = 'network.outbound';
$register = new Registry();
@ -192,6 +229,7 @@ Config::load('providers', __DIR__ . '/config/providers.php');
Config::load('platforms', __DIR__ . '/config/platforms.php');
Config::load('collections', __DIR__ . '/config/collections.php');
Config::load('runtimes', __DIR__ . '/config/runtimes.php');
Config::load('usage', __DIR__ . '/config/usage.php');
Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes
Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes
Config::load('services', __DIR__ . '/config/services.php'); // List of services
@ -539,81 +577,190 @@ $register->set('logger', function () {
$adapter = new $classname($providerConfig);
return new Logger($adapter);
});
$register->set('dbPool', function () {
// Register DB connection
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbPort = App::getEnv('_APP_DB_PORT', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$register->set('pools', function () {
$group = new Group();
$pool = new PDOPool(
(new PDOConfig())
->withHost($dbHost)
->withPort($dbPort)
->withDbName($dbScheme)
->withCharset('utf8mb4')
->withUsername($dbUser)
->withPassword($dbPass)
->withOptions([
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true,
]),
64
);
$fallbackForDB = AppwriteURL::unparse([
'scheme' => 'mariadb',
'host' => App::getEnv('_APP_DB_HOST', 'mariadb'),
'port' => App::getEnv('_APP_DB_PORT', '3306'),
'user' => App::getEnv('_APP_DB_USER', ''),
'pass' => App::getEnv('_APP_DB_PASS', ''),
]);
$fallbackForRedis = AppwriteURL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]);
return $pool;
});
$register->set('redisPool', function () {
$redisHost = App::getEnv('_APP_REDIS_HOST', '');
$redisPort = App::getEnv('_APP_REDIS_PORT', '');
$redisUser = App::getEnv('_APP_REDIS_USER', '');
$redisPass = App::getEnv('_APP_REDIS_PASS', '');
$redisAuth = '';
$connections = [
'console' => [
'type' => 'database',
'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB),
'multiple' => false,
'schemes' => ['mariadb', 'mysql'],
],
'database' => [
'type' => 'database',
'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB),
'multiple' => true,
'schemes' => ['mariadb', 'mysql'],
],
'queue' => [
'type' => 'queue',
'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis),
'multiple' => false,
'schemes' => ['redis'],
],
'pubsub' => [
'type' => 'pubsub',
'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis),
'multiple' => false,
'schemes' => ['redis'],
],
'cache' => [
'type' => 'cache',
'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis),
'multiple' => true,
'schemes' => ['redis'],
],
];
if ($redisUser && $redisPass) {
$redisAuth = $redisUser . ':' . $redisPass;
$maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151);
$instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14);
$multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled';
if ($multiprocessing) {
$workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
} else {
$workerCount = 1;
}
$pool = new RedisPool(
(new RedisConfig())
->withHost($redisHost)
->withPort($redisPort)
->withAuth($redisAuth)
->withDbIndex(0),
64
);
return $pool;
});
$register->set('influxdb', function () {
// Register DB connection
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if (empty($host) || empty($port)) {
return;
if ($workerCount > $instanceConnections) {
throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500);
}
$driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
$client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
$client->setDriver($driver);
return $client;
$poolSize = (int)($instanceConnections / $workerCount);
foreach ($connections as $key => $connection) {
$type = $connection['type'] ?? '';
$dsns = $connection['dsns'] ?? '';
$multipe = $connection['multiple'] ?? false;
$schemes = $connection['schemes'] ?? [];
$config = [];
$dsns = explode(',', $connection['dsns'] ?? '');
foreach ($dsns as &$dsn) {
$dsn = explode('=', $dsn);
$name = ($multipe) ? $key . '_' . $dsn[0] : $key;
$dsn = $dsn[1] ?? '';
$config[] = $name;
if (empty($dsn)) {
//throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}");
continue;
}
$dsn = new DSN($dsn);
$dsnHost = $dsn->getHost();
$dsnPort = $dsn->getPort();
$dsnUser = $dsn->getUser();
$dsnPass = $dsn->getPassword();
$dsnScheme = $dsn->getScheme();
$dsnDatabase = $dsn->getPath();
if (!in_array($dsnScheme, $schemes)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme");
}
/**
* Get Resource
*
* Creation could be reused accross connection types like database, cache, queue, etc.
*
* Resource assignment to an adapter will happen below.
*/
switch ($dsnScheme) {
case 'mysql':
case 'mariadb':
$resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) {
return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array(
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true
));
});
};
break;
case 'redis':
$resource = function () use ($dsnHost, $dsnPort, $dsnPass) {
$redis = new Redis();
@$redis->pconnect($dsnHost, (int)$dsnPort);
if ($dsnPass) {
$redis->auth($dsnPass);
}
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
};
break;
default:
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme");
break;
}
$pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
// Get Adapter
$adapter = null;
switch ($type) {
case 'database':
$adapter = match ($dsn->getScheme()) {
'mariadb' => new MariaDB($resource()),
'mysql' => new MySQL($resource()),
default => null
};
$adapter->setDefaultDatabase($dsn->getPath());
break;
case 'pubsub':
$adapter = $resource();
break;
case 'queue':
$adapter = match ($dsn->getScheme()) {
'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()),
default => null
};
break;
case 'cache':
$adapter = match ($dsn->getScheme()) {
'redis' => new RedisCache($resource()),
default => null
};
break;
default:
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation.");
break;
}
return $adapter;
});
$group->add($pool);
}
Config::setParam('pools-' . $key, $config);
}
return $group;
});
$register->set('statsd', function () {
// Register DB connection
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
return $statsd;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@ -651,37 +798,9 @@ $register->set('passwordsDictionary', function () {
$content = array_flip($content);
return $content;
});
$register->set('db', function () {
// This is usually for our workers or CLI commands scope
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbPort = App::getEnv('_APP_DB_PORT', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true,
));
return $pdo;
});
$register->set('cache', function () {
// This is usually for our workers or CLI commands scope
$redis = new Redis();
$redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', ''));
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
return $redis;
});
$register->set('promiseAdapter', function () {
return new Swoole();
});
/*
* Localization
*/
@ -697,7 +816,6 @@ foreach ($locales as $locale) {
if (!\file_exists($path)) {
$path = __DIR__ . '/config/locale/translations/' . \substr($code, 0, 2) . '.json'; // if `ar-ae` doesn't exist, look for `ar`
if (!\file_exists($path)) {
var_dump('Unable to find tralsnations for ' . $locale['code'] . ' so using en.json');
$path = __DIR__ . '/config/locale/translations/en.json'; // if none translation exists, use default from `en.json`
}
}
@ -727,7 +845,6 @@ App::setResource('loggerBreadcrumbs', function () {
});
App::setResource('register', fn() => $register);
App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
App::setResource('localeCodes', function () {
@ -741,10 +858,15 @@ App::setResource('mails', fn() => new Mail());
App::setResource('deletes', fn() => new Delete());
App::setResource('database', fn() => new EventDatabase());
App::setResource('messaging', fn() => new Phone());
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
App::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);
App::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
App::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
App::setResource('clients', function ($request, $console, $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),
@ -916,34 +1038,65 @@ App::setResource('console', function () {
'legalAddress' => '',
'legalTaxId' => '',
'auths' => [
'invites' => App::getEnv('_APP_CONSOLE_INVITES', 'enabled') === 'enabled',
'limit' => (App::getEnv('_APP_CONSOLE_WHITELIST_ROOT', 'enabled') === 'enabled') ? 1 : 0, // limit signup to 1 user
'duration' => Auth::TOKEN_EXPIRATION_LOGIN_LONG, // 1 Year in seconds
],
'authWhitelistEmails' => (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_EMAILS', null)) : [],
'authWhitelistIPs' => (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_IPS', null)) : [],
'authProviders' => [
'githubEnabled' => true,
'githubSecret' => App::getEnv('_APP_CONSOLE_GITHUB_SECRET', ''),
'githubAppid' => App::getEnv('_APP_CONSOLE_GITHUB_APP_ID', '')
],
]);
}, []);
App::setResource('dbForProject', function ($db, $cache, Document $project) {
$cache = new Cache(new RedisCache($cache));
App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) {
if ($project->isEmpty() || $project->getId() === 'console') {
return $dbForConsole;
}
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace("_{$project->getInternalId()}");
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, $cache);
$database->setNamespace('_' . $project->getInternalId());
return $database;
}, ['db', 'cache', 'project']);
}, ['pools', 'dbForConsole', 'cache', 'project']);
App::setResource('dbForConsole', function ($db, $cache) {
$cache = new Cache(new RedisCache($cache));
App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace('_console');
$database = new Database($dbAdapter, $cache);
$database->setNamespace('console');
return $database;
}, ['db', 'cache']);
}, ['pools', 'cache']);
App::setResource('cache', function (Group $pools) {
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}, ['pools']);
App::setResource('deviceLocal', function () {
return new Local();
@ -963,45 +1116,40 @@ App::setResource('deviceBuilds', function ($project) {
function getDevice($root): Device
{
switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL))) {
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
$acl = 'private';
$device = Storage::DEVICE_LOCAL;
$accessKey = '';
$accessSecret = '';
$bucket = '';
$region = '';
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
$accessKey = $dsn->getUser();
$accessSecret = $dsn->getPassword();
$bucket = $dsn->getPath();
$region = $dsn->getParam('region');
} catch (\Exception $e) {
Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
}
switch ($device) {
case Storage::DEVICE_S3:
return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case STORAGE::DEVICE_DO_SPACES:
return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_BACKBLAZE:
return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LINODE:
return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_WASABI:
return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
case Storage::DEVICE_LOCAL:
default:
return new Local($root);
case Storage::DEVICE_S3:
$s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
$s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
$s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
$s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
$s3Acl = 'private';
return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
case Storage::DEVICE_DO_SPACES:
$doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
$doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
$doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
$doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
$doSpacesAcl = 'private';
return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
case Storage::DEVICE_BACKBLAZE:
$backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
$backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
$backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
$backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
$backblazeAcl = 'private';
return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
case Storage::DEVICE_LINODE:
$linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
$linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
$linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
$linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
$linodeAcl = 'private';
return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
case Storage::DEVICE_WASABI:
$wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
$wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
$wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
$wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
$wasabiAcl = 'private';
return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
}
}
@ -1143,6 +1291,24 @@ App::setResource('schema', function ($utopia, $dbForProject) {
);
}, ['utopia', 'dbForProject']);
App::setResource('contributors', function () {
$path = 'app/config/cloud/contributors.json';
$list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : [];
return $list;
}, []);
App::setResource('employees', function () {
$path = 'app/config/cloud/employees.json';
$list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : [];
return $list;
}, []);
App::setResource('heroes', function () {
$path = 'app/config/cloud/heroes.json';
$list = (file_exists($path)) ? json_decode(file_get_contents($path), true) : [];
return $list;
}, []);
App::setResource('requestTimestamp', function ($request) {
//TODO: Move this to the Request class itself
$timestampHeader = $request->getHeader('x-appwrite-timestamp');

3
app/preload.php
View file

@ -35,7 +35,8 @@ foreach (
realpath(__DIR__ . '/../vendor/symfony'),
realpath(__DIR__ . '/../vendor/mongodb'),
realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload
realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcache autoload
realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload
realpath(__DIR__ . '/../vendor/utopia-php/queue/src/Queue/Adapter/Workerman.php'), // TODO: remove workerman autoload
] as $key => $value
) {
if ($value !== false) {

198
app/realtime.php
View file

@ -16,16 +16,15 @@ use Utopia\CLI\Console;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;
use Utopia\Logger\Log;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Cache;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Appwrite\Utopia\Request;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\WebSocket\Server;
use Utopia\WebSocket\Adapter;
@ -33,6 +32,69 @@ require_once __DIR__ . '/init.php';
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
function getConsoleDB(): Database
{
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools');
$dbAdapter = $pools
->get('console')
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database->setNamespace('console');
return $database;
}
function getProjectDB(Document $project): Database
{
global $register;
/** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools');
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();
}
$dbAdapter = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$database = new Database($dbAdapter, getCache());
$database->setNamespace('_' . $project->getInternalId());
return $database;
}
function getCache(): Cache
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}
$realtime = new Realtime();
/**
@ -95,45 +157,6 @@ $logError = function (Throwable $error, string $action) use ($register) {
$server->error($logError);
function getDatabase(Registry &$register, string $namespace)
{
$attempts = 0;
do {
try {
$attempts++;
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$cache = new Cache(new RedisCache($redis));
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace($namespace);
if (!$database->exists($database->getDefaultDatabase(), 'realtime')) {
throw new Exception('Collection not ready');
}
break; // leave loop if successful
} catch (\Throwable $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep(DATABASE_RECONNECT_SLEEP);
}
} while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
return [
$database,
function () use ($register, $db, $redis) {
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
}
];
}
$server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) {
sleep(5); // wait for the initial database schema to be ready
Console::success('Server started successfully');
@ -143,7 +166,8 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
*/
go(function () use ($register, $containerId, &$statsDocument) {
$attempts = 0;
[$database, $returnDatabase] = getDatabase($register, '_console');
$database = getConsoleDB();
do {
try {
$attempts++;
@ -163,7 +187,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
sleep(DATABASE_RECONNECT_SLEEP);
}
} while (true);
call_user_func($returnDatabase);
$register->get('pools')->reclaim();
});
/**
@ -179,7 +203,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
}
try {
[$database, $returnDatabase] = getDatabase($register, '_console');
$database = getConsoleDB();
$statsDocument
->setAttribute('timestamp', DateTime::now())
@ -189,7 +213,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
} catch (\Throwable $th) {
call_user_func($logError, $th, "updateWorkerDocument");
} finally {
call_user_func($returnDatabase);
$register->get('pools')->reclaim();
}
});
});
@ -205,7 +229,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
* Sending current connections to project channels on the console project every 5 seconds.
*/
if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) {
[$database, $returnDatabase] = getDatabase($register, '_console');
$database = getConsoleDB();
$payload = [];
@ -250,7 +274,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
]));
}
call_user_func($returnDatabase);
$register->get('pools')->reclaim();
}
/**
* Sending test message for SDK E2E tests every 5 seconds.
@ -285,8 +309,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
}
$start = time();
/** @var Redis $redis */
$redis = $register->get('redisPool')->get();
$redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */
$redis->setOption(Redis::OPT_READ_TIMEOUT, -1);
if ($redis->ping(true)) {
@ -305,9 +328,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) {
$connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId]));
[$consoleDatabase, $returnConsoleDatabase] = getDatabase($register, '_console');
$project = Authorization::skip(fn () => $consoleDatabase->getDocument('projects', $projectId));
[$database, $returnDatabase] = getDatabase($register, "_{$project->getInternalId()}");
$consoleDatabase = getConsoleDB();
$project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId));
$database = getProjectDB($project);
$user = $database->getDocument('users', $userId);
@ -315,8 +338,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
$realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']);
call_user_func($returnDatabase);
call_user_func($returnConsoleDatabase);
$register->get('pools')->reclaim();
}
}
@ -344,10 +366,11 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
call_user_func($logError, $th, "pubSubConnection");
Console::error('Pub/sub error: ' . $th->getMessage());
$register->get('redisPool')->put($redis);
$attempts++;
sleep(DATABASE_RECONNECT_SLEEP);
continue;
} finally {
$register->get('pools')->reclaim();
}
}
@ -359,33 +382,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
$request = new Request($request);
$response = new Response(new SwooleResponse());
/** @var PDO $db */
$db = $register->get('dbPool')->get();
/** @var Redis $redis */
$redis = $register->get('redisPool')->get();
Console::info("Connection open (user: {$connection})");
App::setResource('db', fn () => $db);
App::setResource('cache', fn () => $redis);
App::setResource('request', fn () => $request);
App::setResource('response', fn () => $response);
App::setResource('pools', fn() => $register->get('pools'));
App::setResource('request', fn() => $request);
App::setResource('response', fn() => $response);
try {
/** @var \Utopia\Database\Document $user */
$user = $app->getResource('user');
/** @var \Utopia\Database\Document $project */
$project = $app->getResource('project');
/** @var \Utopia\Database\Document $console */
$console = $app->getResource('console');
$cache = new Cache(new RedisCache($redis));
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace("_{$project->getInternalId()}");
/*
* Project Check
*/
@ -393,12 +399,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
throw new Exception('Missing or unknown project ID', 1008);
}
$dbForProject = getProjectDB($project);
$console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */
$user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */
/*
* Abuse Check
*
* Abuse limits are connecting 128 times per minute and ip address.
*/
$timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $database);
$timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject);
$timeLimit
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getURI());
@ -469,16 +479,8 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
Console::error('[Error] Code: ' . $response['data']['code']);
Console::error('[Error] Message: ' . $response['data']['message']);
}
if ($th instanceof PDOException) {
$db = null;
}
} finally {
/**
* Put used PDO and Redis Connections back into their pools.
*/
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
$register->get('pools')->reclaim();
}
});
@ -486,16 +488,14 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
try {
$app = new App('UTC');
$response = new Response(new SwooleResponse());
$db = $register->get('dbPool')->get();
$redis = $register->get('redisPool')->get();
$cache = new Cache(new RedisCache($redis));
$database = new Database(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace("_console");
$projectId = $realtime->connections[$connection]['projectId'];
$project = $projectId === 'console' ? $app->getResource('console') : Authorization::skip(fn () => $database->getDocument('projects', $projectId));
$database->setNamespace("_{$project->getInternalId()}");
$database = getConsoleDB();
if ($projectId !== 'console') {
$project = Authorization::skip(fn() => $database->getDocument('projects', $projectId));
$database = getProjectDB($project);
}
/*
* Abuse Check
*
@ -561,7 +561,6 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
default:
throw new Exception('Message type is not valid.', 1003);
break;
}
} catch (\Throwable $th) {
$response = [
@ -578,8 +577,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
$server->close($connection, $th->getCode());
}
} finally {
$register->get('dbPool')->put($db);
$register->get('redisPool')->put($redis);
$register->get('pools')->reclaim();
}
});

24
app/tasks/ssl.php
View file

@ -1,24 +0,0 @@
<?php
global $cli;
use Appwrite\Event\Certificate;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Document;
use Utopia\Validator\Hostname;
$cli
->task('ssl')
->desc('Validate server certificates')
->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
->action(function ($domain) {
Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
(new Certificate())
->setDomain(new Document([
'domain' => $domain
]))
->setSkipRenewCheck(true)
->trigger();
});

138
app/tasks/usage.php
View file

@ -1,138 +0,0 @@
<?php
global $cli, $register;
use Appwrite\Usage\Calculators\TimeSeries;
use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
use Utopia\Cache\Adapter\Redis as RedisCache;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Adapter\MariaDB;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Utopia\Logger\Log;
use Utopia\Validator\WhiteList;
Authorization::disable();
Authorization::setDefaultStatus(false);
function getDatabase(Registry &$register, string $namespace): UtopiaDatabase
{
$attempts = 0;
do {
try {
$attempts++;
$db = $register->get('db');
$redis = $register->get('cache');
$cache = new Cache(new RedisCache($redis));
$database = new UtopiaDatabase(new MariaDB($db), $cache);
$database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$database->setNamespace($namespace);
if (!$database->exists($database->getDefaultDatabase(), 'projects')) {
throw new Exception('Projects collection not ready');
}
break; // leave loop if successful
} catch (\Exception $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep(DATABASE_RECONNECT_SLEEP);
}
} while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS);
return $database;
}
function getInfluxDB(Registry &$register): InfluxDatabase
{
/** @var InfluxDB\Client $client */
$client = $register->get('influxdb');
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}
$logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) {
$logger = $register->get('logger');
if ($logger) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$log = new Log();
$log->setNamespace("usage");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->addTag('code', $error->getCode());
$log->addTag('verboseType', get_class($error));
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->setAction($action);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Usage stats log pushed with status code: ' . $responseCode);
}
Console::warning("Failed: {$error->getMessage()}");
Console::warning($error->getTraceAsString());
};
$cli
->task('usage')
->desc('Schedules syncing data from influxdb to Appwrite console db')
->action(function () use ($register, $logError) {
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$database = getDatabase($register, '_console');
$influxDB = getInfluxDB($register);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $database, $influxDB, $logError);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
});

15
app/views/general/error.phtml
View file

@ -143,11 +143,18 @@ $title = $this->getParam('title', '')
</td>
</tr>
<?php endforeach; ?>
</table>
<?php endforeach; ?>
</table>
<?php endforeach; ?>
</div>
<?php endif; ?>
</div>
<?php endif; ?>
<style>
.with-borders {
border-top: 1px solid hsl(var(--color-neutral-150));
margin-block-start: 3rem;
padding-block-start: 2rem;
margin-inline: -2rem;
padding-inline: 2rem;
</section-->
<script type="text/javascript">

310
app/views/install/compose.phtml
View file

@ -15,7 +15,7 @@ $image = $this->getParam('image', '');
services:
traefik:
image: traefik:2.7
image: traefik:2.9
container_name: appwrite-traefik
<<: *x-logging
command:
@ -30,6 +30,15 @@ services:
ports:
- <?php echo $httpPort; ?>:80
- <?php echo $httpsPort; ?>:443
ulimits:
nofile:
soft: 655350
hard: 655350
sysctls:
- net.core.somaxconn=1024
- net.ipv4.tcp_rmem=1024 4096 16384
- net.ipv4.tcp_wmem=1024 4096 16384
- net.ipv4.ip_local_port_range=1025 65535
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-config:/storage/config:ro
@ -71,7 +80,6 @@ services:
- mariadb
- redis
# - clamav
- influxdb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
@ -79,6 +87,7 @@ services:
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_IPS
- _APP_CONSOLE_INVITES
- _APP_CONSOLE_ROOT_SESSION
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
@ -89,69 +98,45 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
- _APP_STORAGE_ANTIVIRUS_HOST
- _APP_STORAGE_ANTIVIRUS_PORT
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_CONNECTIONS_STORAGE
- _APP_FUNCTIONS_SIZE_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_FUNCTIONS_RUNTIMES
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_GRAPHQL_MAX_BATCH_SIZE
@ -189,13 +174,16 @@ services:
- _APP_WORKER_PER_CORE
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_MAX
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -214,15 +202,15 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -268,36 +256,16 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_STORAGE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_EXECUTOR_SECRET
@ -317,15 +285,15 @@ services:
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -345,15 +313,15 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -377,15 +345,15 @@ services:
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -400,83 +368,23 @@ services:
depends_on:
- redis
- mariadb
- appwrite-executor
- openruntimes-executor
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_FUNCTIONS_TIMEOUT
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-executor:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: executor
<<: *x-logging
container_name: appwrite-executor
restart: unless-stopped
stop_signal: SIGINT
networks:
appwrite:
runtimes:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-functions:/storage/functions:rw
- appwrite-builds:/storage/builds:rw
- /tmp:/tmp:rw
depends_on:
- redis
- mariadb
- appwrite
environment:
- _APP_ENV
- _APP_VERSION
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_RUNTIMES
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_FUNCTIONS_INACTIVE_THRESHOLD
- _APP_EXECUTOR_SECRET
- OPEN_RUNTIMES_NETWORK
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-worker-mails:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@ -541,15 +449,15 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -557,35 +465,6 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
appwrite-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: usage
container_name: appwrite-usage
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-schedule:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: schedule
@ -603,6 +482,33 @@ services:
- _APP_REDIS_USER
- _APP_REDIS_PASS
openruntimes-executor:
container_name: openruntimes-executor
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.1.4
networks:
- appwrite
- runtimes
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-builds:/storage/builds:rw
- appwrite-functions:/storage/functions:rw
- /tmp:/tmp:rw
environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME
- OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD
- OPR_EXECUTOR_ENV=$_APP_ENV
- OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES
- OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET
- OPR_EXECUTOR_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER
- OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
mariadb:
image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
container_name: appwrite-mariadb
@ -617,7 +523,7 @@ services:
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
- MYSQL_USER=${_APP_DB_USER}
- MYSQL_PASSWORD=${_APP_DB_PASS}
command: 'mysqld --innodb-flush-method=fsync'
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
redis:
image: redis:7.0.4-alpine
@ -643,31 +549,49 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
<<: *x-logging
restart: unless-stopped
container_name: appwrite-worker-usage
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
networks:
gateway:
name: gateway
appwrite:
name: appwrite
runtimes:
name: runtimes
volumes:
appwrite-mariadb:
@ -675,8 +599,6 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-config:
appwrite-functions:
appwrite-builds:
appwrite-influxdb:
appwrite-config:
appwrite-executor:

165
app/worker.php Normal file
View file

@ -0,0 +1,165 @@
<?php
require_once __DIR__ . '/init.php';
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Queue\Adapter\Swoole;
use Utopia\Queue\Message;
use Utopia\Queue\Server;
use Utopia\Registry\Registry;
use Utopia\Logger\Log;
use Utopia\Logger\Logger;
use Utopia\Pools\Group;
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
global $register;
Server::setResource('register', fn() => $register);
Server::setResource('dbForConsole', function (Cache $cache, Registry $register) {
$pools = $register->get('pools');
$database = $pools
->get('console')
->pop()
->getResource()
;
$adapter = new Database($database, $cache);
$adapter->setNamespace('console');
return $adapter;
}, ['cache', 'register']);
Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Database $dbForConsole) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
if ($project->isEmpty() || $project->getId() === 'console') {
return $dbForConsole;
}
$pools = $register->get('pools');
$database = $pools
->get($project->getAttribute('database'))
->pop()
->getResource()
;
$adapter = new Database($database, $cache);
$adapter->setNamespace('_' . $project->getInternalId());
return $adapter;
}, ['cache', 'register', 'message', 'dbForConsole']);
Server::setResource('cache', function (Registry $register) {
$pools = $register->get('pools');
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
}
return new Cache(new Sharding($adapters));
}, ['register']);
Server::setResource('queueForFunctions', function (Registry $register) {
$pools = $register->get('pools');
return new Func(
$pools
->get('queue')
->pop()
->getResource()
);
}, ['register']);
Server::setResource('queueForUsage', function (Registry $register) {
$pools = $register->get('pools');
return new Usage(
$pools
->get('queue')
->pop()
->getResource()
);
}, ['register']);
Server::setResource('log', fn() => new Log());
Server::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
Server::setResource('pools', function ($register) {
return $register->get('pools');
}, ['register']);
$pools = $register->get('pools');
$connection = $pools->get('queue')->pop()->getResource();
$workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
if (empty(App::getEnv('QUEUE'))) {
throw new Exception('Please configure "QUEUE" environemnt variable.');
}
$adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE'));
$server = new Server($adapter);
$server
->shutdown()
->inject('pools')
->action(function (Group $pools) {
$pools->reclaim();
});
$server
->error()
->inject('error')
->inject('logger')
->inject('log')
->action(function (Throwable $error, ?Logger $logger, Log $log) {
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
if ($error instanceof PDOException) {
throw $error;
}
if ($logger && ($error->getCode() >= 500 || $error->getCode() === 0)) {
$log->setNamespace("appwrite-worker");
$log->setServer(\gethostname());
$log->setVersion($version);
$log->setType(Log::TYPE_ERROR);
$log->setMessage($error->getMessage());
$log->setAction('appwrite-queue-' . App::getEnv('QUEUE'));
$log->addTag('verboseType', get_class($error));
$log->addTag('code', $error->getCode());
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles);
$isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
$responseCode = $logger->addLog($log);
Console::info('Usage stats log pushed with status code: ' . $responseCode);
}
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $error->getMessage());
Console::error('[Error] File: ' . $error->getFile());
Console::error('[Error] Line: ' . $error->getLine());
});

3
app/workers/audits.php
View file

@ -1,6 +1,5 @@
<?php
use Appwrite\Event\Event;
use Appwrite\Resque\Worker;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;
@ -37,7 +36,7 @@ class AuditsV1 extends Worker
$userName = $user->getAttribute('name', '');
$userEmail = $user->getAttribute('email', '');
$dbForProject = $this->getProjectDB($project->getId());
$dbForProject = $this->getProjectDB($project);
$audit = new Audit($dbForProject);
$audit->log(
userId: $user->getInternalId(),

115
app/workers/builds.php
View file

@ -1,21 +1,20 @@
<?php
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Resque\Worker;
use Appwrite\Utopia\Response\Model\Deployment;
use Cron\CronExpression;
use Executor\Executor;
use Appwrite\Usage\Stats;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Helpers\ID;
use Utopia\Storage\Storage;
use Utopia\DSN\DSN;
use Utopia\Database\Document;
use Utopia\Config\Config;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Storage\Storage;
require_once __DIR__ . '/../init.php';
@ -57,9 +56,16 @@ class BuildsV1 extends Worker
}
}
/**
* @throws \Utopia\Database\Exception\Authorization
* @throws \Utopia\Database\Exception\Structure
* @throws Throwable
*/
protected function buildDeployment(Document $project, Document $function, Document $deployment)
{
$dbForProject = $this->getProjectDB($project->getId());
global $register;
$dbForProject = $this->getProjectDB($project);
$function = $dbForProject->getDocument('functions', $function->getId());
if ($function->isEmpty()) {
@ -78,6 +84,15 @@ class BuildsV1 extends Worker
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */
$device = Storage::DEVICE_LOCAL;
try {
$dsn = new DSN($connection);
$device = $dsn->getScheme();
} catch (\Exception $e) {
Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
}
$buildId = $deployment->getAttribute('buildId', '');
$startTime = DateTime::now();
if (empty($buildId)) {
@ -86,18 +101,20 @@ class BuildsV1 extends Worker
'$id' => $buildId,
'$permissions' => [],
'startTime' => $startTime,
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'status' => 'processing',
'outputPath' => '',
'runtime' => $function->getAttribute('runtime'),
'source' => $deployment->getAttribute('path'),
'sourceType' => strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)),
'sourceType' => $device,
'stdout' => '',
'stderr' => '',
'endTime' => null,
'duration' => 0
]));
$deployment->setAttribute('buildId', $buildId);
$deployment->setAttribute('buildId', $build->getId());
$deployment->setAttribute('buildInternalId', $build->getInternalId());
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
} else {
$build = $dbForProject->getDocument('builds', $buildId);
@ -120,11 +137,16 @@ class BuildsV1 extends Worker
->trigger();
/** Trigger Functions */
$deploymentUpdate
->setClass(Event::FUNCTIONS_CLASS_NAME)
->setQueue(Event::FUNCTIONS_QUEUE_NAME)
$pools = $register->get('pools');
$connection = $pools->get('queue')->pop();
$functions = new Func($connection->getResource());
$functions
->from($deploymentUpdate)
->trigger();
$connection->reclaim();
/** Trigger Realtime */
$allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
'functionId' => $function->getId(),
@ -147,25 +169,22 @@ class BuildsV1 extends Worker
$source = $deployment->getAttribute('path');
$vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) {
$vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
$carry[$var->getAttribute('key')] = $var->getAttribute('value');
return $carry;
}, []);
$baseImage = $runtime['image'];
try {
$response = $this->executor->createRuntime(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
entrypoint: $deployment->getAttribute('entrypoint'),
projectId: $project->getId(),
source: $source,
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
vars: $vars,
runtime: $key,
baseImage: $baseImage,
workdir: '/usr/code',
image: $runtime['image'],
remove: true,
entrypoint: $deployment->getAttribute('entrypoint'),
workdir: '/usr/code',
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
variables: $vars,
commands: [
'sh', '-c',
'tar -zxf /tmp/code.tar.gz -C /usr/code && \
@ -173,13 +192,16 @@ class BuildsV1 extends Worker
]
);
$endTime = new \DateTime();
$endTime->setTimestamp($response['endTimeUnix']);
/** Update the build document */
$build->setAttribute('endTime', $response['endTime']);
$build->setAttribute('duration', $response['duration']);
$build->setAttribute('endTime', DateTime::format($endTime));
$build->setAttribute('duration', \intval($response['duration']));
$build->setAttribute('status', $response['status']);
$build->setAttribute('outputPath', $response['outputPath']);
$build->setAttribute('stderr', $response['stderr']);
$build->setAttribute('stdout', $response['response']);
$build->setAttribute('stdout', $response['stdout']);
/* Also update the deployment buildTime */
$deployment->setAttribute('buildTime', $response['duration']);
@ -188,16 +210,22 @@ class BuildsV1 extends Worker
/** Set auto deploy */
if ($deployment->getAttribute('activate') === true) {
$function->setAttribute('deploymentInternalId', $deployment->getInternalId());
$function->setAttribute('deployment', $deployment->getId());
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
}
/** Update function schedule */
$schedule = $function->getAttribute('schedule', '');
$cron = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? new CronExpression($schedule) : null;
$next = (empty($function->getAttribute('deployment')) && !empty($schedule)) ? DateTime::format($cron->getNextRunDate()) : null;
$function->setAttribute('scheduleNext', $next);
$function = $dbForProject->updateDocument('functions', $function->getId(), $function);
$dbForConsole = $this->getConsoleDB();
$schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
$schedule->setAttribute('resourceUpdatedAt', DateTime::now());
$schedule
->setAttribute('schedule', $function->getAttribute('schedule'))
->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
} catch (\Throwable $th) {
$endTime = DateTime::now();
$interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
@ -225,23 +253,20 @@ class BuildsV1 extends Worker
channels: $target['channels'],
roles: $target['roles']
);
/** Update usage stats */
global $register;
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1)
->setParam('buildStatus', $build->getAttribute('status', ''))
->setParam('buildTime', $build->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
}
/** Trigger usage queue */
$this
->getUsageQueue()
->setProject($project)
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->trigger()
;
}
public function shutdown(): void

1
app/workers/certificates.php
View file

@ -1,6 +1,5 @@
<?php
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Network\Validator\CNAME;
use Appwrite\Resque\Worker;

38
app/workers/databases.php
View file

@ -38,16 +38,16 @@ class DatabaseV1 extends Worker
switch (strval($type)) {
case DATABASE_TYPE_CREATE_ATTRIBUTE:
$this->createAttribute($database, $collection, $document, $project->getId());
$this->createAttribute($database, $collection, $document, $project);
break;
case DATABASE_TYPE_DELETE_ATTRIBUTE:
$this->deleteAttribute($database, $collection, $document, $project->getId());
$this->deleteAttribute($database, $collection, $document, $project);
break;
case DATABASE_TYPE_CREATE_INDEX:
$this->createIndex($database, $collection, $document, $project->getId());
$this->createIndex($database, $collection, $document, $project);
break;
case DATABASE_TYPE_DELETE_INDEX:
$this->deleteIndex($database, $collection, $document, $project->getId());
$this->deleteIndex($database, $collection, $document, $project);
break;
default:
@ -64,12 +64,13 @@ class DatabaseV1 extends Worker
* @param Document $database
* @param Document $collection
* @param Document $attribute
* @param string $projectId
* @param Document $project
*/
protected function createAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void
protected function createAttribute(Document $database, Document $collection, Document $attribute, Document $project): void
{
$projectId = $project->getId();
$dbForConsole = $this->getConsoleDB();
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [
'databaseId' => $database->getId(),
@ -185,13 +186,14 @@ class DatabaseV1 extends Worker
* @param Document $database
* @param Document $collection
* @param Document $attribute
* @param string $projectId
* @param Document $project
* @throws Throwable
*/
protected function deleteAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void
protected function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project): void
{
$projectId = $project->getId();
$dbForConsole = $this->getConsoleDB();
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [
'databaseId' => $database->getId(),
@ -324,7 +326,7 @@ class DatabaseV1 extends Worker
}
if ($exists) { // Delete the duplicate if created, else update in db
$this->deleteIndex($database, $collection, $index, $projectId);
$this->deleteIndex($database, $collection, $index, $project);
} else {
$dbForProject->updateDocument('indexes', $index->getId(), $index);
}
@ -345,13 +347,14 @@ class DatabaseV1 extends Worker
* @param Document $database
* @param Document $collection
* @param Document $index
* @param string $projectId
* @param Document $project
* @throws \Exception
*/
protected function createIndex(Document $database, Document $collection, Document $index, string $projectId): void
protected function createIndex(Document $database, Document $collection, Document $index, Document $project): void
{
$projectId = $project->getId();
$dbForConsole = $this->getConsoleDB();
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [
'databaseId' => $database->getId(),
@ -411,12 +414,13 @@ class DatabaseV1 extends Worker
* @param Document $database
* @param Document $collection
* @param Document $index
* @param string $projectId
* @param Document $project
*/
protected function deleteIndex(Document $database, Document $collection, Document $index, string $projectId): void
protected function deleteIndex(Document $database, Document $collection, Document $index, Document $project): void
{
$projectId = $project->getId();
$dbForConsole = $this->getConsoleDB();
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [
'databaseId' => $database->getId(),

359
app/workers/deletes.php
View file

@ -8,7 +8,6 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Appwrite\Resque\Worker;
use Executor\Executor;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\CLI\Console;
@ -41,32 +40,35 @@ class DeletesV1 extends Worker
switch ($document->getCollection()) {
case DELETE_TYPE_DATABASES:
$this->deleteDatabase($document, $project->getId());
$this->deleteDatabase($document, $project);
break;
case DELETE_TYPE_COLLECTIONS:
$this->deleteCollection($document, $project);
break;
case DELETE_TYPE_PROJECTS:
$this->deleteProject($document);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($document, $project->getId());
$this->deleteFunction($document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($document, $project->getId());
$this->deleteDeployment($document, $project);
break;
case DELETE_TYPE_USERS:
$this->deleteUser($document, $project->getId());
$this->deleteUser($document, $project);
break;
case DELETE_TYPE_TEAMS:
$this->deleteMemberships($document, $project->getId());
$this->deleteMemberships($document, $project);
if ($project->getId() === 'console') {
$this->deleteProjectsByTeam($document);
}
break;
case DELETE_TYPE_BUCKETS:
$this->deleteBucket($document, $project->getId());
$this->deleteBucket($document, $project);
break;
default:
if (\str_starts_with($document->getCollection(), 'database_')) {
$this->deleteCollection($document, $project->getId());
$this->deleteCollection($document, $project);
break;
}
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@ -87,7 +89,7 @@ class DeletesV1 extends Worker
$document = new Document($this->args['document'] ?? []);
if (!$document->isEmpty()) {
$this->deleteAuditLogsByResource('document/' . $document->getId(), $project->getId());
$this->deleteAuditLogsByResource('document/' . $document->getId(), $project);
}
break;
@ -114,10 +116,13 @@ class DeletesV1 extends Worker
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project->getId());
$this->deleteCacheByResource($project, $this->args['resource']);
break;
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
$this->deleteCacheByDate();
$this->deleteCacheByDate($this->args['datetime']);
break;
case DELETE_TYPE_SCHEDULES:
$this->deleteSchedules($this->args['datetime']);
break;
default:
Console::error('No delete operation for type: ' . $type);
@ -130,31 +135,87 @@ class DeletesV1 extends Worker
}
/**
* @param string $projectId
* @throws Exception
*/
protected function deleteCacheByResource(string $projectId): void
protected function deleteSchedules(string $datetime): void
{
$this->deleteCacheFiles([
Query::equal('resource', [$this->args['resource']]),
]);
$this->listByGroup(
'schedules',
[
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::lessThanEqual('resourceUpdatedAt', $datetime),
Query::equal('active', [false]),
],
$this->getConsoleDB(),
function (Document $document) {
$project = $this->getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
if ($project->isEmpty()) {
$this->getConsoleDB()->deleteDocument('schedules', $document->getId());
Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId'));
return;
}
$function = $this->getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId'));
if ($function->isEmpty()) {
$this->getConsoleDB()->deleteDocument('schedules', $document->getId());
Console::success('Deleted schedule for function ' . $document->getAttribute('resourceId'));
}
}
);
}
protected function deleteCacheByDate(): void
/**
* @param Document $project
* @param string $resource
* @throws Exception
*/
protected function deleteCacheByResource(Document $project, string $resource): void
{
$this->deleteCacheFiles([
Query::lessThan('accessedAt', $this->args['datetime']),
]);
}
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$document = $dbForProject->findOne('cache', [Query::equal('resource', [$resource])]);
protected function deleteCacheFiles($query): void
{
$this->deleteForProjectIds(function (string $projectId) use ($query) {
$dbForProject = $this->getProjectDB($projectId);
if ($document) {
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$this->deleteById(
$document,
$dbForProject,
function ($document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
if ($cache->purge($document->getId())) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('Failed to delete cache file: ' . $path);
}
}
);
}
}
/**
* @param string $datetime
* @throws Exception
*/
protected function deleteCacheByDate(string $datetime): void
{
$this->deleteForProjectIds(function (Document $project) use ($datetime) {
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
$query = [
Query::lessThan('accessedAt', $datetime),
];
$this->deleteByGroup(
'cache',
$query,
@ -172,38 +233,39 @@ class DeletesV1 extends Worker
});
}
/**
* @param Document $document database document
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteDatabase(Document $document, string $projectId): void
protected function deleteDatabase(Document $document, Document $project): void
{
$databaseId = $document->getId();
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($projectId) {
$this->deleteCollection($document, $projectId);
$this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) {
$this->deleteCollection($document, $project);
});
$dbForProject->deleteCollection('database_' . $document->getInternalId());
$this->deleteAuditLogsByResource('database/' . $databaseId, $projectId);
$this->deleteAuditLogsByResource('database/' . $databaseId, $project);
}
/**
* @param Document $document teams document
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteCollection(Document $document, string $projectId): void
protected function deleteCollection(Document $document, Document $project): void
{
$collectionId = $document->getId();
$collectionInternalId = $document->getInternalId();
$databaseId = $document->getAttribute('databaseId');
$databaseInternalId = $document->getAttribute('databaseInternalId');
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$relationships = \array_filter(
$document->getAttribute('attributes'),
@ -223,16 +285,16 @@ class DeletesV1 extends Worker
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
$this->deleteByGroup('attributes', [
Query::equal('databaseId', [$databaseId]),
Query::equal('collectionId', [$collectionId])
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteByGroup('indexes', [
Query::equal('databaseId', [$databaseId]),
Query::equal('collectionId', [$collectionId])
Query::equal('databaseInternalId', [$databaseInternalId]),
Query::equal('collectionInternalId', [$collectionInternalId])
], $dbForProject);
$this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $projectId);
$this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project);
}
/**
@ -241,8 +303,9 @@ class DeletesV1 extends Worker
*/
protected function deleteUsageStats(string $hourlyUsageRetentionDatetime)
{
$this->deleteForProjectIds(function (string $projectId) use ($hourlyUsageRetentionDatetime) {
$dbForProject = $this->getProjectDB($projectId);
$this->deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) {
$dbForProject = $this->getProjectDB($project);
// Delete Usage stats
$this->deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
@ -252,17 +315,25 @@ class DeletesV1 extends Worker
/**
* @param Document $document teams document
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteMemberships(Document $document, string $projectId): void
protected function deleteMemberships(Document $document, Document $project): void
{
$teamId = $document->getAttribute('teamId', '');
$dbForProject = $this->getProjectDB($project);
$teamInternalId = $document->getInternalId();
// Delete Memberships
$this->deleteByGroup('memberships', [
Query::equal('teamId', [$teamId])
], $this->getProjectDB($projectId));
$this->deleteByGroup(
'memberships',
[
Query::equal('teamInternalId', [$teamInternalId])
],
$dbForProject,
function (Document $membership) use ($dbForProject) {
$userId = $membership->getAttribute('userId');
$dbForProject->deleteCachedDocument('users', $userId);
}
);
}
/**
@ -305,7 +376,7 @@ class DeletesV1 extends Worker
}
// Delete project tables
$dbForProject = $this->getProjectDB($projectId, $document);
$dbForProject = $this->getProjectDB($document);
while (true) {
$collections = $dbForProject->listCollections();
@ -361,45 +432,44 @@ class DeletesV1 extends Worker
/**
* @param Document $document user document
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteUser(Document $document, string $projectId): void
protected function deleteUser(Document $document, Document $project): void
{
$userId = $document->getId();
$userInternalId = $document->getInternalId();
$dbForProject = $this->getProjectDB($project);
// Delete all sessions of this user from the sessions table and update the sessions field of the user record
$this->deleteByGroup('sessions', [
Query::equal('userId', [$userId])
], $this->getProjectDB($projectId));
Query::equal('userInternalId', [$userInternalId])
], $dbForProject);
$this->getProjectDB($projectId)->deleteCachedDocument('users', $userId);
$dbForProject->deleteCachedDocument('users', $userId);
// Delete Memberships and decrement team membership counts
$this->deleteByGroup('memberships', [
Query::equal('userId', [$userId])
], $this->getProjectDB($projectId), function (Document $document) use ($projectId) {
Query::equal('userInternalId', [$userInternalId])
], $dbForProject, function (Document $document) use ($dbForProject) {
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
$team = $this->getProjectDB($projectId)->getDocument('teams', $teamId);
$team = $dbForProject->getDocument('teams', $teamId);
if (!$team->isEmpty()) {
$team = $this
->getProjectDB($projectId)
->updateDocument(
'teams',
$teamId,
// Ensure that total >= 0
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
);
$team = $dbForProject->updateDocument(
'teams',
$teamId,
// Ensure that total >= 0
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
);
}
}
});
// Delete tokens
$this->deleteByGroup('tokens', [
Query::equal('userId', [$userId])
], $this->getProjectDB($projectId));
Query::equal('userInternalId', [$userInternalId])
], $dbForProject);
}
/**
@ -408,8 +478,8 @@ class DeletesV1 extends Worker
*/
protected function deleteExecutionLogs(string $datetime): void
{
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
$dbForProject = $this->getProjectDB($projectId);
$this->deleteForProjectIds(function (Document $project) use ($datetime) {
$dbForProject = $this->getProjectDB($project);
// Delete Executions
$this->deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
@ -421,10 +491,10 @@ class DeletesV1 extends Worker
{
$consoleDB = $this->getConsoleDB();
$this->deleteForProjectIds(function (string $projectId) use ($consoleDB) {
$dbForProject = $this->getProjectDB($projectId);
$this->deleteForProjectIds(function (Document $project) use ($consoleDB) {
$dbForProject = $this->getProjectDB($project);
$project = $consoleDB->getDocument('projects', $projectId);
$project = $consoleDB->getDocument('projects', $project->getId());
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
@ -441,8 +511,8 @@ class DeletesV1 extends Worker
*/
protected function deleteRealtimeUsage(string $datetime): void
{
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
$dbForProject = $this->getProjectDB($projectId);
$this->deleteForProjectIds(function (Document $project) use ($datetime) {
$dbForProject = $this->getProjectDB($project);
// Delete Dead Realtime Logs
$this->deleteByGroup('realtime', [
Query::lessThan('timestamp', $datetime)
@ -460,8 +530,9 @@ class DeletesV1 extends Worker
throw new Exception('Failed to delete audit logs. No datetime provided');
}
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
$dbForProject = $this->getProjectDB($projectId);
$this->deleteForProjectIds(function (Document $project) use ($datetime) {
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($datetime);
@ -481,8 +552,9 @@ class DeletesV1 extends Worker
throw new Exception('Failed to delete audit logs. No datetime provided');
}
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
$dbForProject = $this->getProjectDB($projectId);
$this->deleteForProjectIds(function (Document $project) use ($datetime) {
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($datetime);
if (!$status) {
@ -493,12 +565,11 @@ class DeletesV1 extends Worker
/**
* @param string $resource
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteAuditLogsByResource(string $resource, string $projectId): void
protected function deleteAuditLogsByResource(string $resource, Document $project): void
{
$dbForProject = $this->getProjectDB($projectId);
$dbForProject = $this->getProjectDB($project);
$this->deleteByGroup(Audit::COLLECTION, [
Query::equal('resource', [$resource])
@ -507,20 +578,21 @@ class DeletesV1 extends Worker
/**
* @param Document $document function document
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteFunction(Document $document, string $projectId): void
protected function deleteFunction(Document $document, Document $project): void
{
$dbForProject = $this->getProjectDB($projectId);
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$functionId = $document->getId();
$functionInternalId = $document->getInternalId();
/**
* Delete Variables
*/
Console::info("Deleting variables for function " . $functionId);
$this->deleteByGroup('variables', [
Query::equal('functionId', [$functionId])
Query::equal('functionInternalId', [$functionInternalId])
], $dbForProject);
/**
@ -565,28 +637,17 @@ class DeletesV1 extends Worker
Query::equal('functionId', [$functionId])
], $dbForProject);
/**
* Request executor to delete all deployment containers
*/
Console::info("Requesting executor to delete all deployment containers for function " . $functionId);
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
foreach ($deploymentIds as $deploymentId) {
try {
$executor->deleteRuntime($projectId, $deploymentId);
} catch (Throwable $th) {
Console::error($th->getMessage());
}
}
// TODO: Request executor to delete runtime
}
/**
* @param Document $document deployment document
* @param string $projectId
* @throws Exception
* @param Document $project
*/
protected function deleteDeployment(Document $document, string $projectId): void
protected function deleteDeployment(Document $document, Document $project): void
{
$dbForProject = $this->getProjectDB($projectId);
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$deploymentId = $document->getId();
$functionId = $document->getAttribute('resourceId');
@ -616,16 +677,7 @@ class DeletesV1 extends Worker
}
});
/**
* Request executor to delete the deployment container
*/
Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId);
try {
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
$executor->deleteRuntime($projectId, $deploymentId);
} catch (Throwable $th) {
Console::error($th->getMessage());
}
// TODO: Request executor to delete runtime
}
@ -659,6 +711,7 @@ class DeletesV1 extends Worker
*/
protected function deleteForProjectIds(callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
@ -673,13 +726,11 @@ class DeletesV1 extends Worker
$chunk++;
/** @var string[] $projectIds */
$projectIds = array_map(fn (Document $project) => $project->getId(), $projects);
$sum = count($projects);
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
foreach ($projectIds as $projectId) {
$callback($projectId);
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
@ -705,24 +756,75 @@ class DeletesV1 extends Worker
$executionStart = \microtime(true);
try {
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
$sum = count($results);
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents in collection ' . $database->getNamespace() . '_' . $collection);
foreach ($results as $document) {
$this->deleteById($document, $database, $callback);
$count++;
}
}
} catch (\Exception $e) {
Console::error($e->getMessage());
}
$executionEnd = \microtime(true);
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param string $collection collectionID
* @param Query[] $queries
* @param Database $database
* @param callable $callback
*/
protected function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$cursor = null;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
$mergedQueries = \array_merge([Query::limit($limit)], $queries);
if ($cursor instanceof Document) {
$mergedQueries[] = Query::cursorAfter($cursor);
}
$results = $database->find($collection, $mergedQueries);
$sum = count($results);
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
if ($sum > 0) {
$cursor = $results[$sum - 1];
}
foreach ($results as $document) {
$this->deleteById($document, $database, $callback);
if (is_callable($callback)) {
$callback($document);
}
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
/**
@ -773,9 +875,10 @@ class DeletesV1 extends Worker
}
}
protected function deleteBucket(Document $document, string $projectId)
protected function deleteBucket(Document $document, Document $project)
{
$dbForProject = $this->getProjectDB($projectId);
$projectId = $project->getId();
$dbForProject = $this->getProjectDB($project);
$dbForProject->deleteCollection('bucket_' . $document->getInternalId());
$device = $this->getFilesDevice($projectId);

409
app/workers/functions.php
View file

@ -1,243 +1,81 @@
<?php
require_once __DIR__ . '/../worker.php';
use Appwrite\Event\Usage;
use Utopia\Queue\Message;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Resque\Worker;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response\Model\Execution;
use Cron\CronExpression;
use Executor\Executor;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Logger\Log;
use Utopia\Queue\Server;
use Utopia\Database\Helpers\Role;
require_once __DIR__ . '/../init.php';
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('Functions V1 Worker');
Console::success(APP_NAME . ' functions worker v1 has started');
class FunctionsV1 extends Worker
{
private ?Executor $executor = null;
public array $args = [];
public array $allowed = [];
public function getName(): string
{
return "functions";
}
public function init(): void
{
$this->executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
}
public function run(): void
{
$type = $this->args['type'] ?? '';
$events = $this->args['events'] ?? [];
$project = new Document($this->args['project'] ?? []);
$user = new Document($this->args['user'] ?? []);
$payload = json_encode($this->args['payload'] ?? []);
if ($project->getId() === 'console') {
return;
}
$database = $this->getProjectDB($project->getId());
/**
* Handle Event execution.
*/
if (!empty($events)) {
$limit = 30;
$sum = 30;
$offset = 0;
$functions = [];
/** @var Document[] $functions */
while ($sum >= $limit) {
$functions = $database->find('functions', [
Query::limit($limit),
Query::offset($offset),
Query::orderAsc('name'),
]);
$sum = \count($functions);
$offset = $offset + $limit;
Console::log('Fetched ' . $sum . ' functions...');
foreach ($functions as $function) {
if (!array_intersect($events, $function->getAttribute('events', []))) {
continue;
}
Console::success('Iterating function: ' . $function->getAttribute('name'));
try {
$this->execute(
project: $project,
function: $function,
dbForProject: $database,
trigger: 'event',
// Pass first, most verbose event pattern
event: $events[0],
eventData: $payload,
user: $user
);
Console::success('Triggered function: ' . $events[0]);
} catch (\Throwable $th) {
Console::error("Failed to execute " . $function->getId() . " with error: " . $th->getMessage());
}
}
}
return;
}
/**
* Handle Schedule and HTTP execution.
*/
$user = new Document($this->args['user'] ?? []);
$project = new Document($this->args['project'] ?? []);
$execution = new Document($this->args['execution'] ?? []);
$function = new Document($this->args['function'] ?? []);
switch ($type) {
case 'http':
$jwt = $this->args['jwt'] ?? '';
$data = $this->args['data'] ?? '';
$function = $database->getDocument('functions', $execution->getAttribute('functionId'));
$this->execute(
project: $project,
function: $function,
dbForProject: $database,
executionId: $execution->getId(),
trigger: 'http',
data: $data,
user: $user,
jwt: $jwt
);
break;
case 'schedule':
$functionOriginal = $function;
/*
* 1. Get Original Task
* 2. Check for updates
* If has updates skip task and don't reschedule
* If status not equal to play skip task
* 3. Check next run date, update task and add new job at the given date
* 4. Execute task (set optional timeout)
* 5. Update task response to log
* On success reset error count
* On failure add error count
* If error count bigger than allowed change status to pause
*/
// Reschedule
$function = $database->getDocument('functions', $function->getId());
if (empty($function->getId())) {
throw new Exception('Function not found (' . $function->getId() . ')');
}
if ($functionOriginal->getAttribute('schedule') !== $function->getAttribute('schedule')) { // Schedule has changed from previous run, ignore this run.
return;
}
if ($functionOriginal->getAttribute('scheduleUpdatedAt') !== $function->getAttribute('scheduleUpdatedAt')) { // Double execution due to rapid cron changes, ignore this run.
return;
}
$cron = new CronExpression($function->getAttribute('schedule'));
$next = DateTime::format($cron->getNextRunDate());
$function = $function
->setAttribute('scheduleNext', $next)
->setAttribute('schedulePrevious', DateTime::now());
$function = $database->updateDocument(
'functions',
$function->getId(),
$function
);
$reschedule = new Func();
$reschedule
->setFunction($function)
->setType('schedule')
->setUser($user)
->setProject($project)
->schedule(new \DateTime($next));
;
$this->execute(
project: $project,
function: $function,
dbForProject: $database,
trigger: 'schedule'
);
break;
}
}
private function execute(
Server::setResource('execute', function () {
return function (
Log $log,
Func $queueForFunctions,
Database $dbForProject,
Usage $queueForUsage,
Document $project,
Document $function,
Database $dbForProject,
string $trigger,
string $executionId = null,
string $event = null,
string $eventData = null,
string $data = null,
?Document $user = null,
string $jwt = null
string $jwt = null,
string $event = null,
string $eventData = null,
string $executionId = null,
) {
$user ??= new Document();
$functionId = $function->getId();
$functionInternalId = $function->getInternalId();
$deploymentId = $function->getAttribute('deployment', '');
$log->addTag('functionId', $functionId);
$log->addTag('projectId', $project->getId());
/** Check if deployment exists */
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$deploymentInternalId = $deployment->getInternalId();
if ($deployment->getAttribute('resourceId') !== $functionId) {
throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404);
throw new Exception('Deployment not found. Create deployment before trying to execute a function');
}
if ($deployment->isEmpty()) {
throw new Exception('Deployment not found. Create deployment before trying to execute a function', 404);
throw new Exception('Deployment not found. Create deployment before trying to execute a function');
}
/** Check if build has exists */
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''));
if ($build->isEmpty()) {
throw new Exception('Build not found', 404);
throw new Exception('Build not found');
}
if ($build->getAttribute('status') !== 'ready') {
throw new Exception('Build not ready', 400);
throw new Exception('Build not ready');
}
/** Check if runtime is supported */
$runtimes = Config::getParam('runtimes', []);
if (!\array_key_exists($function->getAttribute('runtime'), $runtimes)) {
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported', 400);
throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
$runtime = $runtimes[$function->getAttribute('runtime')];
@ -250,6 +88,8 @@ class FunctionsV1 extends Worker
'$id' => $executionId,
'$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))],
'functionId' => $functionId,
'functionInternalId' => $functionInternalId,
'deploymentInternalId' => $deploymentInternalId,
'deploymentId' => $deploymentId,
'trigger' => $trigger,
'status' => 'waiting',
@ -257,17 +97,28 @@ class FunctionsV1 extends Worker
'response' => '',
'stderr' => '',
'duration' => 0.0,
'search' => implode(' ', [$functionId, $executionId]),
'search' => implode(' ', [$function->getId(), $executionId]),
]));
// TODO: @Meldiron Trigger executions.create event here
if ($execution->isEmpty()) {
throw new Exception('Failed to create or read execution');
}
/**
* Usage
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function
}
$execution->setAttribute('status', 'processing');
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
$vars = array_reduce($function['vars'] ?? [], function (array $carry, Document $var) {
$vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
$carry[$var->getAttribute('key')] = $var->getAttribute('value');
return $carry;
}, []);
@ -290,16 +141,16 @@ class FunctionsV1 extends Worker
/** Execute function */
try {
$executionResponse = $this->executor->createExecution(
$client = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
$executionResponse = $client->createExecution(
projectId: $project->getId(),
deploymentId: $deploymentId,
path: $build->getAttribute('outputPath', ''),
vars: $vars,
entrypoint: $deployment->getAttribute('entrypoint', ''),
data: $vars['APPWRITE_FUNCTION_DATA'] ?? '',
runtime: $function->getAttribute('runtime', ''),
payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '',
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
baseImage: $runtime['image']
image: $runtime['image'],
source: $build->getAttribute('outputPath', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
);
/** Update execution status */
@ -317,7 +168,9 @@ class FunctionsV1 extends Worker
->setAttribute('status', 'failed')
->setAttribute('statusCode', $th->getCode())
->setAttribute('stderr', $th->getMessage());
Console::error($th->getMessage());
$error = $th->getMessage();
$errorCode = $th->getCode();
}
$execution = $dbForProject->updateDocument('executions', $executionId, $execution);
@ -335,9 +188,8 @@ class FunctionsV1 extends Worker
->trigger();
/** Trigger Functions */
$executionUpdate
->setClass(Event::FUNCTIONS_CLASS_NAME)
->setQueue(Event::FUNCTIONS_QUEUE_NAME)
$queueForFunctions
->from($executionUpdate)
->trigger();
/** Trigger realtime event */
@ -365,25 +217,132 @@ class FunctionsV1 extends Worker
roles: $target['roles']
);
/** Update usage stats */
global $register;
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
}
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
};
});
public function shutdown(): void
{
}
}
$server->job()
->inject('message')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('execute')
->inject('log')
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, callable $execute, Log $log) {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
$type = $payload['type'] ?? '';
$events = $payload['events'] ?? [];
$data = $payload['data'] ?? '';
$eventData = $payload['payload'] ?? '';
$project = new Document($payload['project'] ?? []);
$function = new Document($payload['function'] ?? []);
$user = new Document($payload['user'] ?? []);
if ($project->getId() === 'console') {
return;
}
if (!empty($events)) {
$limit = 30;
$sum = 30;
$offset = 0;
$functions = [];
/** @var Document[] $functions */
while ($sum >= $limit) {
$functions = $dbForProject->find('functions', [
Query::limit($limit),
Query::offset($offset)
]);
$sum = \count($functions);
$offset = $offset + $limit;
Console::log('Fetched ' . $sum . ' functions...');
foreach ($functions as $function) {
if (!array_intersect($events, $function->getAttribute('events', []))) {
continue;
}
Console::success('Iterating function: ' . $function->getAttribute('name'));
try {
$execute(
log: $log,
queueForUsage: $queueForUsage,
dbForProject: $dbForProject,
project: $project,
function: $function,
queueForFunctions: $queueForFunctions,
trigger: 'event',
event: $events[0],
eventData: \is_string($eventData) ? $eventData : \json_encode($eventData),
user: $user,
data: null,
executionId: null,
jwt: null
);
Console::success('Triggered function: ' . $events[0]);
} catch (\Throwable $th) {
Console::error("Failed to execute " . $function->getId() . " with error: " . $th->getMessage());
}
}
}
return;
}
/**
* Handle Schedule and HTTP execution.
*/
switch ($type) {
case 'http':
$jwt = $payload['jwt'] ?? '';
$execution = new Document($payload['execution'] ?? []);
$user = new Document($payload['user'] ?? []);
$execute(
log: $log,
project: $project,
function: $function,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
trigger: 'http',
executionId: $execution->getId(),
event: null,
eventData: null,
data: $data,
user: $user,
jwt: $jwt,
queueForUsage: $queueForUsage,
);
break;
case 'schedule':
$execute(
log: $log,
project: $project,
function: $function,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
trigger: 'schedule',
executionId: null,
event: null,
eventData: null,
data: null,
user: null,
jwt: null,
queueForUsage: $queueForUsage,
);
break;
}
});
$server->workerStart();
$server->start();

2
app/workers/messaging.php
View file

@ -1,9 +1,9 @@
<?php
use Appwrite\DSN\DSN;
use Appwrite\Resque\Worker;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\DSN\DSN;
use Utopia\Messaging\Adapter;
use Utopia\Messaging\Adapters\SMS\Mock;
use Utopia\Messaging\Adapters\SMS\Msg91;

288
app/workers/usage.php Normal file
View file

@ -0,0 +1,288 @@
<?php
require_once __DIR__ . '/../worker.php';
use Swoole\Timer;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Message;
use Utopia\CLI\Console;
use Utopia\Queue\Server;
use Utopia\Registry\Registry;
Authorization::disable();
Authorization::setDefaultStatus(false);
$stats = [];
$periods['1h'] = 'Y-m-d H:00';
$periods['1d'] = 'Y-m-d 00:00';
//$periods['1m'] = 'Y-m-1 00:00';
$periods['inf'] = '0000-00-00 00:00';
const INFINITY_PERIOD = '_inf_';
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
*/
Server::setResource('reduce', function (Cache $cache, Registry $register, $pools) {
return function ($database, $projectInternalId, Document $document, array &$metrics) use ($pools, $cache, $register): void {
try {
$dbForProject = new Database(
$pools
->get($database)
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Exception $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
} finally {
$pools->reclaim();
}
};
}, ['cache', 'register', 'pools']);
$server->job()
->inject('message')
->inject('reduce')
->action(function (Message $message, callable $reduce) use (&$stats) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$reduce(
database: $project->getAttribute('database'),
projectInternalId: $project->getInternalId(),
document: new Document($document),
metrics: $payload['metrics'],
);
}
$stats[$projectId]['database'] = $project->getAttribute('database');
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset($stats[$projectId]['keys'][$metric['key']])) {
$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
});
$server
->workerStart()
->inject('register')
->inject('cache')
->inject('pools')
->action(function ($register, $cache, $pools) use ($periods, &$stats) {
Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
$offset = count($stats);
$projects = array_slice($stats, 0, $offset, true);
array_splice($stats, 0, $offset);
foreach ($projects as $projectInternalId => $project) {
try {
$dbForProject = new Database(
$pools
->get($project['database'])
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
foreach ($project['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
if (!empty($project['keys'])) {
$dbForProject->createDocument('statsLogger', new Document([
'time' => DateTime::now(),
'metrics' => $project['keys'],
]));
}
} catch (\Exception $e) {
$now = DateTime::now();
console::error("[Error] " . " Time: {$now} " . " projectInternalId: {$projectInternalId}" . " File: {$e->getFile()}" . " Line: {$e->getLine()} " . " message: {$e->getMessage()}");
} finally {
$pools->reclaim();
}
}
});
});
$server->start();

3
bin/calc-tier-stats Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-tier-stats $@

3
bin/calc-users-stats Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-users-stats $@

3
bin/clear-card-cache Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php clear-card-cache $@

3
bin/executor
View file

@ -1,3 +0,0 @@
#!/bin/sh
php -e /usr/src/code/app/executor.php -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php

3
bin/hamster Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php hamster $@

3
bin/patch-delete-project-collections Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-project-collections $@

3
bin/patch-delete-schedule-updated-at-attribute Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@

9
bin/schedule
View file

@ -1,10 +1,3 @@
#!/bin/sh
if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ]
then
REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
else
REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
fi
INTERVAL=1 REDIS_BACKEND=$REDIS_BACKEND RESQUE_PHP='/usr/src/code/vendor/autoload.php' php /usr/src/code/vendor/bin/resque-scheduler
php /usr/src/code/app/cli.php schedule $@

3
bin/usage
View file

@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php usage $@

3
bin/volume-sync Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php volume-sync $@

9
bin/worker-functions
View file

@ -1,10 +1,3 @@
#!/bin/sh
if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ]
then
REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
else
REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}"
fi
INTERVAL=0.1 QUEUE='v1-functions' APP_INCLUDE='/usr/src/code/app/workers/functions.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php
QUEUE=v1-functions php /usr/src/code/app/workers/functions.php $@

3
bin/worker-usage Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
QUEUE=v1-usage php /usr/src/code/app/workers/usage.php $@

14
composer.json
View file

@ -44,20 +44,24 @@
"appwrite/php-clamav": "2.0.*",
"appwrite/php-runtimes": "0.11.*",
"utopia-php/abuse": "0.27.*",
"utopia-php/analytics": "0.2.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.29.*",
"utopia-php/cache": "0.8.*",
"utopia-php/cli": "0.13.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.38.*",
"utopia-php/domains": "1.1.*",
"utopia-php/dsn": "0.1.*",
"utopia-php/framework": "0.28.*",
"utopia-php/image": "0.5.*",
"utopia-php/locale": "0.4.*",
"utopia-php/logger": "0.3.*",
"utopia-php/messaging": "0.1.*",
"utopia-php/orchestration": "0.6.*",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.4.*",
"utopia-php/pools": "0.4.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/queue": "0.5.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "0.14.*",
"utopia-php/swoole": "0.8.*",
@ -65,12 +69,12 @@
"resque/php-resque": "1.3.6",
"matomo/device-detector": "6.1.*",
"dragonmantank/cron-expression": "3.3.2",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.8.0",
"chillerlan/php-qrcode": "4.3.4",
"adhocore/jwt": "1.1.2",
"webonyx/graphql-php": "14.11.*",
"slickdeals/statsd": "3.1.0",
"webonyx/graphql-php": "14.11.*"
"league/csv": "9.7.1"
},
"repositories": [
{

1065
composer.lock generated
View file

File diff suppressed because it is too large Load diff

445
docker-compose.yml
View file

@ -14,7 +14,7 @@ version: '3'
services:
traefik:
image: traefik:2.7
image: traefik:2.9
<<: *x-logging
container_name: appwrite-traefik
command:
@ -53,6 +53,7 @@ services:
DEBUG: false
TESTING: true
VERSION: dev
VITE_CONSOLE_MODE: cloud
ports:
- 9501:80
networks:
@ -83,7 +84,7 @@ services:
- ./docs:/usr/src/code/docs
- ./public:/usr/src/code/public
- ./src:/usr/src/code/src
- ./dev:/usr/src/code/dev
- ./dev:/usr/local/dev
depends_on:
- mariadb
- redis
@ -96,10 +97,13 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_SERVER_MULTIPROCESS=enabled
- _APP_LOCALE
- _APP_CONSOLE_WHITELIST_ROOT
- _APP_CONSOLE_WHITELIST_EMAILS
- _APP_CONSOLE_WHITELIST_CODES
- _APP_CONSOLE_WHITELIST_IPS
- _APP_CONSOLE_INVITES
- _APP_CONSOLE_ROOT_SESSION
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
@ -110,63 +114,45 @@ services:
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_CONNECTIONS_PUBSUB
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USERS_STATS_RECIPIENTS
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
- _APP_STORAGE_ANTIVIRUS_HOST
- _APP_STORAGE_ANTIVIRUS_PORT
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_CONNECTIONS_STORAGE
- _APP_FUNCTIONS_SIZE_LIMIT
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_FUNCTIONS_RUNTIMES
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@ -175,9 +161,12 @@ services:
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_REGION
- _APP_GRAPHQL_MAX_BATCH_SIZE
- _APP_GRAPHQL_MAX_COMPLEXITY
- _APP_GRAPHQL_MAX_DEPTH
- _APP_CONSOLE_GITHUB_APP_ID
- _APP_CONSOLE_GITHUB_SECRET
appwrite-realtime:
entrypoint: realtime
@ -213,15 +202,25 @@ services:
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_SERVER_MULTIPROCESS=enabled
- _APP_OPTIONS_ABUSE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_PUBSUB
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -241,16 +240,21 @@ services:
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -270,12 +274,16 @@ services:
- request-catcher
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -299,37 +307,24 @@ services:
- ./src:/usr/src/code/src
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_CONNECTIONS_STORAGE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_EXECUTOR_SECRET
@ -350,16 +345,23 @@ services:
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -378,18 +380,26 @@ services:
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_CONNECTIONS_STORAGE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -410,19 +420,26 @@ services:
- ./src:/usr/src/code/src
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -439,87 +456,34 @@ services:
depends_on:
- redis
- mariadb
- appwrite-executor
- openruntimes-executor
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_FUNCTIONS_TIMEOUT
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-executor:
container_name: appwrite-executor
<<: *x-logging
entrypoint: executor
stop_signal: SIGINT
image: appwrite-dev
networks:
appwrite:
runtimes:
ports:
- 9519:80
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./dev:/usr/src/code/dev
- appwrite-functions:/storage/functions:rw
- appwrite-builds:/storage/builds:rw
- /tmp:/tmp:rw
depends_on:
- redis
- mariadb
- appwrite
environment:
- _APP_ENV
- _APP_VERSION
- _APP_FUNCTIONS_TIMEOUT
- _APP_FUNCTIONS_BUILD_TIMEOUT
- _APP_FUNCTIONS_CONTAINERS
- _APP_FUNCTIONS_RUNTIMES
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_FUNCTIONS_INACTIVE_THRESHOLD
- _APP_EXECUTOR_SECRET
- OPEN_RUNTIMES_NETWORK
- _APP_LOGGING_PROVIDER
- _APP_DOCKER_HUB_USERNAME
- _APP_DOCKER_HUB_PASSWORD
- _APP_LOGGING_CONFIG
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
- _APP_LOGGING_PROVIDER
appwrite-worker-mails:
entrypoint: worker-mails
@ -537,6 +501,9 @@ services:
# - smtp
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_SYSTEM_EMAIL_NAME
- _APP_SYSTEM_EMAIL_ADDRESS
@ -544,6 +511,7 @@ services:
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
- _APP_SMTP_HOST
- _APP_SMTP_PORT
- _APP_SMTP_SECURE
@ -567,15 +535,53 @@ services:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_QUEUE
- _APP_SMS_PROVIDER
- _APP_SMS_FROM
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-hamster:
entrypoint: hamster
<<: *x-logging
container_name: appwrite-hamster
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_HAMSTER_INTERVAL
- _APP_HAMSTER_TIME
- _APP_MIXPANEL_TOKEN
appwrite-maintenance:
entrypoint: maintenance
<<: *x-logging
@ -586,34 +592,39 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
#- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database
depends_on:
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_DOMAIN
- _APP_DOMAIN_TARGET
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage:
entrypoint: usage
appwrite-worker-usage:
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-usage
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
@ -621,23 +632,28 @@ services:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- influxdb
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
@ -652,14 +668,28 @@ services:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- mariadb
- redis
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_REGION
appwrite-assistant:
container_name: appwrite-assistant
image: appwrite/assistant:0.1.0
@ -667,7 +697,33 @@ services:
- appwrite
environment:
- OPENAI_API_KEY
openruntimes-executor:
container_name: openruntimes-executor
hostname: exc1
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.1.6
networks:
- appwrite
- runtimes
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-builds:/storage/builds:rw
- appwrite-functions:/storage/functions:rw
- /tmp:/tmp:rw
environment:
- OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
- OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
- OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
- OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
- OPR_EXECUTOR_DOCKER_HUB_USERNAME=$_APP_DOCKER_HUB_USERNAME
- OPR_EXECUTOR_DOCKER_HUB_PASSWORD=$_APP_DOCKER_HUB_PASSWORD
- OPR_EXECUTOR_ENV=$_APP_ENV
- OPR_EXECUTOR_RUNTIMES=$_APP_FUNCTIONS_RUNTIMES
- OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET
- OPR_EXECUTOR_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER
- OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
mariadb:
image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
@ -684,13 +740,11 @@ services:
- MYSQL_DATABASE=${_APP_DB_SCHEMA}
- MYSQL_USER=${_APP_DB_USER}
- MYSQL_PASSWORD=${_APP_DB_PASS}
command: 'mysqld --innodb-flush-method=fsync' # add ' --query_cache_size=0' for DB tests
# command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu
command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
# smtp:
# image: appwrite/smtp:1.2.0
# container_name: appwrite-smtp
# restart: unless-stopped
# networks:
# - appwrite
# environment:
@ -698,7 +752,7 @@ services:
# - RELAY_FROM_HOSTS=192.168.0.0/16 ; *.yourdomain.com
# - SMARTHOST_HOST=smtp
# - SMARTHOST_PORT=587
redis:
image: redis:7.0.4-alpine
<<: *x-logging
@ -722,28 +776,8 @@ services:
# - appwrite
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
# Dev Tools Start ------------------------------------------------------------------------------------------
#
#
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
#
# Here is a description of the different tools and why are we using them:
@ -752,7 +786,6 @@ services:
# RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
# RedisCommander - A nice UI for exploring Redis data
# Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
# Chronograf - A nice UI for exploring InfluxDB data
# Webgrind - A nice UI for exploring and debugging code-level stuff
maildev: # used mainly for dev tests
@ -777,15 +810,26 @@ services:
image: adminer
container_name: appwrite-adminer
<<: *x-logging
restart: always
ports:
- 9506:8080
networks:
- appwrite
# appwrite-volume-sync:
# entrypoint: volume-sync
# <<: *x-logging
# container_name: appwrite-volume-sync
# image: appwrite-dev
# command:
# - --source=/data/src/ --destination=/data/dest/ --interval=10
# networks:
# - appwrite
# # volumes: # Mount the rsync source and destination directories
# # - /nfs/config:/data/src
# # - /storage/config:/data/dest
# redis-commander:
# image: rediscommander/redis-commander:latest
# restart: unless-stopped
# networks:
# - appwrite
# environment:
@ -795,7 +839,6 @@ services:
# resque:
# image: appwrite/resque-web:1.1.0
# restart: unless-stopped
# networks:
# - appwrite
# ports:
@ -805,26 +848,6 @@ services:
# - RESQUE_WEB_PORT=6379
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# chronograf:
# image: chronograf:1.6
# container_name: appwrite-chronograf
# restart: unless-stopped
# networks:
# - appwrite
# volumes:
# - appwrite-chronograf:/var/lib/chronograf
# ports:
# - "8888:8888"
# environment:
# - INFLUXDB_URL=http://influxdb:8086
# - KAPACITOR_URL=http://kapacitor:9092
# - AUTH_DURATION=48h
# - TOKEN_SECRET=duperduper5674829!jwt
# - GH_CLIENT_ID=d86f7145a41eacfc52cc
# - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
# - GH_ORGS=appwrite
# webgrind:
# image: 'jokkedk/webgrind:latest'
# volumes:
@ -848,8 +871,11 @@ services:
networks:
gateway:
name: gateway
appwrite:
name: appwrite
runtimes:
name: runtimes
volumes:
appwrite-mariadb:
@ -859,7 +885,4 @@ volumes:
appwrite-certificates:
appwrite-functions:
appwrite-builds:
appwrite-influxdb:
appwrite-config:
appwrite-executor:
# appwrite-chronograf:

2
docs/references/health/get-cache.md
View file

@ -1 +1 @@
Check the Appwrite in-memory cache server is up and connection is successful.
Check the Appwrite in-memory cache servers are up and connection is successful.

2
docs/references/health/get-db.md
View file

@ -1 +1 @@
Check the Appwrite database server is up and connection is successful.
Check the Appwrite database servers are up and connection is successful.

1
docs/references/health/get-pubsub.md Normal file
View file

@ -0,0 +1 @@
Check the Appwrite pub-sub servers are up and connection is successful.

1
docs/references/health/get-queue.md Normal file
View file

@ -0,0 +1 @@
Check the Appwrite queue messaging servers are up and connection is successful.

880
docs/specs/overview.drawio.svg
View file

File diff suppressed because one or more lines are too long
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 57 KiB

2
docs/tutorials/add-environment-variable.md
View file

@ -12,7 +12,7 @@ This tutorial will cover how to properly add a new environment variable in Appwr
### Naming environment variable
The environment variables in Appwrite are prefixed with `_APP_`. If it belongs to a specific category, the category name is appended as `_APP_REDIS` for the Redis category. The available categories are General, Redis, MariaDB, InfluxDB, StatsD, SMTP, Storage, Functions and Maintenance. Finally, a properly describing name is given to the variable. For example, `_APP_REDIS_HOST` is an environment variable for the hostname of your Redis instance. You can find more information on available categories and existing environment variables in the [environment variables doc](https://appwrite.io/docs/environment-variables).
The environment variables in Appwrite are prefixed with `_APP_`. If it belongs to a specific category, the category name is appended as `_APP_REDIS` for the Redis category. The available categories are General, Redis, MariaDB, SMTP, Storage, Functions and Maintenance. Finally, a properly describing name is given to the variable. For example, `_APP_REDIS_HOST` is an environment variable for the hostname of your Redis instance. You can find more information on available categories and existing environment variables in the [environment variables doc](https://appwrite.io/docs/environment-variables).
### Describe new environment variable

11
docs/tutorials/add-route.md
View file

@ -131,17 +131,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('event', 'buckets.[bucketId].files.[fileId].create')
```
#### Usage
- usage.metric - The metric the route generates.
- usage.params - Additional parameters the metrics can have.
```php
App::post('/v1/storage/buckets/:bucketId/files')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
```
### 5. Param
As the name implies, `param()` is used to define a request parameter.

2
docs/tutorials/multi-architecture-support.md
View file

@ -6,8 +6,6 @@ A list of Appwrite CPU architecture support status. We use this list to track th
|---|---|---|---|---|---|---|---|
| appwrite/appwrite | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 |
| appwrite/mariadb | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 | 🟢 | 🔴 |
| appwrite/influxdb | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 |
| appwrite/telegraf | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 |
| appwrite/clamav | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 |
| traefik | 🟢 | 🔴 | 🟢 | 🔴 | 🟢 | 🔴 | 🔴 |
| redis | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 |

BIN
public/fonts/Inter-Bold.ttf Normal file
View file

Binary file not shown.

BIN
public/fonts/Inter-Medium.ttf Normal file
View file

Binary file not shown.

BIN
public/fonts/Inter-Regular.ttf Normal file
View file

Binary file not shown.

BIN
public/fonts/Inter-SemiBold.ttf Normal file
View file

Binary file not shown.

BIN
public/fonts/Poppins-Bold.ttf Normal file
View file

Binary file not shown.

BIN
public/fonts/SourceCodePro-Regular.ttf Normal file
View file

Binary file not shown.

BIN
public/images/cards/cloud/back-golden.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 539 KiB

BIN
public/images/cards/cloud/back-platinum.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 358 KiB

BIN
public/images/cards/cloud/back.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 359 KiB

BIN
public/images/cards/cloud/contributor-skew.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 4.4 KiB

BIN
public/images/cards/cloud/contributor.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 11 KiB

BIN
public/images/cards/cloud/employee-skew.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
public/images/cards/cloud/employee.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.4 KiB

BIN
public/images/cards/cloud/front-golden.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 500 KiB

BIN
public/images/cards/cloud/front-platinum.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 353 KiB

BIN
public/images/cards/cloud/front.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 193 KiB

BIN
public/images/cards/cloud/github-skew.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 513 B

Some files were not shown because too many files have changed in this diff Show more