1
0
Fork 0
mirror of synced 2024-06-14 00:34:51 +12:00

Merge branch '1.4.x' of https://github.com/appwrite/appwrite into feat-refactor-hamster

This commit is contained in:
德华 刘 2023-11-30 21:20:59 +00:00
commit b6fabf80ef
46 changed files with 449 additions and 106 deletions

View file

@ -18,6 +18,10 @@ jobs:
- run: git checkout HEAD^2
- name: Validate composer.json and composer.lock
run: |
docker run --rm -v $PWD:/app composer sh -c \
"composer validate"
- name: Run Linter
run: |
docker run --rm -v $PWD:/app composer sh -c \

2
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = 3.2.7
branch = 3.2.9

View file

@ -1,3 +1,21 @@
# Version 1.4.13
## Notable changes
* Change enum size validation in update controller [#7164](https://github.com/appwrite/appwrite/pull/7164)
* Bump console to version 3.2.8 in [#7167](https://github.com/appwrite/appwrite/pull/7167)
## Bug fixes
* Fix error after adding bigger enum [#7162](https://github.com/appwrite/appwrite/pull/7162)
* Add chunkId to abuse key to prevent rate limit for SDKs [#7154](https://github.com/appwrite/appwrite/pull/7154)
## Miscellaneous
* Fix enum test case [#7163](https://github.com/appwrite/appwrite/pull/7163)
* Add flag to send logs to logger [#7155](https://github.com/appwrite/appwrite/pull/7155)
* Add a CI task to validate composer file and lock [#7142](https://github.com/appwrite/appwrite/pull/7142)
# Version 1.4.12
## Miscellaneous

View file

@ -94,7 +94,8 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-migrations
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-hamster
# Cloud Executabless
RUN chmod +x /usr/local/bin/hamster && \
@ -106,7 +107,7 @@ RUN chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/clear-card-cache && \
chmod +x /usr/local/bin/calc-users-stats && \
chmod +x /usr/local/bin/calc-tier-stats && \
chmod +x /usr/local/bin/worker-hamster
chmod +x /usr/local/bin/get-migration-stats
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

View file

@ -66,7 +66,7 @@ docker run -it --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
--entrypoint="install" \
appwrite/appwrite:1.4.12
appwrite/appwrite:1.4.13
```
### Windows
@ -78,7 +78,7 @@ docker run -it --rm ^
--volume //var/run/docker.sock:/var/run/docker.sock ^
--volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
--entrypoint="install" ^
appwrite/appwrite:1.4.12
appwrite/appwrite:1.4.13
```
#### PowerShell
@ -88,7 +88,7 @@ docker run -it --rm `
--volume /var/run/docker.sock:/var/run/docker.sock `
--volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
--entrypoint="install" `
appwrite/appwrite:1.4.12
appwrite/appwrite:1.4.13
```
运行后,可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后,服务器可能需要几分钟才能启动。

View file

@ -76,7 +76,7 @@ docker run -it --rm \
--volume /var/run/docker.sock:/var/run/docker.sock \
--volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
--entrypoint="install" \
appwrite/appwrite:1.4.12
appwrite/appwrite:1.4.13
```
### Windows
@ -88,7 +88,7 @@ docker run -it --rm ^
--volume //var/run/docker.sock:/var/run/docker.sock ^
--volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
--entrypoint="install" ^
appwrite/appwrite:1.4.12
appwrite/appwrite:1.4.13
```
#### PowerShell
@ -98,7 +98,7 @@ docker run -it --rm `
--volume /var/run/docker.sock:/var/run/docker.sock `
--volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
--entrypoint="install" `
appwrite/appwrite:1.4.12
appwrite/appwrite:1.4.13
```
Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.

View file

@ -209,6 +209,7 @@ return [
'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED,
'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.',
'code' => 501,
'publish' => false,
],
Exception::USER_PHONE_ALREADY_EXISTS => [
'name' => Exception::USER_PHONE_ALREADY_EXISTS,
@ -764,4 +765,12 @@ return [
'description' => 'An error occurred on the provider\'s side. Please try again later.',
'code' => 400,
],
/** Health */
Exception::QUEUE_SIZE_EXCEEDED => [
'name' => Exception::QUEUE_SIZE_EXCEEDED,
'description' => 'Queue size threshold hit.',
'code' => 503,
'publish' => false
],
];

View file

@ -15,7 +15,7 @@ return [
[
'key' => 'web',
'name' => 'Web',
'version' => '13.0.0',
'version' => '13.0.1',
'url' => 'https://github.com/appwrite/sdk-for-web',
'package' => 'https://www.npmjs.com/package/appwrite',
'enabled' => true,
@ -63,7 +63,7 @@ return [
[
'key' => 'flutter',
'name' => 'Flutter',
'version' => '11.0.0',
'version' => '11.0.1',
'url' => 'https://github.com/appwrite/sdk-for-flutter',
'package' => 'https://pub.dev/packages/appwrite',
'enabled' => true,
@ -81,7 +81,7 @@ return [
[
'key' => 'apple',
'name' => 'Apple',
'version' => '4.0.1',
'version' => '4.0.2',
'url' => 'https://github.com/appwrite/sdk-for-apple',
'package' => 'https://github.com/appwrite/sdk-for-apple',
'enabled' => true,
@ -116,7 +116,7 @@ return [
[
'key' => 'android',
'name' => 'Android',
'version' => '4.0.0',
'version' => '4.0.1',
'url' => 'https://github.com/appwrite/sdk-for-android',
'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-android',
'enabled' => true,
@ -203,7 +203,7 @@ return [
[
'key' => 'cli',
'name' => 'Command Line',
'version' => '4.1.0',
'version' => '4.2.0',
'url' => 'https://github.com/appwrite/sdk-for-cli',
'package' => 'https://www.npmjs.com/package/appwrite-cli',
'enabled' => true,
@ -231,7 +231,7 @@ return [
[
'key' => 'nodejs',
'name' => 'Node.js',
'version' => '11.0.0',
'version' => '11.1.0',
'url' => 'https://github.com/appwrite/sdk-for-node',
'package' => 'https://www.npmjs.com/package/node-appwrite',
'enabled' => true,
@ -249,7 +249,7 @@ return [
[
'key' => 'deno',
'name' => 'Deno',
'version' => '9.0.0',
'version' => '9.1.0',
'url' => 'https://github.com/appwrite/sdk-for-deno',
'package' => 'https://deno.land/x/appwrite',
'enabled' => true,
@ -267,7 +267,7 @@ return [
[
'key' => 'php',
'name' => 'PHP',
'version' => '10.0.0',
'version' => '10.1.0',
'url' => 'https://github.com/appwrite/sdk-for-php',
'package' => 'https://packagist.org/packages/appwrite/appwrite',
'enabled' => true,
@ -285,7 +285,7 @@ return [
[
'key' => 'python',
'name' => 'Python',
'version' => '4.0.0',
'version' => '4.1.0',
'url' => 'https://github.com/appwrite/sdk-for-python',
'package' => 'https://pypi.org/project/appwrite/',
'enabled' => true,
@ -303,7 +303,7 @@ return [
[
'key' => 'ruby',
'name' => 'Ruby',
'version' => '10.0.0',
'version' => '10.1.0',
'url' => 'https://github.com/appwrite/sdk-for-ruby',
'package' => 'https://rubygems.org/gems/appwrite',
'enabled' => true,
@ -321,7 +321,7 @@ return [
[
'key' => 'go',
'name' => 'Go',
'version' => '3.0.0',
'version' => '3.1.0',
'url' => 'https://github.com/appwrite/sdk-for-go',
'package' => '',
'enabled' => false,
@ -339,7 +339,7 @@ return [
[
'key' => 'java',
'name' => 'Java',
'version' => '3.0.0',
'version' => '3.1.0',
'url' => 'https://github.com/appwrite/sdk-for-java',
'package' => '',
'enabled' => false,
@ -357,7 +357,7 @@ return [
[
'key' => 'dotnet',
'name' => '.NET',
'version' => '0.6.0',
'version' => '0.7.0',
'url' => 'https://github.com/appwrite/sdk-for-dotnet',
'package' => 'https://www.nuget.org/packages/Appwrite',
'enabled' => true,
@ -375,7 +375,7 @@ return [
[
'key' => 'dart',
'name' => 'Dart',
'version' => '10.0.0',
'version' => '10.1.0',
'url' => 'https://github.com/appwrite/sdk-for-dart',
'package' => 'https://pub.dev/packages/dart_appwrite',
'enabled' => true,
@ -393,7 +393,7 @@ return [
[
'key' => 'kotlin',
'name' => 'Kotlin',
'version' => '4.0.0',
'version' => '4.1.0',
'url' => 'https://github.com/appwrite/sdk-for-kotlin',
'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-kotlin',
'enabled' => true,
@ -415,7 +415,7 @@ return [
[
'key' => 'swift',
'name' => 'Swift',
'version' => '4.0.1',
'version' => '4.1.0',
'url' => 'https://github.com/appwrite/sdk-for-swift',
'package' => 'https://github.com/appwrite/sdk-for-swift',
'enabled' => true,

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -1231,7 +1231,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
->param('databaseId', '', new UID(), 'Database ID.')
->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
->param('key', '', new Key(), 'Attribute Key.')
->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 0), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.')
->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.')
->param('required', null, new Boolean(), 'Is attribute required?')
->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true)
->param('array', false, new Boolean(), 'Is attribute an array?', true)
@ -1240,16 +1240,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
->inject('queueForDatabase')
->inject('queueForEvents')
->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
// use length of longest string as attribute size
$size = 0;
foreach ($elements as $element) {
$length = \strlen($element);
if ($length === 0) {
throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Each enum element must not be empty');
}
$size = ($length > $size) ? $length : $size;
}
if (!is_null($default) && !in_array($default, $elements)) {
throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Default value not found in elements');
}
@ -1257,7 +1247,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
$attribute = createAttribute($databaseId, $collectionId, new Document([
'key' => $key,
'type' => Database::VAR_STRING,
'size' => $size,
'size' => Database::LENGTH_KEY,
'required' => $required,
'default' => $default,
'array' => $array,
@ -1930,7 +1920,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
->param('databaseId', '', new UID(), 'Database ID.')
->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
->param('key', '', new Key(), 'Attribute Key.')
->param('elements', null, new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.')
->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.')
->param('required', null, new Boolean(), 'Is attribute required?')
->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.')
->inject('response')

View file

@ -355,7 +355,7 @@ App::get('/v1/health/queue/webhooks')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -382,7 +382,7 @@ App::get('/v1/health/queue/logs')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -409,7 +409,7 @@ App::get('/v1/health/queue/certificates')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -436,7 +436,7 @@ App::get('/v1/health/queue/builds')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -464,7 +464,7 @@ App::get('/v1/health/queue/databases')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -491,7 +491,7 @@ App::get('/v1/health/queue/deletes')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -518,7 +518,7 @@ App::get('/v1/health/queue/mails')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -545,7 +545,7 @@ App::get('/v1/health/queue/messaging')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -572,7 +572,7 @@ App::get('/v1/health/queue/migrations')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
@ -599,7 +599,7 @@ App::get('/v1/health/queue/functions')
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);

View file

@ -339,7 +339,7 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId},chunkId:{chunkId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])

View file

@ -608,8 +608,13 @@ App::error()
$version = App::getEnv('_APP_VERSION', 'UNKNOWN');
$route = $utopia->getRoute();
$publish = true;
if ($logger) {
if ($error instanceof AppwriteException) {
$publish = $error->isPublishable();
}
if ($logger && $publish) {
if ($error->getCode() >= 500 || $error->getCode() === 0) {
try {
/** @var Utopia\Database\Document $user */

View file

@ -121,13 +121,16 @@ App::init()
$abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel;
foreach ($abuseKeyLabel as $abuseKey) {
$start = $request->getContentRangeStart();
$end = $request->getContentRangeEnd();
$timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
$timeLimit
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
->setParam('{method}', $request->getMethod())
->setParam('{chunkId}', (int) ($start / ($end + 1 - $start)));
$timeLimitArray[] = $timeLimit;
}

View file

@ -109,8 +109,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
const APP_CACHE_BUSTER = 327;
const APP_VERSION_STABLE = '1.4.12';
const APP_CACHE_BUSTER = 328;
const APP_VERSION_STABLE = '1.4.13';
const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@ -767,6 +767,25 @@ $register->set('pools', function () {
return $group;
});
$register->set('db', function () {
// This is usually for our workers or CLI commands scope
$dbHost = App::getEnv('_APP_DB_HOST', '');
$dbPort = App::getEnv('_APP_DB_PORT', '');
$dbUser = App::getEnv('_APP_DB_USER', '');
$dbPass = App::getEnv('_APP_DB_PASS', '');
$dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
$pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
PDO::ATTR_TIMEOUT => 3, // Seconds
PDO::ATTR_PERSISTENT => true,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::ATTR_EMULATE_PREPARES => true,
PDO::ATTR_STRINGIFY_FETCHES => true,
));
return $pdo;
});
$register->set('influxdb', function () {
// Register DB connection

3
bin/get-migration-stats Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php get-migration-stats $@

View file

@ -86,7 +86,7 @@
],
"require-dev": {
"ext-fileinfo": "*",
"appwrite/sdk-generator": "0.35.*",
"appwrite/sdk-generator": "0.36.*",
"phpunit/phpunit": "9.5.20",
"squizlabs/php_codesniffer": "^3.7",
"swoole/ide-helper": "5.0.2",

14
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "69bc2e21a65b78344393706b39d789b4",
"content-hash": "eff47c9354bb55140097067b71425f91",
"packages": [
{
"name": "adhocore/jwt",
@ -3136,16 +3136,16 @@
"packages-dev": [
{
"name": "appwrite/sdk-generator",
"version": "0.35.2",
"version": "0.36.0",
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator.git",
"reference": "2dfe0430a64ffd2a07078d83b20144b871acac3b"
"reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/2dfe0430a64ffd2a07078d83b20144b871acac3b",
"reference": "2dfe0430a64ffd2a07078d83b20144b871acac3b",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3a10f1f895ed71120442ff71eb6adec3fd6b4e8a",
"reference": "3a10f1f895ed71120442ff71eb6adec3fd6b4e8a",
"shasum": ""
},
"require": {
@ -3181,9 +3181,9 @@
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"support": {
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/0.35.2"
"source": "https://github.com/appwrite/sdk-generator/tree/0.36.0"
},
"time": "2023-09-14T14:59:50+00:00"
"time": "2023-11-20T10:03:06+00:00"
},
{
"name": "doctrine/deprecations",

View file

@ -1 +1,2 @@
appwrite health getQueueBuilds
appwrite health getQueueBuilds \

View file

@ -1 +1,2 @@
appwrite health getQueueCertificates
appwrite health getQueueCertificates \

View file

@ -1,2 +1,3 @@
appwrite health getQueueDatabases \

View file

@ -1 +1,2 @@
appwrite health getQueueDeletes
appwrite health getQueueDeletes \

View file

@ -1 +1,2 @@
appwrite health getQueueFunctions
appwrite health getQueueFunctions \

View file

@ -1 +1,2 @@
appwrite health getQueueLogs
appwrite health getQueueLogs \

View file

@ -1 +1,2 @@
appwrite health getQueueMails
appwrite health getQueueMails \

View file

@ -1 +1,2 @@
appwrite health getQueueMessaging
appwrite health getQueueMessaging \

View file

@ -1 +1,2 @@
appwrite health getQueueMigrations
appwrite health getQueueMigrations \

View file

@ -1 +1,2 @@
appwrite health getQueueWebhooks
appwrite health getQueueWebhooks \

View file

@ -1,3 +1,8 @@
## 10.1.0
* Add new queue health endpoints
* Fix between queries
## 10.0.0
* Parameter `url` is now optional in the `createMembership` endpoint

View file

@ -1,3 +1,7 @@
## 11.0.1
* Fix between queries
## 11.0.0
* Parameter `url` is now optional in the `createMembership` endpoint

View file

@ -229,12 +229,16 @@ class Exception extends \Exception
public const MIGRATION_PROVIDER_ERROR = 'migration_provider_error';
/** Realtime */
public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
/** Health */
public const QUEUE_SIZE_EXCEEDED = 'queue_size_exceeded';
protected string $type = '';
protected array $errors = [];
protected bool $publish = true;
public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
{
@ -244,6 +248,7 @@ class Exception extends \Exception
if (isset($this->errors[$type])) {
$this->code = $this->errors[$type]['code'];
$this->message = $this->errors[$type]['description'];
$this->publish = $this->errors[$type]['publish'] ?? true;
}
$this->message = $message ?? $this->message;
@ -273,4 +278,14 @@ class Exception extends \Exception
{
$this->type = $type;
}
/**
* Check whether the log is publishable for the exception.
*
* @return bool
*/
public function isPublishable(): bool
{
return $this->publish;
}
}

View file

@ -76,7 +76,8 @@ abstract class Migration
'1.4.9' => 'V19',
'1.4.10' => 'V19',
'1.4.11' => 'V19',
'1.4.12' => 'V19'
'1.4.12' => 'V19',
'1.4.13' => 'V19'
];
/**
@ -194,17 +195,24 @@ abstract class Migration
* @return iterable<Document>
* @throws \Exception
*/
public function documentsIterator(string $collectionId): iterable
public function documentsIterator(string $collectionId, $queries = []): iterable
{
$sum = 0;
$nextDocument = null;
$collectionCount = $this->projectDB->count($collectionId);
$queries[] = Query::limit($this->limit);
do {
$queries = [Query::limit($this->limit)];
if ($nextDocument !== null) {
$queries[] = Query::cursorAfter($nextDocument);
$cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries));
if ($cursorQueryIndex !== false) {
$queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument);
} else {
$queries[] = Query::cursorAfter($nextDocument);
}
}
$documents = $this->projectDB->find($collectionId, $queries);
$count = count($documents);
$sum += $count;

View file

@ -10,6 +10,7 @@ use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception;
use Utopia\Database\Query;
class V19 extends Migration
{
@ -41,6 +42,11 @@ class V19 extends Migration
Console::info('Migrating Buckets');
$this->migrateBuckets();
if ($this->project->getId() !== 'console') {
Console::info('Migrating Enum Attribute Size');
$this->migrateEnumAttributeSize();
}
Console::info('Migrating Documents');
$this->forEachDocument([$this, 'fixDocument']);
@ -640,6 +646,22 @@ class V19 extends Migration
return $commands;
}
private function migrateEnumAttributeSize(): void
{
foreach (
$this->documentsIterator('attributes', [
Query::equal('format', ['enum']),
Query::lessThan('size', Database::LENGTH_KEY)
]) as $attribute
) {
$attribute->setAttribute('size', Database::LENGTH_KEY);
$this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute);
$databaseInternalId = $attribute->getAttribute('databaseInternalId');
$collectionInternalId = $attribute->getAttribute('collectionInternalId');
$this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255);
}
}
/**
* Fix run on each document
*

View file

@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\GetMigrationStats;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
class Tasks extends Service
@ -44,6 +45,7 @@ class Tasks extends Service
->addAction(CalcTierStats::getName(), new CalcTierStats())
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
->addAction(GetMigrationStats::getName(), new GetMigrationStats())
;
}

View file

@ -41,6 +41,7 @@ class CalcTierStats extends Action
'Functions',
'Deployments',
'Executions',
'Migrations',
];
protected string $directory = '/usr/local';
@ -99,8 +100,8 @@ class CalcTierStats extends Action
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$limit = 100;
$sum = 100;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
@ -200,7 +201,7 @@ class CalcTierStats extends Action
try {
/** Get Domains */
$stats['Domains'] = $dbForConsole->count('domains', [
$stats['Domains'] = $dbForConsole->count('rules', [
Query::equal('projectInternalId', [$project->getInternalId()]),
]);
} catch (\Throwable) {
@ -290,6 +291,13 @@ class CalcTierStats extends Action
$stats['Executions'] = 0;
}
/** Get Total Migrations */
try {
$stats['Migrations'] = $dbForProject->count('migrations', []);
} catch (\Throwable) {
$stats['Migrations'] = 0;
}
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());

View file

@ -4,6 +4,7 @@ namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
@ -45,6 +46,17 @@ class DeleteOrphanedProjects extends Action
/** @var array $collections */
$collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
$collectionsConfig = array_merge([
'audit' => [
'$id' => ID::custom('audit'),
'$collection' => Database::METADATA
],
'abuse' => [
'$id' => ID::custom('abuse'),
'$collection' => Database::METADATA
]
], $collectionsConfig);
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
@ -54,7 +66,7 @@ class DeleteOrphanedProjects extends Action
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$orphans = 0;
$orphans = 1;
$cnt = 0;
$count = 0;
$limit = 30;
@ -80,6 +92,7 @@ class DeleteOrphanedProjects extends Action
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
$collectionsCreated = 0;
$cnt++;
if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
@ -87,10 +100,8 @@ class DeleteOrphanedProjects extends Action
}
$msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')';
/**
* +2 = audit+abuse
*/
if ($collectionsCreated >= (count($collectionsConfig) + 2)) {
if ($collectionsCreated >= count($collectionsConfig)) {
Console::log($msg . ' ignoring....');
continue;
}
@ -107,16 +118,26 @@ class DeleteOrphanedProjects extends Action
Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')');
}
}
if ($commit) {
$dbForConsole->deleteDocument('projects', $project->getId());
$dbForConsole->deleteCachedDocument('projects', $project->getId());
if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
try {
$dbForProject->deleteCollection(Database::METADATA);
$dbForProject->deleteCachedCollection(Database::METADATA);
} catch (\Throwable $th) {
Console::warning('Metadata collection does not exist');
}
}
}
Console::info('--Deleting project no (' . $project->getInternalId() . ')');
$orphans++;
} catch (\Throwable $th) {
Console::error('Error: ' . $th->getMessage());
Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString());
} finally {
$pools
->get($db)
@ -135,6 +156,6 @@ class DeleteOrphanedProjects extends Action
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans');
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans');
}
}

View file

@ -0,0 +1,187 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use League\Csv\CannotInsertRecord;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class GetMigrationStats extends Action
{
/*
* Csv cols headers
*/
private array $columns = [
'Project ID',
'$id',
'$createdAt',
'status',
'stage',
'source'
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'get-migration-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
/**
* @throws \Utopia\Exception
* @throws CannotInsertRecord
*/
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite get-migration-stats
Console::title('Migration stats calculation V1');
Console::success(APP_NAME . ' Migration stats calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/migration_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 100;
$sum = 100;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Migration details */
$migrations = $dbForProject->find('migrations', [
Query::limit(500)
]);
$migrations = array_map(function ($migration) use ($project) {
return [
$project->getId(),
$migration->getAttribute('$id'),
$migration->getAttribute('$createdAt'),
$migration->getAttribute('status'),
$migration->getAttribute('stage'),
$migration->getAttribute('source'),
];
}, $migrations);
if (!empty($migrations)) {
$csv->insertAll($migrations);
}
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Migration Report for {$this->date}";
$mail->Body = "Please find the migration report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}

View file

@ -11,6 +11,7 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Registry\Registry;
use Utopia\Validator\Text;
class Migrate extends Action
@ -29,7 +30,8 @@ class Migrate extends Action
->inject('cache')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB));
->inject('register')
->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register));
}
private function clearProjectsCache(Cache $cache, Document $project)
@ -41,7 +43,7 @@ class Migrate extends Action
}
}
public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB)
public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register)
{
Authorization::disable();
if (!array_key_exists($version, Migration::$versions)) {
@ -89,9 +91,11 @@ class Migrate extends Action
try {
// TODO: Iterate through all project DBs
/** @var Database $projectDB */
$projectDB = $getProjectDB($project);
$migration
->setProject($project, $projectDB, $dbForConsole)
->setPDO($register->get('db', true))
->execute();
} catch (\Throwable $th) {
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());

View file

@ -235,6 +235,11 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
'X-Appwrite-Response-Format' => '1.4.0',
]);
// Make sure we have a clean slate.
// Otherwise, all files in this dir will be pushed,
// regardless of whether they were just generated or not.
\exec('rm -rf ' . $result);
try {
$sdk->generate($result);
} catch (Exception $exception) {

View file

@ -978,7 +978,7 @@ trait DatabasesBase
]);
$this->assertEquals(400, $badEnum['headers']['status-code']);
$this->assertEquals('Each enum element must not be empty', $badEnum['body']['message']);
$this->assertEquals('Invalid `elements` param: Value must a valid array and Value must be a valid string and at least 1 chars and no longer than 255 chars', $badEnum['body']['message']);
return $data;
}

View file

@ -145,7 +145,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -171,7 +171,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -197,7 +197,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -223,7 +223,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -249,7 +249,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -280,7 +280,7 @@ class HealthCustomServerTest extends Scope
'name' => 'database_db_main',
'threshold' => '0'
]);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -306,7 +306,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -332,7 +332,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -358,7 +358,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}
@ -384,7 +384,7 @@ class HealthCustomServerTest extends Scope
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
$this->assertEquals(503, $response['headers']['status-code']);
return [];
}