Merge branch 'feat-database-indexing' into feat-storage-buckets
This commit is contained in:
commit
ca58351e93
6 changed files with 84 additions and 5 deletions
10
.travis.yml
10
.travis.yml
|
@ -6,6 +6,10 @@ arch:
|
||||||
|
|
||||||
os: linux
|
os: linux
|
||||||
|
|
||||||
|
# Small change
|
||||||
|
vm:
|
||||||
|
size: large
|
||||||
|
|
||||||
language: shell
|
language: shell
|
||||||
|
|
||||||
notifications:
|
notifications:
|
||||||
|
@ -35,6 +39,12 @@ install:
|
||||||
|
|
||||||
script:
|
script:
|
||||||
- docker ps -a
|
- docker ps -a
|
||||||
|
# Tests should fail if any container is in exited status
|
||||||
|
- ALL_UP=`docker ps -aq --filter "status=exited"`
|
||||||
|
- >
|
||||||
|
if [[ "$ALL_UP" != "" ]]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
- docker-compose logs appwrite
|
- docker-compose logs appwrite
|
||||||
- docker-compose logs mariadb
|
- docker-compose logs mariadb
|
||||||
- docker-compose logs appwrite-worker-functions
|
- docker-compose logs appwrite-worker-functions
|
||||||
|
|
|
@ -620,12 +620,15 @@ App::delete('/v1/database/collections/:collectionId')
|
||||||
throw new Exception('Collection not found', 404);
|
throw new Exception('Collection not found', 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
$dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker
|
|
||||||
|
|
||||||
if (!$dbForInternal->deleteDocument('collections', $collectionId)) {
|
if (!$dbForInternal->deleteDocument('collections', $collectionId)) {
|
||||||
throw new Exception('Failed to remove collection from DB', 500);
|
throw new Exception('Failed to remove collection from DB', 500);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$deletes
|
||||||
|
->setParam('type', DELETE_TYPE_DOCUMENT)
|
||||||
|
->setParam('document', $collection)
|
||||||
|
;
|
||||||
|
|
||||||
$usage->setParam('database.collections.delete', 1);
|
$usage->setParam('database.collections.delete', 1);
|
||||||
|
|
||||||
$events
|
$events
|
||||||
|
|
|
@ -42,7 +42,7 @@ $cli
|
||||||
function notifyDeleteUsageStats(int $interval30m, int $interval1d)
|
function notifyDeleteUsageStats(int $interval30m, int $interval1d)
|
||||||
{
|
{
|
||||||
Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
|
Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [
|
||||||
'type' => DELETE_TYPE_USAGE_STATS,
|
'type' => DELETE_TYPE_USAGE,
|
||||||
'timestamp1d' => time() - $interval1d,
|
'timestamp1d' => time() - $interval1d,
|
||||||
'timestamp30m' => time() - $interval30m,
|
'timestamp30m' => time() - $interval30m,
|
||||||
]);
|
]);
|
||||||
|
|
|
@ -255,10 +255,27 @@ $cli
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Aggregate InfluxDB every 30 seconds
|
* Aggregate InfluxDB every 30 seconds
|
||||||
|
* @var InfluxDB\Client $client
|
||||||
*/
|
*/
|
||||||
$client = $register->get('influxdb');
|
$client = $register->get('influxdb');
|
||||||
if ($client) {
|
if ($client) {
|
||||||
|
$attempts = 0;
|
||||||
|
$max = 10;
|
||||||
|
$sleep = 1;
|
||||||
|
|
||||||
$database = $client->selectDB('telegraf');
|
$database = $client->selectDB('telegraf');
|
||||||
|
do { // check if telegraf database is ready
|
||||||
|
$attempts++;
|
||||||
|
if(!in_array('telegraf', $client->listDatabases())) {
|
||||||
|
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
|
||||||
|
if($attempts >= $max) {
|
||||||
|
throw new \Exception('InfluxDB database not ready yet');
|
||||||
|
}
|
||||||
|
sleep($sleep);
|
||||||
|
} else {
|
||||||
|
break; // leave the do-while if successful
|
||||||
|
}
|
||||||
|
} while ($attempts < $max);
|
||||||
|
|
||||||
// sync data
|
// sync data
|
||||||
foreach ($globalMetrics as $metric => $options) { //for each metrics
|
foreach ($globalMetrics as $metric => $options) { //for each metrics
|
||||||
|
@ -335,7 +352,23 @@ $cli
|
||||||
$latestProject = null;
|
$latestProject = null;
|
||||||
|
|
||||||
do { // Loop over all the projects
|
do { // Loop over all the projects
|
||||||
|
$attempts = 0;
|
||||||
|
$max = 10;
|
||||||
|
$sleep = 1;
|
||||||
|
|
||||||
|
do { // list projects
|
||||||
|
try {
|
||||||
|
$attempts++;
|
||||||
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
|
$projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject);
|
||||||
|
break; // leave the do-while if successful
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
Console::warning("Console DB not ready yet. Retrying ({$attempts})...");
|
||||||
|
if ($attempts >= $max) {
|
||||||
|
throw new \Exception('Failed access console db: ' . $e->getMessage());
|
||||||
|
}
|
||||||
|
sleep($sleep);
|
||||||
|
}
|
||||||
|
} while ($attempts < $max);
|
||||||
|
|
||||||
if (empty($projects)) {
|
if (empty($projects)) {
|
||||||
continue;
|
continue;
|
||||||
|
|
|
@ -44,6 +44,9 @@ class DeletesV1 extends Worker
|
||||||
|
|
||||||
switch ($document->getCollection()) {
|
switch ($document->getCollection()) {
|
||||||
// TODO@kodumbeats define these as constants somewhere
|
// TODO@kodumbeats define these as constants somewhere
|
||||||
|
case 'collections':
|
||||||
|
$this->deleteCollection($document, $projectId);
|
||||||
|
break;
|
||||||
case 'projects':
|
case 'projects':
|
||||||
$this->deleteProject($document);
|
$this->deleteProject($document);
|
||||||
break;
|
break;
|
||||||
|
@ -79,7 +82,7 @@ class DeletesV1 extends Worker
|
||||||
$this->deleteCertificates($document);
|
$this->deleteCertificates($document);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DELETE_TYPE_USAGE_STATS:
|
case DELETE_TYPE_USAGE:
|
||||||
$this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']);
|
$this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
|
@ -92,6 +95,28 @@ class DeletesV1 extends Worker
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Document $document teams document
|
||||||
|
* @param string $projectId
|
||||||
|
*/
|
||||||
|
protected function deleteCollection(Document $document, string $projectId): void
|
||||||
|
{
|
||||||
|
$collectionId = $document->getId();
|
||||||
|
|
||||||
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
|
$dbForExternal = $this->getExternalDB($projectId);
|
||||||
|
|
||||||
|
$this->deleteByGroup('attributes', [
|
||||||
|
new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
|
||||||
|
], $dbForInternal);
|
||||||
|
|
||||||
|
$this->deleteByGroup('indexes', [
|
||||||
|
new Query('collectionId', Query::TYPE_EQUAL, [$collectionId])
|
||||||
|
], $dbForInternal);
|
||||||
|
|
||||||
|
$dbForExternal->deleteCollection($collectionId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param int $timestamp1d
|
* @param int $timestamp1d
|
||||||
* @param int $timestamp30m
|
* @param int $timestamp30m
|
||||||
|
|
|
@ -403,5 +403,13 @@ class DatabaseCustomServerTest extends Scope
|
||||||
|
|
||||||
$this->assertEquals(400, $tooMany['headers']['status-code']);
|
$this->assertEquals(400, $tooMany['headers']['status-code']);
|
||||||
$this->assertEquals('Index limit exceeded', $tooMany['body']['message']);
|
$this->assertEquals('Index limit exceeded', $tooMany['body']['message']);
|
||||||
|
|
||||||
|
$collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([
|
||||||
|
'content-type' => 'application/json',
|
||||||
|
'x-appwrite-project' => $this->getProject()['$id'],
|
||||||
|
'x-appwrite-key' => $this->getProject()['apiKey']
|
||||||
|
]));
|
||||||
|
|
||||||
|
$this->assertEquals(204, $collection['headers']['status-code']);
|
||||||
}
|
}
|
||||||
}
|
}
|
Loading…
Reference in a new issue