2020-05-27 05:11:20 +12:00
|
|
|
<?php
|
|
|
|
|
2021-11-25 17:59:38 +13:00
|
|
|
use Utopia\App;
|
2022-07-24 05:42:42 +12:00
|
|
|
use Utopia\Cache\Adapter\Filesystem;
|
|
|
|
use Utopia\Cache\Cache;
|
2021-07-14 07:24:52 +12:00
|
|
|
use Utopia\Database\Database;
|
|
|
|
use Utopia\Database\Document;
|
2021-07-13 09:57:37 +12:00
|
|
|
use Utopia\Database\Query;
|
2021-06-12 02:20:18 +12:00
|
|
|
use Appwrite\Resque\Worker;
|
2022-02-04 14:29:40 +13:00
|
|
|
use Executor\Executor;
|
2021-01-22 21:28:33 +13:00
|
|
|
use Utopia\Storage\Device\Local;
|
2020-12-19 03:08:28 +13:00
|
|
|
use Utopia\Abuse\Abuse;
|
|
|
|
use Utopia\Abuse\Adapters\TimeLimit;
|
2020-12-05 09:47:02 +13:00
|
|
|
use Utopia\CLI\Console;
|
2020-12-18 04:15:47 +13:00
|
|
|
use Utopia\Audit\Audit;
|
2020-05-27 05:11:20 +12:00
|
|
|
|
2021-11-20 05:46:10 +13:00
|
|
|
require_once __DIR__ . '/../init.php';
|
2020-12-19 21:08:03 +13:00
|
|
|
|
2021-01-15 19:02:48 +13:00
|
|
|
Console::title('Deletes V1 Worker');
|
2021-09-01 21:13:23 +12:00
|
|
|
Console::success(APP_NAME . ' deletes worker v1 has started' . "\n");
|
2020-12-19 21:08:03 +13:00
|
|
|
|
2021-06-12 02:20:18 +12:00
|
|
|
class DeletesV1 extends Worker
|
2020-05-27 05:11:20 +12:00
|
|
|
{
|
2022-05-16 21:58:17 +12:00
|
|
|
public function getName(): string
|
|
|
|
{
|
2021-11-24 03:24:25 +13:00
|
|
|
return "deletes";
|
|
|
|
}
|
|
|
|
|
2021-06-12 02:20:18 +12:00
|
|
|
public function init(): void
|
2020-05-27 05:11:20 +12:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2021-06-12 02:20:18 +12:00
|
|
|
public function run(): void
|
|
|
|
{
|
2022-04-14 00:39:31 +12:00
|
|
|
$project = new Document($this->args['project'] ?? []);
|
2021-06-12 22:07:26 +12:00
|
|
|
$type = $this->args['type'] ?? '';
|
2020-12-19 01:28:08 +13:00
|
|
|
switch (strval($type)) {
|
2020-12-19 03:05:15 +13:00
|
|
|
case DELETE_TYPE_DOCUMENT:
|
2021-11-20 05:46:10 +13:00
|
|
|
$document = new Document($this->args['document'] ?? []);
|
|
|
|
|
2021-06-12 22:07:26 +12:00
|
|
|
switch ($document->getCollection()) {
|
2022-06-22 22:51:49 +12:00
|
|
|
case DELETE_TYPE_DATABASES:
|
|
|
|
$this->deleteDatabase($document, $project->getId());
|
|
|
|
break;
|
2021-10-26 13:14:55 +13:00
|
|
|
case DELETE_TYPE_COLLECTIONS:
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteCollection($document, $project->getId());
|
2021-09-03 04:45:03 +12:00
|
|
|
break;
|
2021-10-26 13:14:55 +13:00
|
|
|
case DELETE_TYPE_PROJECTS:
|
2020-12-19 01:28:08 +13:00
|
|
|
$this->deleteProject($document);
|
|
|
|
break;
|
2021-10-26 13:14:55 +13:00
|
|
|
case DELETE_TYPE_FUNCTIONS:
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteFunction($document, $project->getId());
|
2020-12-19 01:28:08 +13:00
|
|
|
break;
|
2022-01-28 12:36:30 +13:00
|
|
|
case DELETE_TYPE_DEPLOYMENTS:
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteDeployment($document, $project->getId());
|
2022-01-28 12:36:30 +13:00
|
|
|
break;
|
2021-10-26 13:14:55 +13:00
|
|
|
case DELETE_TYPE_USERS:
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteUser($document, $project->getId());
|
2020-12-19 01:28:08 +13:00
|
|
|
break;
|
2021-10-26 13:14:55 +13:00
|
|
|
case DELETE_TYPE_TEAMS:
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteMemberships($document, $project->getId());
|
2021-05-12 05:55:38 +12:00
|
|
|
break;
|
2021-11-07 18:54:28 +13:00
|
|
|
case DELETE_TYPE_BUCKETS:
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteBucket($document, $project->getId());
|
2021-07-28 22:09:29 +12:00
|
|
|
break;
|
2020-12-19 01:28:08 +13:00
|
|
|
default:
|
2021-09-01 21:13:23 +12:00
|
|
|
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
|
2020-12-19 01:28:08 +13:00
|
|
|
break;
|
|
|
|
}
|
2020-12-15 10:26:37 +13:00
|
|
|
break;
|
2020-12-22 07:15:52 +13:00
|
|
|
|
2020-12-28 06:57:35 +13:00
|
|
|
case DELETE_TYPE_EXECUTIONS:
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteExecutionLogs($this->args['datetime']);
|
2020-12-22 07:15:52 +13:00
|
|
|
break;
|
|
|
|
|
2020-12-19 03:05:15 +13:00
|
|
|
case DELETE_TYPE_AUDIT:
|
2022-07-12 03:12:41 +12:00
|
|
|
$datetime = $this->args['datetime'] ?? null;
|
|
|
|
if (!empty($datetime)) {
|
|
|
|
$this->deleteAuditLogs($datetime);
|
2022-01-03 23:29:15 +13:00
|
|
|
}
|
|
|
|
|
2022-07-12 03:12:41 +12:00
|
|
|
$document = new Document($this->args['document'] ?? []);
|
|
|
|
|
2022-01-03 23:29:15 +13:00
|
|
|
if (!$document->isEmpty()) {
|
2022-04-14 00:39:31 +12:00
|
|
|
$this->deleteAuditLogsByResource('document/' . $document->getId(), $project->getId());
|
2022-01-03 23:29:15 +13:00
|
|
|
}
|
|
|
|
|
2020-12-05 09:47:02 +13:00
|
|
|
break;
|
2020-12-19 01:28:08 +13:00
|
|
|
|
2020-12-19 03:05:15 +13:00
|
|
|
case DELETE_TYPE_ABUSE:
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteAbuseLogs($this->args['datetime']);
|
2020-05-27 05:11:20 +12:00
|
|
|
break;
|
2021-02-05 23:57:43 +13:00
|
|
|
|
2021-06-16 21:35:37 +12:00
|
|
|
case DELETE_TYPE_REALTIME:
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteRealtimeUsage($this->args['datetime']);
|
2021-06-16 21:35:37 +12:00
|
|
|
break;
|
|
|
|
|
2022-06-02 22:20:03 +12:00
|
|
|
case DELETE_TYPE_SESSIONS:
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteExpiredSessions($this->args['datetime']);
|
2022-06-02 22:20:03 +12:00
|
|
|
break;
|
|
|
|
|
2021-02-05 23:57:43 +13:00
|
|
|
case DELETE_TYPE_CERTIFICATES:
|
|
|
|
$document = new Document($this->args['document']);
|
|
|
|
$this->deleteCertificates($document);
|
|
|
|
break;
|
2021-09-01 21:13:23 +12:00
|
|
|
|
2021-09-16 03:28:40 +12:00
|
|
|
case DELETE_TYPE_USAGE:
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteUsageStats($this->args['dateTime1d'], $this->args['dateTime30m']);
|
2021-09-06 21:39:36 +12:00
|
|
|
break;
|
2022-07-03 21:36:59 +12:00
|
|
|
|
2022-08-15 21:05:41 +12:00
|
|
|
case DELETE_TYPE_CACHE_BY_RESOURCE:
|
|
|
|
$this->deleteCacheByResource($project->getId());
|
|
|
|
break;
|
|
|
|
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
|
|
|
|
$this->deleteCacheByTimestamp();
|
2022-07-03 21:36:59 +12:00
|
|
|
break;
|
2020-12-19 21:08:03 +13:00
|
|
|
default:
|
2021-09-01 21:13:23 +12:00
|
|
|
Console::error('No delete operation for type: ' . $type);
|
2020-12-19 21:08:03 +13:00
|
|
|
break;
|
2021-09-01 21:13:23 +12:00
|
|
|
}
|
2020-05-27 05:11:20 +12:00
|
|
|
}
|
|
|
|
|
2021-06-12 02:20:18 +12:00
|
|
|
public function shutdown(): void
|
2020-05-27 05:11:20 +12:00
|
|
|
{
|
|
|
|
}
|
2021-09-01 21:13:23 +12:00
|
|
|
|
2022-07-03 21:36:59 +12:00
|
|
|
/**
|
2022-08-15 21:05:41 +12:00
|
|
|
* @param string $projectId
|
2022-07-03 21:36:59 +12:00
|
|
|
*/
|
2022-08-15 21:05:41 +12:00
|
|
|
protected function deleteCacheByResource(string $projectId): void
|
|
|
|
{
|
|
|
|
$this->deleteCacheFiles([
|
2022-08-26 02:56:15 +12:00
|
|
|
Query::equal('resource', [$this->args['resource']]),
|
|
|
|
]);
|
2022-08-15 21:05:41 +12:00
|
|
|
}
|
|
|
|
|
|
|
|
protected function deleteCacheByTimestamp(): void
|
|
|
|
{
|
|
|
|
$this->deleteCacheFiles([
|
2022-08-26 02:56:15 +12:00
|
|
|
Query::lessThan('accessedAt', $this->args['timestamp']),
|
|
|
|
]);
|
2022-08-15 21:05:41 +12:00
|
|
|
}
|
|
|
|
|
|
|
|
protected function deleteCacheFiles($query): void
|
2022-07-03 21:36:59 +12:00
|
|
|
{
|
2022-08-15 21:05:41 +12:00
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($query) {
|
2022-07-03 21:36:59 +12:00
|
|
|
|
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2022-07-24 05:42:42 +12:00
|
|
|
$cache = new Cache(
|
|
|
|
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
|
|
|
|
);
|
2022-07-03 21:36:59 +12:00
|
|
|
|
2022-07-05 19:37:46 +12:00
|
|
|
$this->deleteByGroup(
|
2022-07-24 19:12:15 +12:00
|
|
|
'cache',
|
2022-08-15 21:05:41 +12:00
|
|
|
$query,
|
2022-07-05 19:37:46 +12:00
|
|
|
$dbForProject,
|
2022-07-24 05:42:42 +12:00
|
|
|
function (Document $document) use ($cache, $projectId) {
|
2022-07-27 00:50:33 +12:00
|
|
|
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
|
2022-07-24 05:42:42 +12:00
|
|
|
|
|
|
|
if ($cache->purge($document->getId())) {
|
2022-07-03 21:36:59 +12:00
|
|
|
Console::success('Deleting cache file: ' . $path);
|
|
|
|
} else {
|
2022-07-10 06:24:28 +12:00
|
|
|
Console::error('Failed to delete cache file: ' . $path);
|
2022-07-03 21:36:59 +12:00
|
|
|
}
|
2022-07-05 19:37:46 +12:00
|
|
|
}
|
|
|
|
);
|
2022-07-03 21:36:59 +12:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2022-06-22 22:51:49 +12:00
|
|
|
/**
|
|
|
|
* @param Document $document database document
|
|
|
|
* @param string $projectId
|
|
|
|
*/
|
|
|
|
protected function deleteDatabase(Document $document, string $projectId): void
|
|
|
|
{
|
|
|
|
$databaseId = $document->getId();
|
|
|
|
|
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
|
|
|
|
|
|
|
$this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($projectId) {
|
|
|
|
$this->deleteCollection($document, $projectId);
|
|
|
|
});
|
|
|
|
|
|
|
|
$dbForProject->deleteCollection('database_' . $document->getInternalId());
|
|
|
|
|
|
|
|
$this->deleteAuditLogsByResource('database/' . $databaseId, $projectId);
|
|
|
|
}
|
|
|
|
|
2021-09-03 04:45:03 +12:00
|
|
|
/**
|
|
|
|
* @param Document $document teams document
|
|
|
|
* @param string $projectId
|
|
|
|
*/
|
|
|
|
protected function deleteCollection(Document $document, string $projectId): void
|
2020-12-05 09:47:02 +13:00
|
|
|
{
|
|
|
|
$collectionId = $document->getId();
|
2022-06-22 22:51:49 +12:00
|
|
|
$databaseId = str_replace('database_', '', $document->getCollection());
|
2021-09-01 21:13:23 +12:00
|
|
|
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2021-09-03 04:45:03 +12:00
|
|
|
|
2022-06-22 22:51:49 +12:00
|
|
|
$dbForProject->deleteCollection('database_' . $databaseId . '_collection_' . $document->getInternalId());
|
2021-12-16 09:16:11 +13:00
|
|
|
|
2021-09-03 04:45:03 +12:00
|
|
|
$this->deleteByGroup('attributes', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('collectionId', [$collectionId])
|
2021-12-28 01:45:23 +13:00
|
|
|
], $dbForProject);
|
2021-09-03 04:45:03 +12:00
|
|
|
|
|
|
|
$this->deleteByGroup('indexes', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('collectionId', [$collectionId])
|
2021-12-28 01:45:23 +13:00
|
|
|
], $dbForProject);
|
2022-01-03 23:29:15 +13:00
|
|
|
|
|
|
|
$this->deleteAuditLogsByResource('collection/' . $collectionId, $projectId);
|
2020-12-05 09:47:02 +13:00
|
|
|
}
|
2020-05-27 05:11:20 +12:00
|
|
|
|
2021-08-27 23:37:52 +12:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $datetime1d
|
|
|
|
* @param string $datetime30m
|
2021-08-27 23:37:52 +12:00
|
|
|
*/
|
2022-07-12 03:12:41 +12:00
|
|
|
protected function deleteUsageStats(string $datetime1d, string $datetime30m)
|
2021-09-01 21:13:23 +12:00
|
|
|
{
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime30m) {
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2021-08-27 23:37:52 +12:00
|
|
|
// Delete Usage stats
|
|
|
|
$this->deleteByGroup('stats', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::lessThan('time', $datetime1d),
|
|
|
|
Query::equal('period', ['1d']),
|
2021-12-28 01:45:23 +13:00
|
|
|
], $dbForProject);
|
2021-08-27 23:37:52 +12:00
|
|
|
|
|
|
|
$this->deleteByGroup('stats', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::lessThan('time', [$datetime30m]),
|
|
|
|
Query::equal('period', ['30m']),
|
2021-12-28 01:45:23 +13:00
|
|
|
], $dbForProject);
|
2021-08-27 23:37:52 +12:00
|
|
|
});
|
|
|
|
}
|
2021-11-20 05:46:10 +13:00
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
|
|
|
* @param Document $document teams document
|
|
|
|
* @param string $projectId
|
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteMemberships(Document $document, string $projectId): void
|
|
|
|
{
|
2021-07-14 04:08:52 +12:00
|
|
|
$teamId = $document->getAttribute('teamId', '');
|
|
|
|
|
2021-05-12 05:55:38 +12:00
|
|
|
// Delete Memberships
|
2021-07-14 07:24:52 +12:00
|
|
|
$this->deleteByGroup('memberships', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('teamId', [$teamId])
|
2021-05-12 05:55:38 +12:00
|
|
|
], $this->getProjectDB($projectId));
|
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
|
|
|
* @param Document $document project document
|
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteProject(Document $document): void
|
2020-05-27 05:11:20 +12:00
|
|
|
{
|
2021-07-14 06:44:45 +12:00
|
|
|
$projectId = $document->getId();
|
2021-11-20 05:46:10 +13:00
|
|
|
|
2020-05-27 05:11:20 +12:00
|
|
|
// Delete all DBs
|
2021-12-28 01:45:23 +13:00
|
|
|
$this->getProjectDB($projectId)->delete($projectId);
|
2021-07-14 06:44:45 +12:00
|
|
|
|
|
|
|
// Delete all storage directories
|
2021-09-01 21:13:23 +12:00
|
|
|
$uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId());
|
|
|
|
$cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId());
|
2020-05-27 05:11:20 +12:00
|
|
|
|
2020-05-27 07:12:40 +12:00
|
|
|
$uploads->delete($uploads->getRoot(), true);
|
|
|
|
$cache->delete($cache->getRoot(), true);
|
2020-05-27 05:11:20 +12:00
|
|
|
}
|
2020-07-03 10:42:21 +12:00
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
|
|
|
* @param Document $document user document
|
|
|
|
* @param string $projectId
|
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteUser(Document $document, string $projectId): void
|
2020-07-03 10:42:21 +12:00
|
|
|
{
|
2021-07-13 09:57:37 +12:00
|
|
|
$userId = $document->getId();
|
2021-02-20 02:59:36 +13:00
|
|
|
|
2022-01-16 14:02:13 +13:00
|
|
|
// Delete all sessions of this user from the sessions table and update the sessions field of the user record
|
2022-01-16 12:24:38 +13:00
|
|
|
$this->deleteByGroup('sessions', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('userId', [$userId])
|
2022-01-16 12:24:38 +13:00
|
|
|
], $this->getProjectDB($projectId));
|
2022-04-14 00:39:31 +12:00
|
|
|
|
2022-04-26 20:52:59 +12:00
|
|
|
$this->getProjectDB($projectId)->deleteCachedDocument('users', $userId);
|
2022-01-16 12:24:38 +13:00
|
|
|
|
2021-05-29 08:55:22 +12:00
|
|
|
// Delete Memberships and decrement team membership counts
|
2021-07-14 07:24:52 +12:00
|
|
|
$this->deleteByGroup('memberships', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('userId', [$userId])
|
2021-09-01 21:13:23 +12:00
|
|
|
], $this->getProjectDB($projectId), function (Document $document) use ($projectId) {
|
2021-05-29 08:21:56 +12:00
|
|
|
|
|
|
|
if ($document->getAttribute('confirm')) { // Count only confirmed members
|
|
|
|
$teamId = $document->getAttribute('teamId');
|
2021-12-28 01:45:23 +13:00
|
|
|
$team = $this->getProjectDB($projectId)->getDocument('teams', $teamId);
|
2021-09-01 21:13:23 +12:00
|
|
|
if (!$team->isEmpty()) {
|
2022-05-16 21:58:17 +12:00
|
|
|
$team = $this
|
|
|
|
->getProjectDB($projectId)
|
|
|
|
->updateDocument(
|
|
|
|
'teams',
|
|
|
|
$teamId,
|
|
|
|
// Ensure that total >= 0
|
|
|
|
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
|
|
|
|
);
|
2021-05-29 08:55:22 +12:00
|
|
|
}
|
2021-05-29 08:21:56 +12:00
|
|
|
}
|
|
|
|
});
|
2022-04-27 23:06:53 +12:00
|
|
|
|
|
|
|
// Delete tokens
|
|
|
|
$this->deleteByGroup('tokens', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('userId', [$userId])
|
2022-04-27 23:06:53 +12:00
|
|
|
], $this->getProjectDB($projectId));
|
2020-12-05 09:47:02 +13:00
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $datetime
|
2021-07-14 07:24:52 +12:00
|
|
|
*/
|
2022-07-12 03:12:41 +12:00
|
|
|
protected function deleteExecutionLogs(string $datetime): void
|
2020-12-15 10:26:37 +13:00
|
|
|
{
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2020-12-15 10:26:37 +13:00
|
|
|
// Delete Executions
|
2021-07-14 07:24:52 +12:00
|
|
|
$this->deleteByGroup('executions', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::lessThan('$createdAt', $datetime)
|
2021-12-28 01:45:23 +13:00
|
|
|
], $dbForProject);
|
2020-12-23 03:04:09 +13:00
|
|
|
});
|
2020-12-15 10:26:37 +13:00
|
|
|
}
|
|
|
|
|
2022-06-02 22:20:03 +12:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $datetime
|
2022-06-02 22:20:03 +12:00
|
|
|
*/
|
2022-07-12 03:12:41 +12:00
|
|
|
protected function deleteExpiredSessions(string $datetime): void
|
2022-06-02 22:20:03 +12:00
|
|
|
{
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
|
2022-06-02 22:20:03 +12:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
|
|
|
// Delete Sessions
|
|
|
|
$this->deleteByGroup('sessions', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::lessThan('expire', $datetime)
|
2022-06-02 22:20:03 +12:00
|
|
|
], $dbForProject);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2021-11-20 05:46:10 +13:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $datetime
|
2021-11-20 05:46:10 +13:00
|
|
|
*/
|
2022-07-12 03:12:41 +12:00
|
|
|
protected function deleteRealtimeUsage(string $datetime): void
|
2021-11-20 05:46:10 +13:00
|
|
|
{
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2021-11-20 05:46:10 +13:00
|
|
|
// Delete Dead Realtime Logs
|
|
|
|
$this->deleteByGroup('realtime', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::lessThan('timestamp', $datetime)
|
2021-12-28 01:45:23 +13:00
|
|
|
], $dbForProject);
|
2021-11-20 05:46:10 +13:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $datetime
|
|
|
|
* @throws Exception
|
2021-07-14 07:24:52 +12:00
|
|
|
*/
|
2022-07-12 03:12:41 +12:00
|
|
|
protected function deleteAbuseLogs(string $datetime): void
|
2020-12-18 04:15:47 +13:00
|
|
|
{
|
2022-07-12 03:12:41 +12:00
|
|
|
if (empty($datetime)) {
|
|
|
|
throw new Exception('Failed to delete audit logs. No datetime provided');
|
2020-12-19 03:05:15 +13:00
|
|
|
}
|
2020-12-18 04:15:47 +13:00
|
|
|
|
2022-07-12 03:12:41 +12:00
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
|
|
|
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
|
2021-09-01 21:13:23 +12:00
|
|
|
$abuse = new Abuse($timeLimit);
|
2022-07-20 02:52:06 +12:00
|
|
|
$status = $abuse->cleanup($datetime);
|
2020-12-18 04:15:47 +13:00
|
|
|
if (!$status) {
|
2021-09-01 21:13:23 +12:00
|
|
|
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
|
2020-12-18 04:15:47 +13:00
|
|
|
}
|
2020-12-23 03:04:09 +13:00
|
|
|
});
|
2020-12-18 04:15:47 +13:00
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $datetime
|
|
|
|
* @throws Exception
|
2021-07-14 07:24:52 +12:00
|
|
|
*/
|
2022-07-12 03:12:41 +12:00
|
|
|
protected function deleteAuditLogs(string $datetime): void
|
2020-12-18 04:15:47 +13:00
|
|
|
{
|
2022-07-12 03:12:41 +12:00
|
|
|
if (empty($datetime)) {
|
|
|
|
throw new Exception('Failed to delete audit logs. No datetime provided');
|
2020-12-19 03:05:15 +13:00
|
|
|
}
|
2022-07-12 03:12:41 +12:00
|
|
|
|
|
|
|
$this->deleteForProjectIds(function (string $projectId) use ($datetime) {
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
|
|
|
$audit = new Audit($dbForProject);
|
2022-07-20 02:52:06 +12:00
|
|
|
$status = $audit->cleanup($datetime);
|
2020-12-18 04:15:47 +13:00
|
|
|
if (!$status) {
|
2021-09-01 21:13:23 +12:00
|
|
|
throw new Exception('Failed to delete Audit logs for project' . $projectId);
|
2020-12-18 04:15:47 +13:00
|
|
|
}
|
2020-12-23 03:04:09 +13:00
|
|
|
});
|
2020-12-18 04:15:47 +13:00
|
|
|
}
|
|
|
|
|
2022-01-03 23:29:15 +13:00
|
|
|
/**
|
2022-07-12 03:12:41 +12:00
|
|
|
* @param string $resource
|
|
|
|
* @param string $projectId
|
2022-01-03 23:29:15 +13:00
|
|
|
*/
|
|
|
|
protected function deleteAuditLogsByResource(string $resource, string $projectId): void
|
|
|
|
{
|
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
|
|
|
|
|
|
|
$this->deleteByGroup(Audit::COLLECTION, [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('resource', [$resource])
|
2022-01-03 23:29:15 +13:00
|
|
|
], $dbForProject);
|
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
|
|
|
* @param Document $document function document
|
|
|
|
* @param string $projectId
|
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteFunction(Document $document, string $projectId): void
|
2020-12-05 09:47:02 +13:00
|
|
|
{
|
2021-12-28 01:45:23 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2022-02-13 11:34:16 +13:00
|
|
|
$functionId = $document->getId();
|
2022-01-28 13:25:28 +13:00
|
|
|
|
2022-01-28 14:21:11 +13:00
|
|
|
/**
|
|
|
|
* Delete Deployments
|
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Deleting deployments for function " . $functionId);
|
2022-01-28 14:21:11 +13:00
|
|
|
$storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
|
2022-01-28 14:29:20 +13:00
|
|
|
$deploymentIds = [];
|
|
|
|
$this->deleteByGroup('deployments', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('resourceId', [$functionId])
|
2022-01-28 14:29:20 +13:00
|
|
|
], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) {
|
|
|
|
$deploymentIds[] = $document->getId();
|
|
|
|
if ($storageFunctions->delete($document->getAttribute('path', ''), true)) {
|
2022-02-01 12:44:55 +13:00
|
|
|
Console::success('Deleted deployment files: ' . $document->getAttribute('path', ''));
|
2022-01-28 14:29:20 +13:00
|
|
|
} else {
|
|
|
|
Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', ''));
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Delete builds
|
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Deleting builds for function " . $functionId);
|
2022-02-03 10:36:21 +13:00
|
|
|
$storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId);
|
2022-05-16 21:58:17 +12:00
|
|
|
foreach ($deploymentIds as $deploymentId) {
|
2022-01-29 13:52:06 +13:00
|
|
|
$this->deleteByGroup('builds', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('deploymentId', [$deploymentId])
|
2022-02-15 13:58:12 +13:00
|
|
|
], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) {
|
2022-01-29 13:52:06 +13:00
|
|
|
if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
|
|
|
|
Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
|
|
|
|
} else {
|
|
|
|
Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
2022-01-28 14:29:20 +13:00
|
|
|
|
2022-05-24 02:54:50 +12:00
|
|
|
/**
|
2022-02-13 11:34:16 +13:00
|
|
|
* Delete Executions
|
2022-05-16 21:58:17 +12:00
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Deleting executions for function " . $functionId);
|
2022-01-28 14:29:20 +13:00
|
|
|
$this->deleteByGroup('executions', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('functionId', [$functionId])
|
2022-01-28 14:29:20 +13:00
|
|
|
], $dbForProject);
|
|
|
|
|
2022-02-03 10:36:21 +13:00
|
|
|
/**
|
|
|
|
* Request executor to delete all deployment containers
|
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Requesting executor to delete all deployment containers for function " . $functionId);
|
2022-04-14 04:15:25 +12:00
|
|
|
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
|
2022-02-03 10:36:21 +13:00
|
|
|
foreach ($deploymentIds as $deploymentId) {
|
|
|
|
try {
|
2022-02-25 03:18:46 +13:00
|
|
|
$executor->deleteRuntime($projectId, $deploymentId);
|
2022-02-03 10:36:21 +13:00
|
|
|
} catch (Throwable $th) {
|
|
|
|
Console::error($th->getMessage());
|
|
|
|
}
|
|
|
|
}
|
2022-01-28 14:29:20 +13:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @param Document $document deployment document
|
|
|
|
* @param string $projectId
|
|
|
|
*/
|
|
|
|
protected function deleteDeployment(Document $document, string $projectId): void
|
|
|
|
{
|
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2022-02-06 23:00:44 +13:00
|
|
|
$deploymentId = $document->getId();
|
2022-02-13 11:34:16 +13:00
|
|
|
$functionId = $document->getAttribute('resourceId');
|
2022-01-28 14:29:20 +13:00
|
|
|
|
|
|
|
/**
|
2022-01-29 13:00:25 +13:00
|
|
|
* Delete deployment files
|
2022-01-28 14:29:20 +13:00
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Deleting deployment files for deployment " . $deploymentId);
|
2022-01-28 14:29:20 +13:00
|
|
|
$storageFunctions = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
|
2022-01-28 14:32:56 +13:00
|
|
|
if ($storageFunctions->delete($document->getAttribute('path', ''), true)) {
|
2022-02-01 12:44:55 +13:00
|
|
|
Console::success('Deleted deployment files: ' . $document->getAttribute('path', ''));
|
2022-01-28 14:32:56 +13:00
|
|
|
} else {
|
|
|
|
Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', ''));
|
|
|
|
}
|
2020-12-05 09:47:02 +13:00
|
|
|
|
2022-01-28 14:21:11 +13:00
|
|
|
/**
|
|
|
|
* Delete builds
|
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Deleting builds for deployment " . $deploymentId);
|
2022-01-28 14:21:11 +13:00
|
|
|
$storageBuilds = new Local(APP_STORAGE_BUILDS . '/app-' . $projectId);
|
2022-01-28 12:36:30 +13:00
|
|
|
$this->deleteByGroup('builds', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('deploymentId', [$deploymentId])
|
2022-02-15 13:58:12 +13:00
|
|
|
], $dbForProject, function (Document $document) use ($storageBuilds) {
|
2022-01-28 13:17:59 +13:00
|
|
|
if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
|
2022-01-28 13:02:44 +13:00
|
|
|
Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
|
2021-09-01 21:13:23 +12:00
|
|
|
} else {
|
2022-01-28 13:02:44 +13:00
|
|
|
Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
|
2020-12-05 09:47:02 +13:00
|
|
|
}
|
|
|
|
});
|
|
|
|
|
2022-02-03 10:36:21 +13:00
|
|
|
/**
|
|
|
|
* Request executor to delete the deployment container
|
|
|
|
*/
|
2022-02-13 11:34:16 +13:00
|
|
|
Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId);
|
2022-02-03 10:36:21 +13:00
|
|
|
try {
|
2022-04-14 04:15:25 +12:00
|
|
|
$executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
|
2022-02-25 03:18:46 +13:00
|
|
|
$executor->deleteRuntime($projectId, $deploymentId);
|
2022-02-03 10:36:21 +13:00
|
|
|
} catch (Throwable $th) {
|
|
|
|
Console::error($th->getMessage());
|
|
|
|
}
|
2020-12-05 09:47:02 +13:00
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
|
|
|
|
/**
|
|
|
|
* @param Document $document to be deleted
|
|
|
|
* @param Database $database to delete it from
|
|
|
|
* @param callable $callback to perform after document is deleted
|
|
|
|
*
|
|
|
|
* @return bool
|
|
|
|
*/
|
2020-12-05 09:47:02 +13:00
|
|
|
protected function deleteById(Document $document, Database $database, callable $callback = null): bool
|
|
|
|
{
|
2021-11-20 05:46:10 +13:00
|
|
|
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
|
2021-09-01 21:13:23 +12:00
|
|
|
Console::success('Deleted document "' . $document->getId() . '" successfully');
|
2020-12-05 09:47:02 +13:00
|
|
|
|
2021-09-01 21:13:23 +12:00
|
|
|
if (is_callable($callback)) {
|
2020-12-05 09:47:02 +13:00
|
|
|
$callback($document);
|
2020-07-03 10:42:21 +12:00
|
|
|
}
|
2020-12-05 09:47:02 +13:00
|
|
|
|
|
|
|
return true;
|
2021-09-01 21:13:23 +12:00
|
|
|
} else {
|
|
|
|
Console::error('Failed to delete document: ' . $document->getId());
|
2020-12-05 09:47:02 +13:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
|
|
|
* @param callable $callback
|
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteForProjectIds(callable $callback): void
|
2020-12-22 07:15:52 +13:00
|
|
|
{
|
2020-12-23 03:04:09 +13:00
|
|
|
$count = 0;
|
|
|
|
$chunk = 0;
|
|
|
|
$limit = 50;
|
|
|
|
$projects = [];
|
|
|
|
$sum = $limit;
|
2020-12-22 07:15:52 +13:00
|
|
|
|
2020-12-23 03:04:09 +13:00
|
|
|
$executionStart = \microtime(true);
|
2021-09-01 21:13:23 +12:00
|
|
|
|
|
|
|
while ($sum === $limit) {
|
2022-08-12 11:53:52 +12:00
|
|
|
$projects = $this->getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
|
2020-12-23 03:04:09 +13:00
|
|
|
|
2021-08-27 16:37:15 +12:00
|
|
|
$chunk++;
|
|
|
|
|
2021-11-20 05:46:10 +13:00
|
|
|
/** @var string[] $projectIds */
|
2022-05-16 21:58:17 +12:00
|
|
|
$projectIds = array_map(fn (Document $project) => $project->getId(), $projects);
|
2020-12-23 03:04:09 +13:00
|
|
|
|
|
|
|
$sum = count($projects);
|
|
|
|
|
2021-09-01 21:13:23 +12:00
|
|
|
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
|
2020-12-23 03:04:09 +13:00
|
|
|
foreach ($projectIds as $projectId) {
|
|
|
|
$callback($projectId);
|
|
|
|
$count++;
|
|
|
|
}
|
|
|
|
}
|
2020-12-22 07:15:52 +13:00
|
|
|
|
2020-12-23 03:04:09 +13:00
|
|
|
$executionEnd = \microtime(true);
|
|
|
|
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
|
2020-12-22 07:15:52 +13:00
|
|
|
}
|
|
|
|
|
2021-07-13 09:57:37 +12:00
|
|
|
/**
|
|
|
|
* @param string $collection collectionID
|
|
|
|
* @param Query[] $queries
|
2021-07-14 07:24:52 +12:00
|
|
|
* @param Database $database
|
2021-07-13 09:57:37 +12:00
|
|
|
* @param callable $callback
|
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
|
2020-12-05 09:47:02 +13:00
|
|
|
{
|
|
|
|
$count = 0;
|
|
|
|
$chunk = 0;
|
|
|
|
$limit = 50;
|
|
|
|
$results = [];
|
|
|
|
$sum = $limit;
|
|
|
|
|
|
|
|
$executionStart = \microtime(true);
|
2021-09-01 21:13:23 +12:00
|
|
|
|
|
|
|
while ($sum === $limit) {
|
2020-12-05 09:47:02 +13:00
|
|
|
$chunk++;
|
|
|
|
|
2022-08-12 11:53:52 +12:00
|
|
|
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
|
2022-01-29 13:52:06 +13:00
|
|
|
|
2020-12-05 09:47:02 +13:00
|
|
|
$sum = count($results);
|
|
|
|
|
2021-09-01 21:13:23 +12:00
|
|
|
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
|
2020-12-05 09:47:02 +13:00
|
|
|
|
|
|
|
foreach ($results as $document) {
|
|
|
|
$this->deleteById($document, $database, $callback);
|
|
|
|
$count++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
$executionEnd = \microtime(true);
|
|
|
|
|
|
|
|
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
|
|
|
|
}
|
|
|
|
|
2021-07-14 07:24:52 +12:00
|
|
|
/**
|
2022-05-24 02:54:50 +12:00
|
|
|
* @param Document $document certificates document
|
2021-07-14 07:24:52 +12:00
|
|
|
*/
|
2021-08-27 16:37:15 +12:00
|
|
|
protected function deleteCertificates(Document $document): void
|
2021-02-05 22:05:26 +13:00
|
|
|
{
|
2022-03-24 04:05:50 +13:00
|
|
|
$consoleDB = $this->getConsoleDB();
|
|
|
|
|
|
|
|
// If domain has certificate generated
|
2022-05-16 21:58:17 +12:00
|
|
|
if (isset($document['certificateId'])) {
|
2022-03-24 04:05:50 +13:00
|
|
|
$domainUsingCertificate = $consoleDB->findOne('domains', [
|
2022-08-12 11:53:52 +12:00
|
|
|
Query::equal('certificateId', [$document['certificateId']])
|
2022-03-24 04:05:50 +13:00
|
|
|
]);
|
|
|
|
|
2022-05-16 21:58:17 +12:00
|
|
|
if (!$domainUsingCertificate) {
|
2022-03-31 02:31:46 +13:00
|
|
|
$mainDomain = App::getEnv('_APP_DOMAIN_TARGET', '');
|
2022-05-16 21:58:17 +12:00
|
|
|
if ($mainDomain === $document->getAttribute('domain')) {
|
2022-03-31 02:31:46 +13:00
|
|
|
$domainUsingCertificate = $mainDomain;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-24 04:05:50 +13:00
|
|
|
// If certificate is still used by some domain, mark we can't delete.
|
|
|
|
// Current domain should not be found, because we only have copy. Original domain is already deleted from database.
|
2022-05-16 21:58:17 +12:00
|
|
|
if ($domainUsingCertificate) {
|
2022-03-24 04:05:50 +13:00
|
|
|
Console::warning("Skipping certificate deletion, because a domain is still using it.");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-05 23:57:43 +13:00
|
|
|
$domain = $document->getAttribute('domain');
|
2021-02-05 22:05:26 +13:00
|
|
|
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
|
2021-02-06 00:18:12 +13:00
|
|
|
$checkTraversal = realpath($directory) === $directory;
|
2021-02-05 22:05:26 +13:00
|
|
|
|
2021-09-01 21:13:23 +12:00
|
|
|
if ($domain && $checkTraversal && is_dir($directory)) {
|
2022-03-24 03:54:25 +13:00
|
|
|
// Delete certificate document, so Appwrite is aware of change
|
2022-05-16 21:58:17 +12:00
|
|
|
if (isset($document['certificateId'])) {
|
2022-03-24 03:54:25 +13:00
|
|
|
$consoleDB->deleteDocument('certificates', $document['certificateId']);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Delete files, so Traefik is aware of change
|
2021-09-01 21:13:23 +12:00
|
|
|
array_map('unlink', glob($directory . '/*.*'));
|
2021-02-05 22:05:26 +13:00
|
|
|
rmdir($directory);
|
2021-02-05 23:57:43 +13:00
|
|
|
Console::info("Deleted certificate files for {$domain}");
|
|
|
|
} else {
|
|
|
|
Console::info("No certificate files found for {$domain}");
|
2021-02-05 22:05:26 +13:00
|
|
|
}
|
|
|
|
}
|
2021-07-18 19:03:48 +12:00
|
|
|
|
|
|
|
protected function deleteBucket(Document $document, string $projectId)
|
|
|
|
{
|
2022-01-27 01:54:18 +13:00
|
|
|
$dbForProject = $this->getProjectDB($projectId);
|
2022-02-18 14:36:25 +13:00
|
|
|
$dbForProject->deleteCollection('bucket_' . $document->getInternalId());
|
2021-07-28 22:14:47 +12:00
|
|
|
|
2022-05-24 02:54:50 +12:00
|
|
|
$device = $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId);
|
|
|
|
|
2022-02-18 14:36:25 +13:00
|
|
|
$device->deletePath($document->getId());
|
2021-07-18 19:03:48 +12:00
|
|
|
}
|
2021-09-01 21:13:23 +12:00
|
|
|
}
|