1
0
Fork 0
mirror of synced 2024-08-31 01:41:15 +12:00
appwrite/app/workers/deletes.php

731 lines
23 KiB
PHP

<?php
require_once __DIR__ . '/../worker.php';
use Appwrite\Auth\Auth;
use Utopia\App;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Executor\Executor;
use Utopia\Storage\Device\Local;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\CLI\Console;
use Utopia\Audit\Audit;
use Utopia\Database\DateTime;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Message;
Authorization::disable();
Authorization::setDefaultStatus(false);
$server->job()
->inject('message')
->action(function (Message $message) {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
$project = new Document($payload['project'] ?? []);
$type = $payload['type'] ?? '';
switch (strval($type)) {
case DELETE_TYPE_DOCUMENT:
$document = new Document($payload['document'] ?? []);
switch ($document->getCollection()) {
case DELETE_TYPE_DATABASES:
deleteDatabase($document, $project);
break;
case DELETE_TYPE_COLLECTIONS:
deleteCollection($document, $project);
break;
case DELETE_TYPE_PROJECTS:
deleteProject($document);
break;
case DELETE_TYPE_FUNCTIONS:
deleteFunction($document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
deleteDeployment($document, $project);
break;
case DELETE_TYPE_USERS:
deleteUser($document, $project);
break;
case DELETE_TYPE_TEAMS:
deleteMemberships($document, $project);
break;
case DELETE_TYPE_BUCKETS:
deleteBucket($document, $project);
break;
default:
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
break;
}
break;
case DELETE_TYPE_EXECUTIONS:
deleteExecutionLogs($payload['datetime']);
break;
case DELETE_TYPE_AUDIT:
$datetime = $payload['datetime'] ?? null;
if (!empty($datetime)) {
deleteAuditLogs($datetime);
}
$document = new Document($payload['document'] ?? []);
if (!$document->isEmpty()) {
deleteAuditLogsByResource('document/' . $document->getId(), $project);
}
break;
case DELETE_TYPE_ABUSE:
deleteAbuseLogs($payload['datetime']);
break;
case DELETE_TYPE_REALTIME:
deleteRealtimeUsage($payload['datetime']);
break;
case DELETE_TYPE_SESSIONS:
deleteExpiredSessions();
break;
case DELETE_TYPE_CERTIFICATES:
$document = new Document($payload['document']);
deleteCertificates($document);
break;
case DELETE_TYPE_USAGE:
deleteUsageStats($payload['hourlyUsageRetentionDatetime']);
break;
case DELETE_TYPE_CACHE_BY_RESOURCE:
deleteCacheByResource($payload['resource']);
break;
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
deleteCacheByDate($payload);
break;
case DELETE_TYPE_SCHEDULES:
deleteSchedules($payload['datetime']);
break;
default:
Console::error('No delete operation for type: ' . $type);
break;
}
});
/**
* @throws Exception
*/
function deleteSchedules(string $datetime): void
{
listByGroup(
'schedules',
[
Query::equal('region', [App::getEnv('_APP_REGION', 'default')]),
Query::equal('resourceType', ['function']),
Query::lessThanEqual('resourceUpdatedAt', $datetime),
Query::equal('active', [false]),
],
getConsoleDB(),
function (Document $document) {
$project = getConsoleDB()->getDocument('projects', $document->getAttribute('projectId'));
if ($project->isEmpty()) {
Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId'));
return;
}
$function = getProjectDB($project)->getDocument('functions', $document->getAttribute('resourceId'));
if ($function->isEmpty()) {
getConsoleDB()->deleteDocument('schedules', $document->getId());
Console::success('Deleting schedule for function ' . $document->getAttribute('resourceId'));
}
}
);
}
/**
* @param string $resource
*/
function deleteCacheByResource(string $resource): void
{
deleteCacheFiles([
Query::equal('resource', [$resource]),
]);
}
function deleteCacheByDate($payload): void
{
deleteCacheFiles([
Query::lessThan('accessedAt', $payload['datetime']),
]);
}
function deleteCacheFiles($query): void
{
deleteForProjectIds(function (Document $project) use ($query) {
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
$cache = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId)
);
deleteByGroup(
'cache',
$query,
$dbForProject,
function (Document $document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
if ($cache->purge($document->getId())) {
Console::success('Deleting cache file: ' . $path);
} else {
Console::error('Failed to delete cache file: ' . $path);
}
}
);
});
}
/**
* @param Document $document database document
* @param Document $projectId
*/
function deleteDatabase(Document $document, Document $project): void
{
$databaseId = $document->getId();
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) {
deleteCollection($document, $project);
});
$dbForProject->deleteCollection('database_' . $document->getInternalId());
deleteAuditLogsByResource('database/' . $databaseId, $project);
}
/**
* @param Document $document teams document
* @param Document $project
*/
function deleteCollection(Document $document, Document $project): void
{
$collectionId = $document->getId();
$databaseId = $document->getAttribute('databaseId');
$databaseInternalId = $document->getAttribute('databaseInternalId');
$dbForProject = getProjectDB($project);
$dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId());
deleteByGroup('attributes', [
Query::equal('databaseId', [$databaseId]),
Query::equal('collectionId', [$collectionId])
], $dbForProject);
deleteByGroup('indexes', [
Query::equal('databaseId', [$databaseId]),
Query::equal('collectionId', [$collectionId])
], $dbForProject);
deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project);
}
/**
* @param string $hourlyUsageRetentionDatetime
*/
function deleteUsageStats(string $hourlyUsageRetentionDatetime)
{
deleteForProjectIds(function (Document $project) use ($hourlyUsageRetentionDatetime) {
$dbForProject = getProjectDB($project);
// Delete Usage stats
deleteByGroup('stats', [
Query::lessThan('time', $hourlyUsageRetentionDatetime),
Query::equal('period', ['1h']),
], $dbForProject);
});
}
/**
* @param Document $document teams document
* @param Document $project
*/
function deleteMemberships(Document $document, Document $project): void
{
$teamId = $document->getAttribute('teamId', '');
// Delete Memberships
deleteByGroup('memberships', [
Query::equal('teamId', [$teamId])
], getProjectDB($project));
}
/**
* @param Document $document project document
*/
function deleteProject(Document $document): void
{
$projectId = $document->getId();
// Delete all DBs
getProjectDB($document)->delete($projectId);
// Delete all storage directories
$uploads = getFilesDevice($document->getId());
$cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId());
$uploads->delete($uploads->getRoot(), true);
$cache->delete($cache->getRoot(), true);
}
/**
* @param Document $document user document
* @param Document $project
*/
function deleteUser(Document $document, Document $project): void
{
$userId = $document->getId();
$dbForProject = getProjectDB($project);
// Delete all sessions of this user from the sessions table and update the sessions field of the user record
deleteByGroup('sessions', [
Query::equal('userId', [$userId])
], $dbForProject);
$dbForProject->deleteCachedDocument('users', $userId);
// Delete Memberships and decrement team membership counts
deleteByGroup('memberships', [
Query::equal('userId', [$userId])
], $dbForProject, function (Document $document) use ($dbForProject) {
if ($document->getAttribute('confirm')) { // Count only confirmed members
$teamId = $document->getAttribute('teamId');
$team = $dbForProject->getDocument('teams', $teamId);
if (!$team->isEmpty()) {
$team = $dbForProject->updateDocument(
'teams',
$teamId,
// Ensure that total >= 0
$team->setAttribute('total', \max($team->getAttribute('total', 0) - 1, 0))
);
}
}
});
// Delete tokens
deleteByGroup('tokens', [
Query::equal('userId', [$userId])
], $dbForProject);
}
/**
* @param string $datetime
*/
function deleteExecutionLogs(string $datetime): void
{
deleteForProjectIds(function (Document $project) use ($datetime) {
$dbForProject = getProjectDB($project);
// Delete Executions
deleteByGroup('executions', [
Query::lessThan('$createdAt', $datetime)
], $dbForProject);
});
}
function deleteExpiredSessions(): void
{
$consoleDB = getConsoleDB();
deleteForProjectIds(function (Document $project) use ($consoleDB) {
$dbForProject = getProjectDB($project);
$project = $consoleDB->getDocument('projects', $project->getId());
$duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
$expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
// Delete Sessions
deleteByGroup('sessions', [
Query::lessThan('$createdAt', $expired)
], $dbForProject);
});
}
/**
* @param string $datetime
*/
function deleteRealtimeUsage(string $datetime): void
{
deleteForProjectIds(function (Document $project) use ($datetime) {
$dbForProject = getProjectDB($project);
// Delete Dead Realtime Logs
deleteByGroup('realtime', [
Query::lessThan('timestamp', $datetime)
], $dbForProject);
});
}
/**
* @param string $datetime
* @throws Exception
*/
function deleteAbuseLogs(string $datetime): void
{
if (empty($datetime)) {
throw new Exception('Failed to delete audit logs. No datetime provided');
}
deleteForProjectIds(function (Document $project) use ($datetime) {
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
$timeLimit = new TimeLimit("", 0, 1, $dbForProject);
$abuse = new Abuse($timeLimit);
$status = $abuse->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
}
});
}
/**
* @param string $datetime
* @throws Exception
*/
function deleteAuditLogs(string $datetime): void
{
if (empty($datetime)) {
throw new Exception('Failed to delete audit logs. No datetime provided');
}
deleteForProjectIds(function (Document $project) use ($datetime) {
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
$audit = new Audit($dbForProject);
$status = $audit->cleanup($datetime);
if (!$status) {
throw new Exception('Failed to delete Audit logs for project' . $projectId);
}
});
}
/**
* @param string $resource
* @param Document $project
*/
function deleteAuditLogsByResource(string $resource, Document $project): void
{
$dbForProject = getProjectDB($project);
deleteByGroup(Audit::COLLECTION, [
Query::equal('resource', [$resource])
], $dbForProject);
}
/**
* @param Document $document function document
* @param Document $project
*/
function deleteFunction(Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
$functionId = $document->getId();
/**
* Delete Variables
*/
Console::info("Deleting variables for function " . $functionId);
deleteByGroup('variables', [
Query::equal('functionId', [$functionId])
], $dbForProject);
/**
* Delete Deployments
*/
Console::info("Deleting deployments for function " . $functionId);
$storageFunctions = getFunctionsDevice($projectId);
$deploymentIds = [];
deleteByGroup('deployments', [
Query::equal('resourceId', [$functionId])
], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) {
$deploymentIds[] = $document->getId();
if ($storageFunctions->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted deployment files: ' . $document->getAttribute('path', ''));
} else {
Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', ''));
}
});
/**
* Delete builds
*/
Console::info("Deleting builds for function " . $functionId);
$storageBuilds = getBuildsDevice($projectId);
foreach ($deploymentIds as $deploymentId) {
deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) {
if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
} else {
Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
}
});
}
/**
* Delete Executions
*/
Console::info("Deleting executions for function " . $functionId);
deleteByGroup('executions', [
Query::equal('functionId', [$functionId])
], $dbForProject);
// TODO: Request executor to delete runtime
}
/**
* @param Document $document deployment document
* @param Document $project
*/
function deleteDeployment(Document $document, Document $project): void
{
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
$deploymentId = $document->getId();
$functionId = $document->getAttribute('resourceId');
/**
* Delete deployment files
*/
Console::info("Deleting deployment files for deployment " . $deploymentId);
$storageFunctions = getFunctionsDevice($projectId);
if ($storageFunctions->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted deployment files: ' . $document->getAttribute('path', ''));
} else {
Console::error('Failed to delete deployment files: ' . $document->getAttribute('path', ''));
}
/**
* Delete builds
*/
Console::info("Deleting builds for deployment " . $deploymentId);
$storageBuilds = getBuildsDevice($projectId);
deleteByGroup('builds', [
Query::equal('deploymentId', [$deploymentId])
], $dbForProject, function (Document $document) use ($storageBuilds) {
if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
} else {
Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
}
});
// TODO: Request executor to delete runtime
}
/**
* @param Document $document to be deleted
* @param Database $database to delete it from
* @param callable $callback to perform after document is deleted
*
* @return bool
*/
function deleteById(Document $document, Database $database, callable $callback = null): bool
{
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
Console::success('Deleted document "' . $document->getId() . '" successfully');
if (is_callable($callback)) {
$callback($document);
}
return true;
} else {
Console::error('Failed to delete document: ' . $document->getId());
return false;
}
}
/**
* @param callable $callback
*/
function deleteForProjectIds(callable $callback): void
{
// TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
$count = 0;
$chunk = 0;
$limit = 50;
$projects = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$projects = getConsoleDB()->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
$chunk++;
/** @var string[] $projectIds */
$sum = count($projects);
Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
foreach ($projects as $project) {
$callback($project);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param string $collection collectionID
* @param Query[] $queries
* @param Database $database
* @param callable $callback
*/
function deleteByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
$sum = count($results);
Console::info('Deleting chunk #' . $chunk . '. Found ' . $sum . ' documents');
foreach ($results as $document) {
deleteById($document, $database, $callback);
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Deleted {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param string $collection collectionID
* @param Query[] $queries
* @param Database $database
* @param callable $callback
*/
function listByGroup(string $collection, array $queries, Database $database, callable $callback = null): void
{
$count = 0;
$chunk = 0;
$limit = 50;
$results = [];
$sum = $limit;
$executionStart = \microtime(true);
while ($sum === $limit) {
$chunk++;
$results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
$count++;
}
}
$executionEnd = \microtime(true);
Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
/**
* @param Document $document certificates document
*/
function deleteCertificates(Document $document): void
{
$consoleDB = getConsoleDB();
// If domain has certificate generated
if (isset($document['certificateId'])) {
$domainUsingCertificate = $consoleDB->findOne('domains', [
Query::equal('certificateId', [$document['certificateId']])
]);
if (!$domainUsingCertificate) {
$mainDomain = App::getEnv('_APP_DOMAIN_TARGET', '');
if ($mainDomain === $document->getAttribute('domain')) {
$domainUsingCertificate = $mainDomain;
}
}
// If certificate is still used by some domain, mark we can't delete.
// Current domain should not be found, because we only have copy. Original domain is already deleted from database.
if ($domainUsingCertificate) {
Console::warning("Skipping certificate deletion, because a domain is still using it.");
return;
}
}
$domain = $document->getAttribute('domain');
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($domain && $checkTraversal && is_dir($directory)) {
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {
$consoleDB->deleteDocument('certificates', $document['certificateId']);
}
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
Console::info("Deleted certificate files for {$domain}");
} else {
Console::info("No certificate files found for {$domain}");
}
}
function deleteBucket(Document $document, Document $project)
{
$projectId = $project->getId();
$dbForProject = getProjectDB($project);
$dbForProject->deleteCollection('bucket_' . $document->getInternalId());
$device = getFilesDevice($projectId);
$device->deletePath($document->getId());
}
$server->workerStart();
$server->start();