Merge pull request #2427 from appwrite/feat-improve-deletes-worker
fix(deletes): add realtime logs and improve code
This commit is contained in:
commit
446827732d
|
@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit;
|
||||||
use Utopia\CLI\Console;
|
use Utopia\CLI\Console;
|
||||||
use Utopia\Audit\Audit;
|
use Utopia\Audit\Audit;
|
||||||
|
|
||||||
require_once __DIR__.'/../init.php';
|
require_once __DIR__ . '/../init.php';
|
||||||
|
|
||||||
Console::title('Deletes V1 Worker');
|
Console::title('Deletes V1 Worker');
|
||||||
Console::success(APP_NAME . ' deletes worker v1 has started' . "\n");
|
Console::success(APP_NAME . ' deletes worker v1 has started' . "\n");
|
||||||
|
@ -31,12 +31,11 @@ class DeletesV1 extends Worker
|
||||||
{
|
{
|
||||||
$projectId = $this->args['projectId'] ?? '';
|
$projectId = $this->args['projectId'] ?? '';
|
||||||
$type = $this->args['type'] ?? '';
|
$type = $this->args['type'] ?? '';
|
||||||
|
|
||||||
switch (strval($type)) {
|
switch (strval($type)) {
|
||||||
case DELETE_TYPE_DOCUMENT:
|
case DELETE_TYPE_DOCUMENT:
|
||||||
$document = $this->args['document'] ?? [];
|
$document = new Document($this->args['document'] ?? []);
|
||||||
$document = new Document($document);
|
|
||||||
|
|
||||||
switch ($document->getCollection()) {
|
switch ($document->getCollection()) {
|
||||||
case DELETE_TYPE_COLLECTIONS:
|
case DELETE_TYPE_COLLECTIONS:
|
||||||
$this->deleteCollection($document, $projectId);
|
$this->deleteCollection($document, $projectId);
|
||||||
|
@ -72,8 +71,7 @@ class DeletesV1 extends Worker
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DELETE_TYPE_REALTIME:
|
case DELETE_TYPE_REALTIME:
|
||||||
//$this->deleteRealtimeUsage($this->args['timestamp']);
|
$this->deleteRealtimeUsage($this->args['timestamp']);
|
||||||
//TODO: implement this
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DELETE_TYPE_CERTIFICATES:
|
case DELETE_TYPE_CERTIFICATES:
|
||||||
|
@ -120,12 +118,10 @@ class DeletesV1 extends Worker
|
||||||
* @param int $timestamp1d
|
* @param int $timestamp1d
|
||||||
* @param int $timestamp30m
|
* @param int $timestamp30m
|
||||||
*/
|
*/
|
||||||
protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) {
|
protected function deleteUsageStats(int $timestamp1d, int $timestamp30m)
|
||||||
$this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) {
|
{
|
||||||
if (!($dbForInternal = $this->getInternalDB($projectId))) {
|
$this->deleteForProjectIds(function (string $projectId) use ($timestamp1d, $timestamp30m) {
|
||||||
throw new Exception('Failed to get projectDB for project '.$projectId);
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
}
|
|
||||||
|
|
||||||
// Delete Usage stats
|
// Delete Usage stats
|
||||||
$this->deleteByGroup('stats', [
|
$this->deleteByGroup('stats', [
|
||||||
new Query('time', Query::TYPE_LESSER, [$timestamp1d]),
|
new Query('time', Query::TYPE_LESSER, [$timestamp1d]),
|
||||||
|
@ -138,7 +134,7 @@ class DeletesV1 extends Worker
|
||||||
], $dbForInternal);
|
], $dbForInternal);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param Document $document teams document
|
* @param Document $document teams document
|
||||||
* @param string $projectId
|
* @param string $projectId
|
||||||
|
@ -159,13 +155,14 @@ class DeletesV1 extends Worker
|
||||||
protected function deleteProject(Document $document): void
|
protected function deleteProject(Document $document): void
|
||||||
{
|
{
|
||||||
$projectId = $document->getId();
|
$projectId = $document->getId();
|
||||||
|
|
||||||
// Delete all DBs
|
// Delete all DBs
|
||||||
$this->getExternalDB($projectId)->delete();
|
$this->getExternalDB($projectId)->delete();
|
||||||
$this->getInternalDB($projectId)->delete();
|
$this->getInternalDB($projectId)->delete();
|
||||||
|
|
||||||
// Delete all storage directories
|
// Delete all storage directories
|
||||||
$uploads = new Local(APP_STORAGE_UPLOADS.'/app-'.$document->getId());
|
$uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId());
|
||||||
$cache = new Local(APP_STORAGE_CACHE.'/app-'.$document->getId());
|
$cache = new Local(APP_STORAGE_CACHE . '/app-' . $document->getId());
|
||||||
|
|
||||||
$uploads->delete($uploads->getRoot(), true);
|
$uploads->delete($uploads->getRoot(), true);
|
||||||
$cache->delete($cache->getRoot(), true);
|
$cache->delete($cache->getRoot(), true);
|
||||||
|
@ -183,12 +180,12 @@ class DeletesV1 extends Worker
|
||||||
// Delete Memberships and decrement team membership counts
|
// Delete Memberships and decrement team membership counts
|
||||||
$this->deleteByGroup('memberships', [
|
$this->deleteByGroup('memberships', [
|
||||||
new Query('userId', Query::TYPE_EQUAL, [$userId])
|
new Query('userId', Query::TYPE_EQUAL, [$userId])
|
||||||
], $this->getInternalDB($projectId), function(Document $document) use ($projectId, $userId) {
|
], $this->getInternalDB($projectId), function (Document $document) use ($projectId) {
|
||||||
|
|
||||||
if ($document->getAttribute('confirm')) { // Count only confirmed members
|
if ($document->getAttribute('confirm')) { // Count only confirmed members
|
||||||
$teamId = $document->getAttribute('teamId');
|
$teamId = $document->getAttribute('teamId');
|
||||||
$team = $this->getInternalDB($projectId)->getDocument('teams', $teamId);
|
$team = $this->getInternalDB($projectId)->getDocument('teams', $teamId);
|
||||||
if(!$team->isEmpty()) {
|
if (!$team->isEmpty()) {
|
||||||
$team = $this->getInternalDB($projectId)->updateDocument('teams', $teamId, new Document(\array_merge($team->getArrayCopy(), [
|
$team = $this->getInternalDB($projectId)->updateDocument('teams', $teamId, new Document(\array_merge($team->getArrayCopy(), [
|
||||||
'sum' => \max($team->getAttribute('sum', 0) - 1, 0), // Ensure that sum >= 0
|
'sum' => \max($team->getAttribute('sum', 0) - 1, 0), // Ensure that sum >= 0
|
||||||
])));
|
])));
|
||||||
|
@ -202,11 +199,8 @@ class DeletesV1 extends Worker
|
||||||
*/
|
*/
|
||||||
protected function deleteExecutionLogs(int $timestamp): void
|
protected function deleteExecutionLogs(int $timestamp): void
|
||||||
{
|
{
|
||||||
$this->deleteForProjectIds(function(string $projectId) use ($timestamp) {
|
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
|
||||||
if (!($dbForInternal = $this->getInternalDB($projectId))) {
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
throw new Exception('Failed to get projectDB for project '.$projectId);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete Executions
|
// Delete Executions
|
||||||
$this->deleteByGroup('executions', [
|
$this->deleteByGroup('executions', [
|
||||||
new Query('dateCreated', Query::TYPE_LESSER, [$timestamp])
|
new Query('dateCreated', Query::TYPE_LESSER, [$timestamp])
|
||||||
|
@ -214,18 +208,33 @@ class DeletesV1 extends Worker
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int $timestamp
|
||||||
|
*/
|
||||||
|
protected function deleteRealtimeUsage(int $timestamp): void
|
||||||
|
{
|
||||||
|
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
|
||||||
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
|
// Delete Dead Realtime Logs
|
||||||
|
$this->deleteByGroup('realtime', [
|
||||||
|
new Query('timestamp', Query::TYPE_LESSER, [$timestamp])
|
||||||
|
], $dbForInternal);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param int $timestamp
|
* @param int $timestamp
|
||||||
*/
|
*/
|
||||||
protected function deleteAbuseLogs(int $timestamp): void
|
protected function deleteAbuseLogs(int $timestamp): void
|
||||||
{
|
{
|
||||||
if($timestamp == 0) {
|
if ($timestamp == 0) {
|
||||||
throw new Exception('Failed to delete audit logs. No timestamp provided');
|
throw new Exception('Failed to delete audit logs. No timestamp provided');
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->deleteForProjectIds(function($projectId) use ($timestamp){
|
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
|
||||||
$timeLimit = new TimeLimit("", 0, 1, $this->getInternalDB($projectId));
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
$abuse = new Abuse($timeLimit);
|
$timeLimit = new TimeLimit("", 0, 1, $dbForInternal);
|
||||||
|
$abuse = new Abuse($timeLimit);
|
||||||
|
|
||||||
$status = $abuse->cleanup($timestamp);
|
$status = $abuse->cleanup($timestamp);
|
||||||
if (!$status) {
|
if (!$status) {
|
||||||
|
@ -239,11 +248,12 @@ class DeletesV1 extends Worker
|
||||||
*/
|
*/
|
||||||
protected function deleteAuditLogs(int $timestamp): void
|
protected function deleteAuditLogs(int $timestamp): void
|
||||||
{
|
{
|
||||||
if($timestamp == 0) {
|
if ($timestamp == 0) {
|
||||||
throw new Exception('Failed to delete audit logs. No timestamp provided');
|
throw new Exception('Failed to delete audit logs. No timestamp provided');
|
||||||
}
|
}
|
||||||
$this->deleteForProjectIds(function($projectId) use ($timestamp){
|
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
|
||||||
$audit = new Audit($this->getInternalDB($projectId));
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
|
$audit = new Audit($dbForInternal);
|
||||||
$status = $audit->cleanup($timestamp);
|
$status = $audit->cleanup($timestamp);
|
||||||
if (!$status) {
|
if (!$status) {
|
||||||
throw new Exception('Failed to delete Audit logs for project' . $projectId);
|
throw new Exception('Failed to delete Audit logs for project' . $projectId);
|
||||||
|
@ -258,12 +268,12 @@ class DeletesV1 extends Worker
|
||||||
protected function deleteFunction(Document $document, string $projectId): void
|
protected function deleteFunction(Document $document, string $projectId): void
|
||||||
{
|
{
|
||||||
$dbForInternal = $this->getInternalDB($projectId);
|
$dbForInternal = $this->getInternalDB($projectId);
|
||||||
$device = new Local(APP_STORAGE_FUNCTIONS.'/app-'.$projectId);
|
$device = new Local(APP_STORAGE_FUNCTIONS . '/app-' . $projectId);
|
||||||
|
|
||||||
// Delete Tags
|
// Delete Tags
|
||||||
$this->deleteByGroup('tags', [
|
$this->deleteByGroup('tags', [
|
||||||
new Query('functionId', Query::TYPE_EQUAL, [$document->getId()])
|
new Query('functionId', Query::TYPE_EQUAL, [$document->getId()])
|
||||||
], $dbForInternal, function(Document $document) use ($device) {
|
], $dbForInternal, function (Document $document) use ($device) {
|
||||||
|
|
||||||
if ($device->delete($document->getAttribute('path', ''))) {
|
if ($device->delete($document->getAttribute('path', ''))) {
|
||||||
Console::success('Delete code tag: ' . $document->getAttribute('path', ''));
|
Console::success('Delete code tag: ' . $document->getAttribute('path', ''));
|
||||||
|
@ -290,8 +300,8 @@ class DeletesV1 extends Worker
|
||||||
{
|
{
|
||||||
Authorization::disable();
|
Authorization::disable();
|
||||||
|
|
||||||
if($database->deleteDocument($document->getCollection(), $document->getId())) {
|
if ($database->deleteDocument($document->getCollection(), $document->getId())) {
|
||||||
Console::success('Deleted document "'.$document->getId().'" successfully');
|
Console::success('Deleted document "' . $document->getId() . '" successfully');
|
||||||
|
|
||||||
if (is_callable($callback)) {
|
if (is_callable($callback)) {
|
||||||
$callback($document);
|
$callback($document);
|
||||||
|
@ -319,16 +329,15 @@ class DeletesV1 extends Worker
|
||||||
|
|
||||||
$executionStart = \microtime(true);
|
$executionStart = \microtime(true);
|
||||||
|
|
||||||
while($sum === $limit) {
|
while ($sum === $limit) {
|
||||||
Authorization::disable();
|
Authorization::disable();
|
||||||
$projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit));
|
$projects = $this->getConsoleDB()->find('projects', [], $limit, ($chunk * $limit));
|
||||||
Authorization::reset();
|
Authorization::reset();
|
||||||
|
|
||||||
$chunk++;
|
$chunk++;
|
||||||
|
|
||||||
$projectIds = array_map (function ($project) {
|
/** @var string[] $projectIds */
|
||||||
return $project->getId();
|
$projectIds = array_map(fn(Document $project) => $project->getId(), $projects);
|
||||||
}, $projects);
|
|
||||||
|
|
||||||
$sum = count($projects);
|
$sum = count($projects);
|
||||||
|
|
||||||
|
@ -359,7 +368,7 @@ class DeletesV1 extends Worker
|
||||||
|
|
||||||
$executionStart = \microtime(true);
|
$executionStart = \microtime(true);
|
||||||
|
|
||||||
while($sum === $limit) {
|
while ($sum === $limit) {
|
||||||
$chunk++;
|
$chunk++;
|
||||||
|
|
||||||
Authorization::disable();
|
Authorization::disable();
|
||||||
|
|
Loading…
Reference in a new issue