2022-11-04 18:12:08 +13:00
|
|
|
<?php
|
2022-11-08 00:22:52 +13:00
|
|
|
|
2022-11-07 23:37:07 +13:00
|
|
|
ini_set('memory_limit', -1);
|
|
|
|
ini_set('max_execution_time', -1);
|
2022-11-04 18:12:08 +13:00
|
|
|
global $cli;
|
|
|
|
global $register;
|
|
|
|
|
2022-11-07 02:10:26 +13:00
|
|
|
use Cron\CronExpression;
|
|
|
|
use Utopia\App;
|
2022-11-04 18:12:08 +13:00
|
|
|
use Utopia\CLI\Console;
|
|
|
|
use Utopia\Database\DateTime;
|
|
|
|
use Utopia\Database\Query;
|
|
|
|
use Swoole\Timer;
|
|
|
|
|
2022-11-07 02:10:26 +13:00
|
|
|
const FUNCTION_VALIDATION_TIMER = 180; //seconds
|
|
|
|
const FUNCTION_ENQUEUE_TIMER = 60; //seconds
|
|
|
|
const ENQUEUE_TIME_FRAME = 60 * 5; // 5 min
|
2022-11-07 03:40:08 +13:00
|
|
|
sleep(4); // Todo prevent PDOException
|
2022-11-04 18:12:08 +13:00
|
|
|
|
2022-11-07 23:37:07 +13:00
|
|
|
|
2022-11-08 00:22:52 +13:00
|
|
|
/**
|
|
|
|
* 1. first load from db with limit+offset --line 82--
|
|
|
|
* 2. creating a 5-min offset array ($queue) --line 102--
|
|
|
|
* 3. First timer runs every minute, looping over $queue time slots (each slot is 1-min delta)
|
|
|
|
* if the function matches the current minute it should be dispatched to the functions worker.
|
|
|
|
* Then another translation is made to the cron pattern if it is in the next 5-min window
|
|
|
|
* it is assigned again to the $queue. --line 172--.
|
|
|
|
* 4. Second timer runs every X min and updates the $functions (large) list.
|
|
|
|
* The query fetches only functions that [resourceUpdatedAt] attr changed from the
|
|
|
|
* last time the timer that was fired (X min) --line 120--
|
|
|
|
* If the function was deleted it is unsets from the list ($functions) and the $queue.
|
|
|
|
* In the end of the timer the $queue is created again.
|
|
|
|
*
|
|
|
|
*/
|
2022-11-07 23:37:07 +13:00
|
|
|
|
2022-11-04 18:12:08 +13:00
|
|
|
$cli
|
2022-11-07 10:41:33 +13:00
|
|
|
->task('schedule')
|
2022-11-04 18:12:08 +13:00
|
|
|
->desc('Function scheduler task')
|
|
|
|
->action(function () use ($register) {
|
|
|
|
Console::title('Scheduler V1');
|
|
|
|
Console::success(APP_NAME . ' Scheduler v1 has started');
|
|
|
|
|
2022-11-07 02:10:26 +13:00
|
|
|
$createQueue = function () use (&$functions, &$queue) {
|
2022-11-07 23:37:07 +13:00
|
|
|
$loadStart = \microtime(true);
|
2022-11-07 02:10:26 +13:00
|
|
|
/**
|
|
|
|
* Creating smaller functions list containing 5-min timeframe.
|
|
|
|
*/
|
|
|
|
$timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME);
|
|
|
|
foreach ($functions as $function) {
|
|
|
|
$cron = new CronExpression($function['schedule']);
|
|
|
|
$next = DateTime::format($cron->getNextRunDate());
|
|
|
|
if ($next < $timeFrame) {
|
2022-11-07 10:41:33 +13:00
|
|
|
$queue[$next][$function['resourceId']] = $function;
|
2022-11-07 02:10:26 +13:00
|
|
|
}
|
|
|
|
}
|
2022-11-07 23:37:07 +13:00
|
|
|
$loadEnd = \microtime(true);
|
|
|
|
Console::error("Queue was built in " . ($loadEnd - $loadStart) . " seconds");
|
2022-11-07 02:10:26 +13:00
|
|
|
};
|
2022-11-04 18:12:08 +13:00
|
|
|
|
2022-11-07 03:40:08 +13:00
|
|
|
$removeFromQueue = function ($scheduleId) use (&$queue) {
|
|
|
|
foreach ($queue as $slot => $schedule) {
|
|
|
|
foreach ($schedule as $function) {
|
2022-11-07 10:41:33 +13:00
|
|
|
if ($scheduleId === $function['resourceId']) {
|
|
|
|
Console::error("Unsetting :{$function['resourceId']} from queue slot $slot");
|
|
|
|
unset($queue[$slot][$function['resourceId']]);
|
2022-11-07 03:40:08 +13:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2022-11-04 18:12:08 +13:00
|
|
|
$dbForConsole = getConsoleDB();
|
2022-11-07 23:37:07 +13:00
|
|
|
$limit = 200;
|
2022-11-04 18:12:08 +13:00
|
|
|
$sum = $limit;
|
|
|
|
$functions = [];
|
2022-11-07 02:10:26 +13:00
|
|
|
$queue = [];
|
2022-11-07 23:37:07 +13:00
|
|
|
$count = 0;
|
|
|
|
$loadStart = \microtime(true);
|
|
|
|
$total = 0;
|
2022-11-07 02:10:26 +13:00
|
|
|
/**
|
|
|
|
* Initial run fill $functions list
|
|
|
|
*/
|
2022-11-04 18:12:08 +13:00
|
|
|
while ($sum === $limit) {
|
|
|
|
$results = $dbForConsole->find('schedules', [
|
2022-11-07 02:10:26 +13:00
|
|
|
Query::equal('region', [App::getEnv('_APP_REGION')]),
|
2022-11-07 10:41:33 +13:00
|
|
|
Query::equal('resourceType', ['function']),
|
2022-11-04 18:12:08 +13:00
|
|
|
Query::equal('active', [true]),
|
2022-11-07 23:37:07 +13:00
|
|
|
Query::offset($count * $limit),
|
|
|
|
Query::limit($limit),
|
2022-11-04 18:12:08 +13:00
|
|
|
]);
|
|
|
|
|
|
|
|
$sum = count($results);
|
2022-11-07 23:37:07 +13:00
|
|
|
$total = $total + $sum;
|
2022-11-04 18:12:08 +13:00
|
|
|
foreach ($results as $document) {
|
2022-11-07 10:41:33 +13:00
|
|
|
$functions[$document['resourceId']] = $document;
|
2022-11-04 18:12:08 +13:00
|
|
|
}
|
2022-11-07 23:37:07 +13:00
|
|
|
$count++;
|
2022-11-04 18:12:08 +13:00
|
|
|
}
|
|
|
|
|
2022-11-07 23:37:07 +13:00
|
|
|
$loadEnd = \microtime(true);
|
|
|
|
Console::error("{$total} functions where loaded in " . ($loadEnd - $loadStart) . " seconds");
|
|
|
|
|
2022-11-07 02:10:26 +13:00
|
|
|
$createQueue();
|
2022-11-07 23:37:07 +13:00
|
|
|
|
2022-11-07 02:10:26 +13:00
|
|
|
$lastUpdate = DateTime::addSeconds(new \DateTime(), -FUNCTION_VALIDATION_TIMER);
|
2022-11-04 18:12:08 +13:00
|
|
|
|
|
|
|
Co\run(
|
2022-11-07 03:40:08 +13:00
|
|
|
function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) {
|
|
|
|
Timer::tick(FUNCTION_VALIDATION_TIMER * 1000, function () use ($removeFromQueue, $createQueue, $dbForConsole, &$functions, &$queue, &$lastUpdate) {
|
2022-11-04 18:12:08 +13:00
|
|
|
$time = DateTime::now();
|
|
|
|
$count = 0;
|
2022-11-08 00:22:52 +13:00
|
|
|
$limit = 200;
|
2022-11-04 18:12:08 +13:00
|
|
|
$sum = $limit;
|
2022-11-08 00:22:52 +13:00
|
|
|
$total = 0;
|
|
|
|
$timerStart = \microtime(true);
|
2022-11-07 02:10:26 +13:00
|
|
|
|
|
|
|
Console::info("Update proc run at: $time last update was at $lastUpdate");
|
|
|
|
/**
|
|
|
|
* Updating functions list from DB.
|
|
|
|
*/
|
2022-11-07 23:37:07 +13:00
|
|
|
while (!empty($sum)) {
|
2022-11-04 18:12:08 +13:00
|
|
|
$results = $dbForConsole->find('schedules', [
|
2022-11-07 02:10:26 +13:00
|
|
|
Query::equal('region', [App::getEnv('_APP_REGION')]),
|
2022-11-07 10:41:33 +13:00
|
|
|
Query::equal('resourceType', ['function']),
|
|
|
|
Query::greaterThan('resourceUpdatedAt', $lastUpdate),
|
2022-11-07 23:37:07 +13:00
|
|
|
Query::limit($limit),
|
|
|
|
Query::offset($count * $limit),
|
2022-11-04 18:12:08 +13:00
|
|
|
]);
|
|
|
|
$sum = count($results);
|
2022-11-08 00:22:52 +13:00
|
|
|
$total = $total + $sum;
|
2022-11-04 18:12:08 +13:00
|
|
|
foreach ($results as $document) {
|
2022-11-07 10:41:33 +13:00
|
|
|
$org = isset($functions[$document['resourceId']]) ? strtotime($functions[$document['resourceId']]['resourceUpdatedAt']) : null;
|
|
|
|
$new = strtotime($document['resourceUpdatedAt']);
|
2022-11-07 02:10:26 +13:00
|
|
|
if ($document['active'] === false) {
|
2022-11-07 10:41:33 +13:00
|
|
|
Console::error("Removing: {$document['resourceId']}");
|
|
|
|
unset($functions[$document['resourceId']]);
|
2022-11-07 02:10:26 +13:00
|
|
|
} elseif ($new > $org) {
|
2022-11-07 10:41:33 +13:00
|
|
|
Console::error("Updating: {$document['resourceId']}");
|
|
|
|
$functions[$document['resourceId']] = $document;
|
2022-11-07 02:10:26 +13:00
|
|
|
}
|
2022-11-07 10:41:33 +13:00
|
|
|
$removeFromQueue($document['resourceId']);
|
2022-11-04 18:12:08 +13:00
|
|
|
}
|
2022-11-07 23:37:07 +13:00
|
|
|
$count++;
|
2022-11-04 18:12:08 +13:00
|
|
|
}
|
2022-11-07 02:10:26 +13:00
|
|
|
$lastUpdate = DateTime::now();
|
|
|
|
$createQueue();
|
2022-11-08 00:22:52 +13:00
|
|
|
$timerEnd = \microtime(true);
|
|
|
|
|
2022-11-08 01:20:02 +13:00
|
|
|
Console::error("Update timer: {$total} functions where updated in " . ($timerEnd - $timerStart) . " seconds");
|
2022-11-04 18:12:08 +13:00
|
|
|
});
|
|
|
|
|
2022-11-07 02:10:26 +13:00
|
|
|
Timer::tick(FUNCTION_ENQUEUE_TIMER * 1000, function () use ($dbForConsole, &$functions, &$queue) {
|
2022-11-08 00:22:52 +13:00
|
|
|
$timerStart = \microtime(true);
|
2022-11-04 18:12:08 +13:00
|
|
|
$time = DateTime::now();
|
2022-11-07 02:10:26 +13:00
|
|
|
$timeFrame = DateTime::addSeconds(new \DateTime(), ENQUEUE_TIME_FRAME); /** 5 min */
|
2022-11-08 01:20:02 +13:00
|
|
|
$slot = (new \DateTime())->format('Y-m-d H:i:00.000');
|
2022-11-07 02:10:26 +13:00
|
|
|
|
|
|
|
Console::info("Enqueue proc run at: $time");
|
2022-11-07 10:41:33 +13:00
|
|
|
// Debug
|
2022-11-08 00:22:52 +13:00
|
|
|
// foreach ($queue as $slot => $schedule) {
|
|
|
|
// Console::log("Slot: $slot");
|
|
|
|
// foreach ($schedule as $function) {
|
|
|
|
// Console::log("{$function['resourceId']} {$function['schedule']}");
|
|
|
|
// }
|
|
|
|
// }
|
2022-11-07 02:10:26 +13:00
|
|
|
|
2022-11-08 01:20:02 +13:00
|
|
|
if (array_key_exists($slot, $queue)) {
|
|
|
|
$schedule = $queue[$slot];
|
|
|
|
console::log("Number of function sent to worker (" . count($schedule));
|
|
|
|
|
|
|
|
foreach ($schedule as $function) {
|
|
|
|
/**
|
|
|
|
* Enqueue function (here should be the Enqueue call
|
|
|
|
*/
|
|
|
|
//Console::warning("Enqueueing :{$function['resourceId']}");
|
|
|
|
$cron = new CronExpression($function['schedule']);
|
|
|
|
$next = DateTime::format($cron->getNextRunDate());
|
|
|
|
|
|
|
|
/**
|
|
|
|
* If next schedule is in 5-min timeframe
|
|
|
|
* and it was not removed or changed, re-enqueue the function.
|
|
|
|
*/
|
|
|
|
if (
|
|
|
|
$next < $timeFrame &&
|
|
|
|
!empty($functions[$function['resourceId']] &&
|
|
|
|
$function['schedule'] === $functions[$function['resourceId']]['schedule'])
|
|
|
|
) {
|
|
|
|
//Console::warning("re-enqueueing :{$function['resourceId']}");
|
|
|
|
$queue[$next][$function['resourceId']] = $function;
|
2022-11-07 02:10:26 +13:00
|
|
|
}
|
2022-11-08 01:20:02 +13:00
|
|
|
unset($queue[$slot][$function['resourceId']]); /** removing function from slot */
|
2022-11-07 02:10:26 +13:00
|
|
|
}
|
2022-11-08 01:20:02 +13:00
|
|
|
unset($queue[$slot]); /** removing slot */
|
2022-11-04 18:12:08 +13:00
|
|
|
}
|
2022-11-08 00:22:52 +13:00
|
|
|
$timerEnd = \microtime(true);
|
2022-11-08 01:20:02 +13:00
|
|
|
Console::error("Queue timer: finished in " . ($timerEnd - $timerStart) . " seconds");
|
2022-11-04 18:12:08 +13:00
|
|
|
});
|
|
|
|
}
|
|
|
|
);
|
|
|
|
});
|