7#include "greedyscheduler.h"
9#include <ekos_scheduler_debug.h>
14#include "ui_scheduler.h"
15#include "schedulerjob.h"
16#include "schedulerutils.h"
18#define TEST_PRINT if (false) fprintf
21constexpr int SCHEDULE_RESOLUTION_MINUTES = 2;
26GreedyScheduler::GreedyScheduler()
30void GreedyScheduler::setParams(
bool restartImmediately,
bool restartQueue,
31 bool rescheduleErrors,
int abortDelay,
32 int errorHandlingDelay)
34 setRescheduleAbortsImmediate(restartImmediately);
35 setRescheduleAbortsQueue(restartQueue);
36 setRescheduleErrors(rescheduleErrors);
37 setAbortDelaySeconds(abortDelay);
38 setErrorDelaySeconds(errorHandlingDelay);
58 scheduledJob =
nullptr;
61 prepareJobsForEvaluation(jobs, now, capturedFramesCount, logger);
63 scheduledJob = selectNextJob(jobs, now,
nullptr, SIMULATE, &when,
nullptr,
nullptr, &capturedFramesCount);
64 auto schedule = getSchedule();
65 if (logger !=
nullptr)
67 if (!schedule.
empty())
72 for (
int i = schedule.
size() - 1; i >= 0; i--)
73 logger->appendLogText(GreedyScheduler::jobScheduleString(schedule[i]));
74 logger->appendLogText(
QString(
"Greedy Scheduler plan for the next 48 hours starting %1 (%2)s:")
77 else logger->appendLogText(
QString(
"Greedy Scheduler: empty plan (%1s)").
arg(timer.elapsed() / 1000.0));
79 if (scheduledJob !=
nullptr)
81 qCDebug(KSTARS_EKOS_SCHEDULER)
82 <<
QString(
"Greedy Scheduler scheduling next job %1 at %2")
83 .
arg(scheduledJob->getName(), when.
toString(
"hh:mm"));
85 scheduledJob->setStartupTime(when);
97 const SchedulerJob *
const currentJob)
100 if (currentJob && currentJob->getStateTime().secsTo(now) < 5)
107 SimulationType simType = SIMULATE_EACH_JOB_ONCE;
108 if (m_SimSeconds > 0.2 ||
109 (m_LastCheckJobSim.
isValid() && m_LastCheckJobSim.
secsTo(now) < 60))
110 simType = DONT_SIMULATE;
112 const SchedulerJob *
next = selectNextJob(jobs, now, currentJob, simType, &startTime);
113 if (next == currentJob && now.
secsTo(startTime) <= 1)
115 if (simType != DONT_SIMULATE)
116 m_LastCheckJobSim = now;
123 qCDebug(KSTARS_EKOS_SCHEDULER)
124 <<
QString(
"Greedy Scheduler bumping current job %1 for %2 at %3")
125 .
arg(currentJob->getName(), next ?
next->getName() :
"---", now.
toString(
"hh:mm"));
144void GreedyScheduler::prepareJobsForEvaluation(
149 foreach (SchedulerJob *job, jobs)
151 switch (job->getCompletionCondition())
155 if (job->getCompletionTime().isValid() && job->getCompletionTime() < now)
165 if (job->getRepeatsRemaining() == 0)
167 if (logger !=
nullptr)
logger->appendLogText(
i18n(
"Job '%1' has no more batches remaining.", job->getName()));
169 job->setEstimatedTime(0);
180 foreach (SchedulerJob *job, jobs)
182 switch (job->getState())
204 foreach (SchedulerJob *job, jobs)
212 if (reestimateJobTimes)
214 job->setEstimatedTime(-1);
215 if (SchedulerUtils::estimateJobTime(job, capturedFramesCount, logger) ==
false)
221 if (job->getEstimatedTime() == 0)
223 job->setRepeatsRemaining(0);
235bool allowJob(
const SchedulerJob *job,
bool rescheduleAbortsImmediate,
bool rescheduleAbortsQueue,
bool rescheduleErrors)
239 if (job->getState() ==
SCHEDJOB_ABORTED && !rescheduleAbortsImmediate && !rescheduleAbortsQueue)
250 bool rescheduleAbortsQueue,
int abortDelaySeconds,
251 bool rescheduleErrors,
int errorDelaySeconds)
254 const QDateTime &abortTime = job->getLastAbortTime();
255 const QDateTime &errorTime = job->getLastErrorTime();
257 if (abortTime.
isValid() && rescheduleAbortsQueue)
259 auto abortStartTime = abortTime.
addSecs(abortDelaySeconds);
260 if (abortStartTime > now)
261 possibleStart = abortStartTime;
265 if (errorTime.
isValid() && rescheduleErrors)
267 auto errorStartTime = errorTime.
addSecs(errorDelaySeconds);
268 if (errorStartTime > now)
269 possibleStart = errorStartTime;
272 if (!possibleStart.
isValid() || possibleStart < now)
274 return possibleStart;
297 const SchedulerJob *
const currentJob, SimulationType simType,
QDateTime *when,
302 constexpr int MIN_RUN_SECS = 10 * 60;
305 constexpr int MAX_INTERRUPT_SECS = 30;
308 bool currentJobIsStartAt = (currentJob && currentJob->getFileStartupCondition() == START_AT &&
309 currentJob->getFileStartupTime().isValid());
311 SchedulerJob * nextJob =
nullptr;
314 for (
int i = 0; i < jobs.
size(); ++i)
316 SchedulerJob *
const job = jobs[i];
317 const bool evaluatingCurrentJob = (currentJob && (job == currentJob));
319 if (!allowJob(job, rescheduleAbortsImmediate, rescheduleAbortsQueue, rescheduleErrors))
323 QDateTime startSearchingtAt = firstPossibleStart(
324 job, now, rescheduleAbortsQueue, abortDelaySeconds, rescheduleErrors, errorDelaySeconds);
329 const QDateTime startTime = job->getNextPossibleStartTime(startSearchingtAt, SCHEDULE_RESOLUTION_MINUTES,
330 evaluatingCurrentJob);
333 if (nextJob ==
nullptr)
336 nextStart = startTime;
338 if (nextInterruption) *nextInterruption =
QDateTime();
341 else if (Options::greedyScheduling())
345 const int runSecs = evaluatingCurrentJob ? MAX_INTERRUPT_SECS : MIN_RUN_SECS;
348 if (evaluatingCurrentJob && currentJobIsStartAt)
350 if (nextInterruption) *nextInterruption =
QDateTime();
351 nextStart = startTime;
355 else if (startTime.
secsTo(nextStart) > runSecs)
359 if (nextInterruption) *nextInterruption = nextStart;
360 interruptStr =
QString(
"interrupted by %1").
arg(nextJob->getName());
361 nextStart = startTime;
367 if (!currentJob && nextStart.
isValid() && now.
secsTo(nextStart) < MIN_RUN_SECS)
370 else if (evaluatingCurrentJob)
378 if (evaluatingCurrentJob)
break;
380 if (nextJob !=
nullptr)
386 for (
int i = 0; i < jobs.
size(); ++i)
388 SchedulerJob *
const atJob = jobs[i];
389 if (atJob == nextJob)
391 const QDateTime atTime = atJob->getFileStartupTime();
392 if (atJob->getFileStartupCondition() == START_AT && atTime.
isValid())
394 if (!allowJob(atJob, rescheduleAbortsImmediate, rescheduleAbortsQueue, rescheduleErrors))
397 QDateTime startSearchingtAt = firstPossibleStart(
398 atJob, now, rescheduleAbortsQueue, abortDelaySeconds, rescheduleErrors,
402 const QDateTime atJobStartTime = atJob->getNextPossibleStartTime(startSearchingtAt, SCHEDULE_RESOLUTION_MINUTES, currentJob
403 && (atJob == currentJob));
407 const double startDelta = atJobStartTime.
secsTo(atTime);
408 if (fabs(startDelta) < 20 * 60)
414 const int gap = currentJob ==
nullptr ? MIN_RUN_SECS : 30;
415 if (nextStart.
secsTo(atJobStartTime) <= gap)
418 nextStart = atJobStartTime;
419 if (nextInterruption) *nextInterruption =
QDateTime();
421 else if (nextInterruption)
425 if (!nextInterruption->
isValid() ||
426 atJobStartTime.
secsTo(*nextInterruption) < 0)
428 *nextInterruption = atJobStartTime;
429 interruptStr =
QString(
"interrupted by %1").
arg(atJob->getName());
441 if (nextJob && !nextJob->getGroup().isEmpty() && Options::greedyScheduling() && nextJob->getCompletedIterations() > 0)
444 bool foundSelectedJob =
false;
445 for (
int i = 0; i < jobs.
size(); ++i)
447 SchedulerJob *
const job = jobs[i];
450 foundSelectedJob =
true;
458 if (!foundSelectedJob ||
459 (job->getGroup() != nextJob->getGroup()) ||
460 (job->getCompletedIterations() >= nextJob->getCompletedIterations()) ||
461 !allowJob(job, rescheduleAbortsImmediate, rescheduleAbortsQueue, rescheduleErrors))
464 const bool evaluatingCurrentJob = (currentJob && (job == currentJob));
467 QDateTime startSearchingtAt = firstPossibleStart(
468 job, now, rescheduleAbortsQueue, abortDelaySeconds, rescheduleErrors, errorDelaySeconds);
471 const QDateTime startTime = job->getNextPossibleStartTime(startSearchingtAt, SCHEDULE_RESOLUTION_MINUTES,
472 evaluatingCurrentJob);
475 if (!startTime.
isValid() || startTime.
secsTo(nextStart) > MAX_INTERRUPT_SECS)
479 if (evaluatingCurrentJob && currentJobIsStartAt)
481 if (nextInterruption) *nextInterruption =
QDateTime();
482 nextStart = startTime;
486 else if (startTime.
secsTo(nextStart) >= -MAX_INTERRUPT_SECS)
489 nextStart = startTime;
495 if (when !=
nullptr) *when = nextStart;
496 if (interruptReason !=
nullptr) *interruptReason = interruptStr;
503 unsetEvaluation(jobs);
507 constexpr int twoDays = 48 * 3600;
508 if (simType != DONT_SIMULATE && nextJob !=
nullptr)
512 QDateTime simEnd = simulate(jobs, now, simulationLimit, capturedFramesCount, simType);
516 if (!Options::rememberJobProgress() && Options::schedulerRepeatEverything())
518 int repeats = 0, maxRepeats = 5;
519 while (simEnd.
isValid() && simEnd.
secsTo(simulationLimit) > 0 && ++repeats < maxRepeats)
522 simEnd = simulate(jobs, simEnd, simulationLimit,
nullptr, simType);
526 m_SimSeconds = simTimer.
elapsed() / 1000.0;
535 TEST_PRINT(stderr,
"%d simulate()\n", __LINE__);
541 foreach (SchedulerJob *job, jobs)
543 SchedulerJob *newJob =
new SchedulerJob();
546 copiedJobs.
append(newJob);
547 job->setGreedyCompletionTime(
QDateTime());
552 int numStartupCandidates = 0, numStartups = 0;
554 foreach (SchedulerJob *job, copiedJobs)
557 const auto state = job->getState();
560 numStartupCandidates++;
564 if (capturedFramesCount !=
nullptr)
565 capturedFramesCopy = *capturedFramesCount;
567 prepareJobsForEvaluation(copiedJobs, time, capturedFramesCopy,
nullptr,
false);
571 bool exceededIterations =
false;
575 for(
int i = 0; i < simJobs.
size(); ++i)
576 workDone[simJobs[i]] = 0.0;
586 SchedulerJob *selectedJob = selectNextJob(
587 simJobs, simTime,
nullptr, DONT_SIMULATE, &jobStartTime, &jobInterruptTime, &interruptReason);
588 if (selectedJob ==
nullptr)
591 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
"%1 starting at %2 interrupted at \"%3\" reason \"%4\"")
592 .arg(selectedJob->getName()).
arg(jobStartTime.
toString(
"MM/dd hh:mm"))
595 if (endTime.
isValid() && jobStartTime.
secsTo(endTime) < 0)
break;
601 foreach (SchedulerJob *job, simJobs)
603 if (job != selectedJob &&
604 job->getStartupCondition() == START_AT &&
605 jobStartTime.
secsTo(job->getStartupTime()) > 0 &&
609 QDateTime startAtTime = job->getStartupTime();
610 if (!nextStartAtTime.
isValid() || nextStartAtTime.
secsTo(startAtTime) < 0)
611 nextStartAtTime = startAtTime;
615 QDateTime constraintStopTime = jobInterruptTime;
616 if (nextStartAtTime.
isValid() &&
617 (!constraintStopTime.
isValid() ||
618 nextStartAtTime.
secsTo(constraintStopTime) < 0))
619 constraintStopTime = nextStartAtTime;
623 QDateTime jobConstraintTime = selectedJob->getNextEndTime(jobStartTime, SCHEDULE_RESOLUTION_MINUTES, &constraintReason,
626 std::abs(jobConstraintTime.
secsTo(nextStartAtTime)) < 2 * SCHEDULE_RESOLUTION_MINUTES)
627 constraintReason =
"interrupted by start-at job";
628 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" constraint \"%1\" reason \"%2\"")
631 if (selectedJob->getEstimatedTime() > 0)
634 const int timeLeft = selectedJob->getEstimatedTime() - workDone[selectedJob];
635 jobCompletionTime = jobStartTime.
addSecs(timeLeft);
636 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" completion \"%1\" time left %2s")
641 QDateTime jobStopTime = jobInterruptTime;
642 QString stopReason = jobStopTime.
isValid() ? interruptReason :
"";
643 if (jobConstraintTime.
isValid() && (!jobStopTime.
isValid() || jobStopTime.
secsTo(jobConstraintTime) < 0))
645 stopReason = constraintReason;
646 jobStopTime = jobConstraintTime;
647 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" picked constraint").toLatin1().data());
649 if (jobCompletionTime.
isValid() && (!jobStopTime.
isValid() || jobStopTime.
secsTo(jobCompletionTime) < 0))
651 stopReason =
"job completion";
652 jobStopTime = jobCompletionTime;
653 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" picked completion").toLatin1().data());
658 if (!selectedJob->getGroup().isEmpty() &&
659 (selectedJob->getCompletionCondition() == FINISH_LOOP ||
660 selectedJob->getCompletionCondition() == FINISH_REPEAT ||
661 selectedJob->getCompletionCondition() == FINISH_AT))
663 if (originalIteration.
find(selectedJob) == originalIteration.
end())
664 originalIteration[selectedJob] = selectedJob->getCompletedIterations();
665 if (originalSecsLeftIteration.
find(selectedJob) == originalSecsLeftIteration.
end())
666 originalSecsLeftIteration[selectedJob] = selectedJob->getEstimatedTimeLeftThisRepeat();
669 int leftThisRepeat = selectedJob->getEstimatedTimeLeftThisRepeat();
670 int secsPerRepeat = selectedJob->getEstimatedTimePerRepeat();
671 int secsLeftThisRepeat = (workDone[selectedJob] < leftThisRepeat) ?
672 leftThisRepeat - workDone[selectedJob] : secsPerRepeat;
674 if (workDone[selectedJob] == 0)
675 secsLeftThisRepeat += selectedJob->getEstimatedStartupTime();
678 if (secsLeftThisRepeat > 0 &&
679 (!jobStopTime.
isValid() || secsLeftThisRepeat < jobStartTime.
secsTo(jobStopTime)))
681 auto tempStart = jobStartTime;
682 auto tempInterrupt = jobInterruptTime;
683 auto tempReason = stopReason;
684 SchedulerJob keepJob = *selectedJob;
686 auto t = jobStartTime.
addSecs(secsLeftThisRepeat);
687 int iteration = selectedJob->getCompletedIterations();
688 int iters = 0, maxIters = 20;
689 while ((!jobStopTime.
isValid() || t.secsTo(jobStopTime) > 0) && iters++ < maxIters)
691 selectedJob->setCompletedIterations(++iteration);
692 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" iteration=%1").arg(iteration).toLatin1().data());
693 SchedulerJob *
next = selectNextJob(simJobs, t,
nullptr, DONT_SIMULATE, &tempStart, &tempInterrupt, &tempReason);
694 if (next != selectedJob)
696 stopReason =
"interrupted for group member";
698 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" switched to group member %1 at %2")
699 .arg(next ==
nullptr ?
"null" :
next->getName()).arg(t.
toString(
"MM/dd hh:mm")).toLatin1().data());
703 t = t.addSecs(secsPerRepeat);
705 *selectedJob = keepJob;
712 const int secondsRun = jobStartTime.
secsTo(jobStopTime);
713 workDone[selectedJob] += secondsRun;
715 if ((originalIteration.
find(selectedJob) != originalIteration.
end()) &&
716 (originalSecsLeftIteration.
find(selectedJob) != originalSecsLeftIteration.
end()))
718 int completedIterations = originalIteration[selectedJob];
719 if (workDone[selectedJob] >= originalSecsLeftIteration[selectedJob] &&
720 selectedJob->getEstimatedTimePerRepeat() > 0)
721 completedIterations +=
722 1 + (workDone[selectedJob] - originalSecsLeftIteration[selectedJob]) / selectedJob->getEstimatedTimePerRepeat();
723 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
724 QString(
" work sets interations=%1").arg(completedIterations).toLatin1().data());
725 selectedJob->setCompletedIterations(completedIterations);
731 if (!selectedJob->getStartupTime().isValid())
734 selectedJob->setStartupTime(jobStartTime);
735 selectedJob->setGreedyCompletionTime(jobStopTime);
736 selectedJob->setStopReason(stopReason);
738 scheduledJobs.
append(selectedJob);
739 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" Scheduled: %1 %2 -> %3 %4 work done %5s")
740 .arg(selectedJob->getName()).
arg(selectedJob->getStartupTime().toString(
"MM/dd hh:mm"))
741 .
arg(selectedJob->getGreedyCompletionTime().toString(
"MM/dd hh:mm")).
arg(selectedJob->getStopReason())
746 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" Added: %1 %2 -> %3 %4 work done %5s")
747 .arg(selectedJob->getName()).
arg(selectedJob->getStartupTime().toString(
"MM/dd hh:mm"))
748 .
arg(selectedJob->getGreedyCompletionTime().toString(
"MM/dd hh:mm")).
arg(selectedJob->getStopReason())
753 if (selectedJob->getEstimatedTime() >= 0 &&
754 workDone[selectedJob] >= selectedJob->getEstimatedTime())
757 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
" job %1 is complete")
760 schedule.
append(JobSchedule(jobs[copiedJobs.
indexOf(selectedJob)], jobStartTime, jobStopTime, stopReason));
761 simEndTime = jobStopTime;
762 simTime = jobStopTime.
addSecs(60);
769 if (++iterations > std::max(20, numStartupCandidates))
771 exceededIterations =
true;
772 TEST_PRINT(stderr,
"%d %s\n", __LINE__,
QString(
"ending simulation after %1 iterations")
773 .arg(iterations).toLatin1().data());
777 if (simType == SIMULATE_EACH_JOB_ONCE)
779 bool allJobsProcessedOnce =
true;
780 for (
const auto job : simJobs)
782 if (allowJob(job, rescheduleAbortsImmediate, rescheduleAbortsQueue, rescheduleErrors) &&
783 !job->getStartupTime().isValid())
785 allJobsProcessedOnce =
false;
789 if (allJobsProcessedOnce)
791 TEST_PRINT(stderr,
"%d ending simulation, all jobs processed once\n", __LINE__);
800 for (
int i = 0; i < jobs.
size(); ++i)
802 if (scheduledJobs.
indexOf(copiedJobs[i]) >= 0)
808 jobs[i]->setStartupTime(copiedJobs[i]->getStartupTime());
811 jobs[i]->setGreedyCompletionTime(copiedJobs[i]->getGreedyCompletionTime());
812 jobs[i]->setStopReason(copiedJobs[i]->getStopReason());
817 unsetEvaluation(jobs);
819 return exceededIterations ?
QDateTime() : simEndTime;
824 for (
int i = 0; i < jobs.
size(); ++i)
831QString GreedyScheduler::jobScheduleString(
const JobSchedule &jobSchedule)
833 return QString(
"%1\t%2 --> %3 \t%4")
834 .
arg(jobSchedule.job->getName(), -10)
835 .
arg(jobSchedule.startTime.toString(
"MM/dd hh:mm"),
836 jobSchedule.stopTime.toString(
"hh:mm"), jobSchedule.stopReason);
841 foreach (
auto &line, schedule)
843 fprintf(stderr,
"%s\n",
QString(
"%1 %2 --> %3 (%4)")
844 .arg(jobScheduleString(line)).toLatin1().data());
QString i18n(const char *text, const TYPE &arg...)
char * toString(const EngineQuery &query)
Ekos is an advanced Astrophotography tool for Linux.
@ SCHEDJOB_ABORTED
Job encountered a transitory issue while processing, and will be rescheduled.
@ SCHEDJOB_INVALID
Job has an incorrect configuration, and cannot proceed.
@ SCHEDJOB_ERROR
Job encountered a fatal issue while processing, and must be reset manually.
@ SCHEDJOB_COMPLETE
Job finished all required captures.
@ SCHEDJOB_EVALUATION
Job is being evaluated.
@ SCHEDJOB_SCHEDULED
Job was evaluated, and has a schedule.
@ SCHEDJOB_BUSY
Job is being processed.
@ SCHEDJOB_IDLE
Job was just created, and is not evaluated yet.
QAction * next(const QObject *recvr, const char *slot, QObject *parent)
QCA_EXPORT Logger * logger()
QDateTime addSecs(qint64 s) const const
bool isValid() const const
qint64 secsTo(const QDateTime &other) const const
QString toString(QStringView format, QCalendar cal) const const
qint64 elapsed() const const
iterator find(const Key &key)
void append(QList< T > &&value)
qsizetype indexOf(const AT &value, qsizetype from) const const
qsizetype size() const const
QString arg(Args &&... args) const const
QByteArray toLatin1() const const