Reset terminal states before updating (#3386)

* reset terminal states updating Signed-off-by: Mohamed Abdelfatah <[email protected]> * remove returned Signed-off-by: Mohamed Abdelfatah <[email protected]> * comment update Signed-off-by: Mohamed Abdelfatah <[email protected]> * better naming Signed-off-by: Mohamed Abdelfatah <[email protected]> * better checking order Signed-off-by: Mohamed Abdelfatah <[email protected]> --------- Signed-off-by: Mohamed Abdelfatah <[email protected]>
armadaproject · Feb 8, 2024 · 1fd83fd · 1fd83fd
1 parent 4b323df
commit 1fd83fd
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 10 deletions.
diff --git a/internal/scheduler/jobdb/job_run.go b/internal/scheduler/jobdb/job_run.go
@@ -443,6 +443,17 @@ func (run *JobRun) Created() int64 {
 	return run.created
 }
 
+// WithoutTerminal returns a copy of the job run with the terminal states set to false to prevent conflicts.
+// terminal states are {succeeded, failed, cancelled, preempted}.
+func (run *JobRun) WithoutTerminal() *JobRun {
+	run = run.DeepCopy()
+	run.succeeded = false
+	run.failed = false
+	run.cancelled = false
+	run.preempted = false
+	return run
+}
+
 // InTerminalState returns true if the JobRun is in a terminal state
 func (run *JobRun) InTerminalState() bool {
 	return run.succeeded || run.failed || run.cancelled || run.returned

diff --git a/internal/scheduler/jobdb/reconciliation.go b/internal/scheduler/jobdb/reconciliation.go
@@ -182,21 +182,25 @@ func (jobDb *JobDb) reconcileRunDifferences(jobRun *JobRun, jobRepoRun *database
 			jobRun = jobRun.WithRunning(true).WithRunningTime(jobRepoRun.RunningTimestamp)
 			rst.Running = true
 		}
-		if jobRepoRun.Succeeded && !jobRun.Succeeded() {
-			jobRun = jobRun.WithSucceeded(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp)
-			rst.Succeeded = true
-		}
-		if jobRepoRun.Failed && !jobRun.Failed() {
-			jobRun = jobRun.WithFailed(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp)
-			rst.Failed = true
+		if jobRepoRun.Preempted && !jobRun.Preempted() {
+			jobRun = jobRun.WithoutTerminal()
+			jobRun = jobRun.WithPreempted(true).WithRunning(false).WithPreemptedTime(jobRepoRun.PreemptedTimestamp)
+			rst.Preempted = true
 		}
 		if jobRepoRun.Cancelled && !jobRun.Cancelled() {
+			jobRun = jobRun.WithoutTerminal()
 			jobRun = jobRun.WithCancelled(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp)
 			rst.Cancelled = true
 		}
-		if jobRepoRun.Preempted && !jobRun.Preempted() {
-			jobRun = jobRun.WithPreempted(true).WithRunning(false).WithPreemptedTime(jobRepoRun.TerminatedTimestamp)
-			rst.Preempted = true
+		if jobRepoRun.Failed && !jobRun.Failed() {
+			jobRun = jobRun.WithoutTerminal()
+			jobRun = jobRun.WithFailed(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp)
+			rst.Failed = true
+		}
+		if jobRepoRun.Succeeded && !jobRun.Succeeded() {
+			jobRun = jobRun.WithoutTerminal()
+			jobRun = jobRun.WithSucceeded(true).WithRunning(false).WithTerminatedTime(jobRepoRun.TerminatedTimestamp)
+			rst.Succeeded = true
 		}
 		if jobRepoRun.Returned && !jobRun.Returned() {
 			jobRun = jobRun.WithReturned(true).WithRunning(false)