From fb32c85c7b485b2bb100ae5506a3cb9d7aaf9241 Mon Sep 17 00:00:00 2001 From: carlyin Date: Fri, 3 Jan 2025 15:07:05 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20taskControl=E7=9A=84loop=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E5=A4=AA=E4=B9=85=E5=AF=BC=E8=87=B4Consumer=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=B1=A0=E8=A2=AB=E5=8D=A0=E6=BB=A1=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#11352?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/process/engine/atom/IAtomTask.kt | 27 +++++-------------- .../engine/control/VmOperateTaskGenerator.kt | 9 ++++++- .../tencent/devops/worker/common/Runner.kt | 2 +- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt index 7706dd959265..0d7489605bab 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt @@ -39,6 +39,7 @@ import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.RunCondition import com.tencent.devops.common.pipeline.type.docker.DockerDispatchType import com.tencent.devops.process.engine.common.Timeout +import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.engine.pojo.PipelineBuildTask import org.slf4j.LoggerFactory import java.util.concurrent.TimeUnit @@ -101,26 +102,12 @@ interface IAtomTask { // 未结束？检查是否超时 if (!atomResponse.buildStatus.isFinish()) { val startTime = task.startTime?.timestampmilli() ?: 0L - val timeoutMills: Long = - if (param is Element) { - val additionalOptions = param.additionalOptions - Timeout.transMinuteTimeoutToMills(additionalOptions?.timeout?.toInt()) - } else if (param is NormalContainer) { - Timeout.transMinuteTimeoutToMills( - (param.jobControlOption?.prepareTimeout ?: Timeout.DEFAULT_PREPARE_MINUTES) - ) - } else if (param is VMBuildContainer) { - // docker 构建机要求10分钟内超时 - if (param.dispatchType is DockerDispatchType || !param.dockerBuildVersion.isNullOrBlank()) { - Timeout.transMinuteTimeoutToMills( - (param.jobControlOption?.prepareTimeout ?: Timeout.DEFAULT_PREPARE_MINUTES) - ) - } else { - Timeout.transMinuteTimeoutToMills(param.jobControlOption?.timeout) - } - } else { - 0L - } + var timeout = task.additionalOptions?.timeout?.toInt() + if (timeout == null && VMUtils.isVMTask(task.taskId)) { + // 如果timeout为空且task为开关机插件任务，则给timeout赋默认值 + timeout = Timeout.DEFAULT_PREPARE_MINUTES + } + val timeoutMills = Timeout.transMinuteTimeoutToMills(timeout) val runCondition = task.additionalOptions?.runCondition if (timeoutMills > 0 && System.currentTimeMillis() - startTime >= timeoutMills) { logger.info( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt index e2575cf5a859..d803f7ec161c 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt @@ -28,6 +28,7 @@ package com.tencent.devops.process.engine.control import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.container.NormalContainer import com.tencent.devops.common.pipeline.container.VMBuildContainer import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.EnvControlTaskType @@ -90,6 +91,7 @@ class VmOperateTaskGenerator { val taskType: String val taskName: String val taskAtom: String + var timeout: Long? = null if (container is VMBuildContainer) { val buildType = container.dispatchType?.buildType()?.name ?: BuildType.DOCKER.name val baseOS = container.baseOS.name @@ -97,14 +99,19 @@ class VmOperateTaskGenerator { taskType = EnvControlTaskType.VM.name taskName = "Prepare_Job#${container.id!!}" taskAtom = START_VM_TASK_ATOM + timeout = container.jobControlOption?.prepareTimeout?.toLong() } else { atomCode = START_NORMAL_TASK_ATOM taskType = EnvControlTaskType.NORMAL.name taskName = "Prepare_Job#${container.id!!}(N)" taskAtom = START_NORMAL_TASK_ATOM + if (container is NormalContainer) { + timeout = container.jobControlOption?.prepareTimeout?.toLong() + } } val additionalOptions = ElementAdditionalOptions( - runCondition = RunCondition.PRE_TASK_FAILED_BUT_CANCEL + runCondition = RunCondition.PRE_TASK_FAILED_BUT_CANCEL, + timeout = timeout ) return PipelineBuildTask( projectId = projectId, diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt index 9fa3abb45827..ab8773e44142 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt @@ -75,7 +75,7 @@ import kotlin.system.exitProcess object Runner { - private const val maxSleepStep = 50L + private const val maxSleepStep = 80L private const val windows = 5L private const val millsStep = 100L private val logger = LoggerFactory.getLogger(Runner::class.java) From 5f032fab90e6f62d37f32435bb7203f34f26510a Mon Sep 17 00:00:00 2001 From: carlyin Date: Fri, 3 Jan 2025 20:17:34 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20taskControl=E7=9A=84loop=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E5=A4=AA=E4=B9=85=E5=AF=BC=E8=87=B4Consumer=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=B1=A0=E8=A2=AB=E5=8D=A0=E6=BB=A1=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#11352?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/vmbuild/EngineVMBuildService.kt | 44 +++++++++---------- .../process/engine/atom/TaskAtomService.kt | 30 +++++++++---- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt index 783445ad2992..1738cca09c38 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt @@ -450,8 +450,10 @@ class EngineVMBuildService @Autowired(required = false) constructor( return false } val finalBuildStatus = getFinalBuildStatus(buildStatus, buildId, vmSeqId, startUpVMTask) - // 如果是完成状态，则更新构建机启动插件的状态 - if (finalBuildStatus.isFinish()) { + + // #2043 上报启动构建机状态时，重新刷新开始时间，以防止调度的耗时占用了Job的超时时间 + if (!startUpVMTask.status.isFinish() && finalBuildStatus.isFinish()) { // #2043 构建机当前启动状态是未结束状态，才进行刷新开始时间 + // 如果是完成状态，则更新构建机启动插件的状态 pipelineTaskService.updateTaskStatus( task = startUpVMTask, userId = startUpVMTask.starter, @@ -460,27 +462,23 @@ class EngineVMBuildService @Autowired(required = false) constructor( errorCode = errorCode, errorMsg = errorMsg ) - - // #2043 上报启动构建机状态时，重新刷新开始时间，以防止调度的耗时占用了Job的超时时间 - if (!startUpVMTask.status.isFinish()) { // #2043 构建机当前启动状态是未结束状态，才进行刷新开始时间 - pipelineContainerService.updateContainerStatus( - projectId = projectId, - buildId = buildId, - stageId = startUpVMTask.stageId, - containerId = startUpVMTask.containerId, - startTime = LocalDateTime.now(), - endTime = null, - buildStatus = BuildStatus.RUNNING - ) - containerBuildRecordService.containerStarted( - projectId = projectId, - pipelineId = pipelineId, - buildId = buildId, - containerId = vmSeqId, - executeCount = startUpVMTask.executeCount ?: 1, - containerBuildStatus = finalBuildStatus - ) - } + pipelineContainerService.updateContainerStatus( + projectId = projectId, + buildId = buildId, + stageId = startUpVMTask.stageId, + containerId = startUpVMTask.containerId, + startTime = LocalDateTime.now(), + endTime = null, + buildStatus = BuildStatus.RUNNING + ) + containerBuildRecordService.containerStarted( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + containerId = vmSeqId, + executeCount = startUpVMTask.executeCount ?: 1, + containerBuildStatus = finalBuildStatus + ) } // 失败的话就发终止事件 diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt index ca877cc41e70..9ea0c9a9f26e 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt @@ -344,15 +344,27 @@ class TaskAtomService @Autowired(required = false) constructor( private fun log(atomResponse: AtomResponse, task: PipelineBuildTask, stopFlag: Boolean) { if (atomResponse.buildStatus.isFinish()) { - buildLogPrinter.addLine( - buildId = task.buildId, - message = "Task [${task.taskName}] ${atomResponse.buildStatus}!", - tag = task.taskId, - containerHashId = task.containerHashId, - executeCount = task.executeCount ?: 1, - jobId = null, - stepId = task.stepId - ) + if (atomResponse.errorCode != null) { + buildLogPrinter.addErrorLine( + buildId = task.buildId, + message = "Task [${task.taskName}] ${atomResponse.buildStatus} (${atomResponse.errorMsg})!", + tag = task.taskId, + containerHashId = task.containerHashId, + executeCount = task.executeCount ?: 1, + jobId = null, + stepId = task.stepId + ) + } else { + buildLogPrinter.addLine( + buildId = task.buildId, + message = "Task [${task.taskName}] ${atomResponse.buildStatus}!", + tag = task.taskId, + containerHashId = task.containerHashId, + executeCount = task.executeCount ?: 1, + jobId = null, + stepId = task.stepId + ) + } } else { if (stopFlag) { buildLogPrinter.addLine( From 4eb97e9a3cb87bc8168ed6472f395d2cee4cc4af Mon Sep 17 00:00:00 2001 From: carlyin Date: Mon, 6 Jan 2025 10:30:58 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20taskControl=E7=9A=84loop=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E5=A4=AA=E4=B9=85=E5=AF=BC=E8=87=B4Consumer=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=B1=A0=E8=A2=AB=E5=8D=A0=E6=BB=A1=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#11352?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/process/engine/atom/IAtomTask.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt index 0d7489605bab..e42d45985195 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/atom/IAtomTask.kt @@ -32,12 +32,8 @@ import com.tencent.devops.common.api.pojo.ErrorType import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.event.enums.ActionType -import com.tencent.devops.common.pipeline.container.NormalContainer -import com.tencent.devops.common.pipeline.container.VMBuildContainer import com.tencent.devops.common.pipeline.enums.BuildStatus -import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.RunCondition -import com.tencent.devops.common.pipeline.type.docker.DockerDispatchType import com.tencent.devops.process.engine.common.Timeout import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.engine.pojo.PipelineBuildTask From 1fe5cd45b16ead60fc07c431c21a25ec87d6e4a9 Mon Sep 17 00:00:00 2001 From: carlyin Date: Tue, 7 Jan 2025 12:17:43 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20taskControl=E7=9A=84loop=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E5=A4=AA=E4=B9=85=E5=AF=BC=E8=87=B4Consumer=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=B1=A0=E8=A2=AB=E5=8D=A0=E6=BB=A1=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#11352?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/engine/control/VmOperateTaskGenerator.kt | 10 +++++++--- .../engine/atom/vm/DispatchVMStartupTaskAtom.kt | 2 +- .../devops/process/engine/control/TaskControl.kt | 5 ++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt index d803f7ec161c..facd7848b6d5 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt @@ -92,6 +92,7 @@ class VmOperateTaskGenerator { val taskName: String val taskAtom: String var timeout: Long? = null + var timeoutVar: String? = null if (container is VMBuildContainer) { val buildType = container.dispatchType?.buildType()?.name ?: BuildType.DOCKER.name val baseOS = container.baseOS.name @@ -99,19 +100,22 @@ class VmOperateTaskGenerator { taskType = EnvControlTaskType.VM.name taskName = "Prepare_Job#${container.id!!}" taskAtom = START_VM_TASK_ATOM - timeout = container.jobControlOption?.prepareTimeout?.toLong() + timeout = container.jobControlOption?.timeout?.toLong() + timeoutVar = container.jobControlOption?.timeoutVar } else { atomCode = START_NORMAL_TASK_ATOM taskType = EnvControlTaskType.NORMAL.name taskName = "Prepare_Job#${container.id!!}(N)" taskAtom = START_NORMAL_TASK_ATOM if (container is NormalContainer) { - timeout = container.jobControlOption?.prepareTimeout?.toLong() + timeout = container.jobControlOption?.timeout?.toLong() + timeoutVar = container.jobControlOption?.timeoutVar } } val additionalOptions = ElementAdditionalOptions( runCondition = RunCondition.PRE_TASK_FAILED_BUT_CANCEL, - timeout = timeout + timeout = timeout, + timeoutVar = timeoutVar ) return PipelineBuildTask( projectId = projectId, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt index 3aedb903f5f3..a95960a65382 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt @@ -230,7 +230,7 @@ class DispatchVMStartupTaskAtom @Autowired constructor( ) dispatch(task, pipelineInfo!!, param, vmNames, container!!, ignoreEnvAgentIds, pipelineAuthorizer) logger.info("[$buildId]|STARTUP_VM|VM=${param.baseOS}-$vmNames($vmSeqId)|Dispatch startup") - return AtomResponse(BuildStatus.CALL_WAITING) + return AtomResponse(BuildStatus.RUNNING) } private fun dispatch( diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/TaskControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/TaskControl.kt index 28345c393c1a..79ee6108d490 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/TaskControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/TaskControl.kt @@ -40,6 +40,7 @@ import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatch import com.tencent.devops.process.engine.atom.AtomResponse import com.tencent.devops.process.engine.atom.TaskAtomService import com.tencent.devops.process.engine.common.BS_ATOM_STATUS_REFRESH_DELAY_MILLS +import com.tencent.devops.process.engine.common.VMUtils.getStartVmLabel import com.tencent.devops.process.engine.control.lock.ContainerIdLock import com.tencent.devops.process.engine.pojo.PipelineBuildTask import com.tencent.devops.process.engine.pojo.event.PipelineBuildAtomTaskEvent @@ -175,7 +176,9 @@ class TaskControl @Autowired constructor( if (buildStatus.isRunning()) { // 仍然在运行中--没有结束的 // 如果是要轮循的才需要定时消息轮循 - loopDispatch(buildTask = buildTask) + if (!taskId.startsWith(getStartVmLabel())) { + loopDispatch(buildTask = buildTask) + } } else { pipelineBuildTaskService.finishTask( buildTask = buildTask,