From b4549f5f6f8a982f9f15e011105a95c05e26eae6 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 18 Dec 2023 17:50:37 +0800 Subject: [PATCH 01/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dao/repository/TurboPlanRepository.kt | 5 + .../devops/turbo/dto/TBSDaySummaryDto.kt | 30 +++++ .../devops/turbo/job/TBSDaySummaryJob.kt | 108 ++++++++++++++++++ .../com/tencent/devops/turbo/sdk/TBSSdkApi.kt | 39 ++++++- .../turbo/model/TTbsDaySummaryEntity.kt | 103 +++++++++++++++++ 5 files changed, 282 insertions(+), 3 deletions(-) create mode 100644 src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/TBSDaySummaryDto.kt create mode 100644 src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt create mode 100644 src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt index 0c4f322dd..66c0af1a7 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt @@ -29,4 +29,9 @@ interface TurboPlanRepository : MongoRepository { * 根据项目id和编译加速方案名称和id判断是否存在 */ fun existsByProjectIdAndPlanNameAndIdNot(projectId: String, planName: String, planId: String): Boolean + + /** + * 根据方案id集合批量查询 + */ + fun findByIdIn(planIdSet: List) } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/TBSDaySummaryDto.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/TBSDaySummaryDto.kt new file mode 100644 index 000000000..7104f9f20 --- /dev/null +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/TBSDaySummaryDto.kt @@ -0,0 +1,30 @@ +package com.tencent.devops.turbo.dto + +import com.fasterxml.jackson.annotation.JsonProperty + +data class TBSDaySummaryDto( + val day: String, + + val user: String?, + + @JsonProperty("project_id") + val projectId: String, + + /** + * 加速时长(单位：秒) + */ + @JsonProperty("total_time") + val totalTime: Double, + + /** + * 加速时长*cpu核数(单位秒*核) + */ + @JsonProperty("total_time_with_cpu") + val totalTimeWithCpu: Double, + + /** + * 加速次数 + */ + @JsonProperty("total_record_number") + val totalRecordNumber: Double, +) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt new file mode 100644 index 000000000..ed2191c7c --- /dev/null +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -0,0 +1,108 @@ +package com.tencent.devops.turbo.job + +import com.tencent.devops.common.util.DateTimeUtils +import com.tencent.devops.common.util.JsonUtil +import com.tencent.devops.turbo.dao.repository.TurboEngineConfigRepository +import com.tencent.devops.turbo.dao.repository.TurboPlanRepository +import com.tencent.devops.turbo.dto.TBSDaySummaryDto +import com.tencent.devops.turbo.model.TTbsDaySummaryEntity +import com.tencent.devops.turbo.sdk.TBSSdkApi +import org.quartz.Job +import org.quartz.JobExecutionContext +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import java.time.LocalDate +import java.time.LocalDateTime + +@Suppress("SpringJavaAutowiredMembersInspection") +class TBSDaySummaryJob @Autowired constructor( + private val turboEngineConfigRepository: TurboEngineConfigRepository, + private val turboPlanRepository: TurboPlanRepository +) : Job { + + companion object { + private val logger = LoggerFactory.getLogger(this::class.java) + private const val PAGE_SIZE = 3000 + } + + /** + * 执行入口 + */ + override fun execute(context: JobExecutionContext) { + logger.info("TBS day summary job start executing: ${JsonUtil.toJson(context.jobDetail)}") + + val jobParam = context.jobDetail.jobDataMap + val statisticsDateStr = if (jobParam.containsKey("statisticsDate")) { + jobParam["statisticsDate"] as String + } else { + // 统计昨天 + val statLocalDate = LocalDate.now().minusDays(1) + DateTimeUtils.localDate2DateStr(statLocalDate) + } + + val engineConfigEntities = turboEngineConfigRepository.findAll() + engineConfigEntities.forEach { engineConfig -> + val daySummaryDtoList = TBSSdkApi.queryTbsDaySummary( + engineCode = engineConfig.engineCode, + queryParam = mapOf( + "day" to statisticsDateStr + ) + ) + + logger.info("daySummaryDtoList size: ${daySummaryDtoList.size}") + if (daySummaryDtoList.isEmpty()) { + logger.warn("queryTbsDaySummary result is empty! engineCode: ${engineConfig.engineCode}") + return@forEach + } + + val summaryEntityList = this.dto2SummaryEntityList(daySummaryList = daySummaryDtoList) + val planIdEntityMap = summaryEntityList.associateBy { it.planId } + + val planIdsList = planIdEntityMap.keys.chunked(PAGE_SIZE) + for (planIds in planIdsList) { + val turboPlanList = turboPlanRepository.findByIdIn(planIds) + + } + } + } + + /** + * 把TBS的接口数据整理成entity + */ + private fun dto2SummaryEntityList(daySummaryList: List): List { + val summaryEntities = mutableListOf() + + daySummaryList.forEach { summary -> + // distcc与其它不一样，它的projectId就是planId + val planIdAndEngineCode = summary.projectId + + val planId: String + val engineCode: String + + // "60d54b87a26123319d011bob_cc" + if (planIdAndEngineCode.contains("_")) { + val stringArr = planIdAndEngineCode.split("_") + planId = stringArr[0] + engineCode = if (stringArr[1] == "cc") "disttask-cc" else if (stringArr[1] == "ue4") "disttask-ue4" + else stringArr[1] + } else { + planId = planIdAndEngineCode + engineCode = "distcc" + } + + val entity = TTbsDaySummaryEntity( + day = summary.day, + engineCode = engineCode, + planId = planId, + user = if (engineCode == "disttask-ue4") summary.user else null, + totalTime = summary.totalTime, + totalTimeWithCpu = summary.totalTimeWithCpu, + totalRecordNumber = summary.totalRecordNumber, + createdDate = LocalDateTime.now() + ) + summaryEntities.add(entity) + } + return summaryEntities + } + +} \ No newline at end of file diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index 56a7fb143..5ad796e12 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -9,6 +9,7 @@ import com.tencent.devops.turbo.config.TBSProperties import com.tencent.devops.turbo.dto.DistccRequestBody import com.tencent.devops.turbo.dto.DistccResponse import com.tencent.devops.turbo.dto.ParamEnumDto +import com.tencent.devops.turbo.dto.TBSDaySummaryDto import com.tencent.devops.turbo.dto.TBSTurboStatDto import com.tencent.devops.turbo.dto.WhiteListDto import com.tencent.devops.web.util.SpringContextHolder @@ -122,13 +123,17 @@ object TBSSdkApi { queryParam: Map = mutableMapOf(), jsonBody: String = "", headers: MutableMap = mutableMapOf(), - method: String = "GET" + method: String = "GET", + customPath: String? = null ): String { val requestBody = jsonBody.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) val properties = SpringContextHolder.getBean() - val customPath = + val templatePath = properties.urlTemplate!!.replace("{engine}", engineCode).replace("{resource_type}", resourceName) - var url = "${properties.rootPath}/$customPath" + var url = "${properties.rootPath}/$templatePath" + if (customPath.isNullOrBlank()) { + url = url.plus(customPath) + } if (pathParam.isNotEmpty()) { pathParam.forEach { url = url.plus("/$it") @@ -207,4 +212,32 @@ object TBSSdkApi { JsonUtil.to(responseStr, object : TypeReference>>() {}).data } } + + /** + * 获取TBS每日汇总统计 + */ + fun queryTbsDaySummary(engineCode: String, queryParam: Map): List { + val responseStr = if (engineCode.contains("disttask")) { + // tbs后台接口路径处理 + tbsCommonRequest( + engineCode = "disttask", + resourceName = "summary", + queryParam = queryParam, + customPath = if (engineCode.contains("ue4"))"/groupbyuser/scene/ue4" else null + ) + } else { + tbsCommonRequest( + engineCode = "distcc", + resourceName = "summary", + queryParam = queryParam + ) + } + val response = JsonUtil.to(responseStr, object : TypeReference>>() {}) + if (response.code != 0 || !response.result) { + throw TurboException(errorCode = TURBO_THIRDPARTY_SYSTEM_FAIL, errorMessage = "fail to invoke request: " + + response.message + ) + } + return response.data ?: listOf() + } } diff --git a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt new file mode 100644 index 000000000..54eb1869d --- /dev/null +++ b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt @@ -0,0 +1,103 @@ +package com.tencent.devops.turbo.model + +import org.springframework.data.annotation.Id +import org.springframework.data.mongodb.core.mapping.Document +import org.springframework.data.mongodb.core.mapping.Field +import java.time.LocalDateTime + +/** + * 每日运营数据统计实体类 + */ +@Document(collection = "t_tbs_day_summary_entity") +data class TTbsDaySummaryEntity( + @Id + var id: String? = null, + /** + * 统计数据归属日期 + */ + @Field + var day: String?, + + /** + * 蓝盾项目英文id + */ + @Field("project_id") + var projectId: String? = null, + + /** + * 方案id + */ + @Field("plan_id") + var planId: String?, + + /** + * 方案创建人 + */ + @Field("plan_creator") + var planCreator: String? = null, + + /** + * UE加速活跃 Agent + * 项目+加速方案类型（即同一项目同一用户使用多个UE加速方案时不重复统计） + */ + @Field + var user: String? = null, + + /** + * 加速模式 + */ + @Field("engine_code") + var engineCode: String?, + + /** + * 加速时长(单位：秒) + */ + @Field("total_time") + var totalTime: Double?, + + /** + * 加速时长*cpu核数(单位秒*核) + */ + @Field("total_time_with_cpu") + var totalTimeWithCpu: Double?, + + /** + * 加速次数 + */ + @Field("total_record_number") + var totalRecordNumber: Double?, + + /** + * 项目所属运营产品 + */ + @Field("product_id") + var productId: Int? = null, + + /** + * BG信息 + */ + @Field("bg_name") + var bgName: String? = null, + @Field("bg_id") + var bgId: Int? = null, + + /** + * 部门信息 + */ + @Field("dept_name") + var deptName: String? = null, + @Field("dept_id") + var deptId: Int? = null, + + /** + * 项目所属组织架构 + */ + @Field("org_path") + var orgPath: String? = null, + + /** + * 本entity创建时间 + */ + @Field("created_date") + var createdDate: LocalDateTime +) From 235f83cff607afe7daf5cae1faa0496979352ae2 Mon Sep 17 00:00:00 2001 From: RJ Date: Wed, 20 Dec 2023 11:16:05 +0800 Subject: [PATCH 02/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dao/repository/TbsDaySummaryRepository.kt | 8 ++++ .../dao/repository/TurboPlanRepository.kt | 2 +- .../devops/turbo/job/TBSDaySummaryJob.kt | 42 +++++++++++++++++-- src/backend/turbo/build.gradle.kts | 6 ++- .../buildSrc/src/main/kotlin/Versions.kt | 2 +- .../turbo/model/TTbsDaySummaryEntity.kt | 12 ++++++ 6 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt new file mode 100644 index 000000000..86aeb6482 --- /dev/null +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt @@ -0,0 +1,8 @@ +package com.tencent.devops.turbo.dao.repository + +import com.tencent.devops.turbo.model.TTbsDaySummaryEntity +import org.springframework.data.mongodb.repository.MongoRepository +import org.springframework.stereotype.Repository + +@Repository +interface TbsDaySummaryRepository : MongoRepository diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt index 66c0af1a7..3ebc5b6a6 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt @@ -33,5 +33,5 @@ interface TurboPlanRepository : MongoRepository { /** * 根据方案id集合批量查询 */ - fun findByIdIn(planIdSet: List) + fun findByIdIn(planIdSet: List): List } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index ed2191c7c..d0cf9c14e 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -1,7 +1,10 @@ package com.tencent.devops.turbo.job +import com.tencent.devops.common.client.Client import com.tencent.devops.common.util.DateTimeUtils import com.tencent.devops.common.util.JsonUtil +import com.tencent.devops.project.pojo.ProjectVO +import com.tencent.devops.turbo.dao.repository.TbsDaySummaryRepository import com.tencent.devops.turbo.dao.repository.TurboEngineConfigRepository import com.tencent.devops.turbo.dao.repository.TurboPlanRepository import com.tencent.devops.turbo.dto.TBSDaySummaryDto @@ -16,6 +19,8 @@ import java.time.LocalDateTime @Suppress("SpringJavaAutowiredMembersInspection") class TBSDaySummaryJob @Autowired constructor( + private val client: Client, + private val tbsDaySummaryRepository: TbsDaySummaryRepository, private val turboEngineConfigRepository: TurboEngineConfigRepository, private val turboPlanRepository: TurboPlanRepository ) : Job { @@ -40,6 +45,8 @@ class TBSDaySummaryJob @Autowired constructor( DateTimeUtils.localDate2DateStr(statLocalDate) } + val projectVOMap = mutableMapOf() + val engineConfigEntities = turboEngineConfigRepository.findAll() engineConfigEntities.forEach { engineConfig -> val daySummaryDtoList = TBSSdkApi.queryTbsDaySummary( @@ -55,14 +62,44 @@ class TBSDaySummaryJob @Autowired constructor( return@forEach } + // 把TBS的接口数据整理成entity val summaryEntityList = this.dto2SummaryEntityList(daySummaryList = daySummaryDtoList) val planIdEntityMap = summaryEntityList.associateBy { it.planId } + // 取出项目ID集合用于获取项目组织架构信息 + val projectIdSet = summaryEntityList.map { it.projectId!! }.toSet() + val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) + if (notInProjectMapKeySet.isEmpty()) { +// val projectVO = client.get(ServiceProjectResource::class.java).get("") // TODO 换成批量 +// val projectVOMap = listOf().associateBy { it.projectId } + val resultMap = mapOf() + projectVOMap.putAll(resultMap) + } + val planIdsList = planIdEntityMap.keys.chunked(PAGE_SIZE) for (planIds in planIdsList) { val turboPlanList = turboPlanRepository.findByIdIn(planIds) - + logger.info("turboPlanRepository.findByIdIn result size: ${turboPlanList.size}") + if (turboPlanList.isEmpty()) { + logger.warn("turboPlanList is empty! continue") + return@forEach + } + + turboPlanList.forEach { + val summaryEntity = planIdEntityMap[it.id] + summaryEntity?.planCreator = it.createdBy + summaryEntity?.planName = it.planName + summaryEntity?.projectId = it.projectId + summaryEntity?.projectName = projectVOMap[it.projectId]?.projectName + summaryEntity?.bgName = projectVOMap[it.projectId]?.bgName + summaryEntity?.bgId = projectVOMap[it.projectId]?.bgId?.toInt() + summaryEntity?.deptName = projectVOMap[it.projectId]?.deptName + summaryEntity?.deptId = projectVOMap[it.projectId]?.deptId?.toInt() + summaryEntity?.productId = projectVOMap[it.projectId]?.productId + } } + + tbsDaySummaryRepository.saveAll(summaryEntityList) } } @@ -104,5 +141,4 @@ class TBSDaySummaryJob @Autowired constructor( } return summaryEntities } - -} \ No newline at end of file +} diff --git a/src/backend/turbo/build.gradle.kts b/src/backend/turbo/build.gradle.kts index a55513976..77ab6a607 100644 --- a/src/backend/turbo/build.gradle.kts +++ b/src/backend/turbo/build.gradle.kts @@ -1,6 +1,6 @@ import com.tencent.devops.utils.findPropertyOrEmpty plugins { - id("com.tencent.devops.boot") version "0.0.6" + id("com.tencent.devops.boot") version "0.0.7" id("org.owasp.dependencycheck") version "7.1.0.1" } @@ -14,6 +14,10 @@ allprojects { val property = project.findPropertyOrEmpty("devops.assemblyMode").trim() + repositories { + maven(url = "https://mirrors.tencent.com/repository/maven/bkdevops_maven") + } + configurations.forEach { it.exclude(group = "org.slf4j", module = "log4j-over-slf4j") it.exclude(group = "org.slf4j", module = "slf4j-log4j12") diff --git a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt index 7bf6cc658..ecec9c37d 100644 --- a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt @@ -4,7 +4,7 @@ object Versions { const val bouncyCastleVersion = "1.70" const val swaggerVersion = "1.6.2" const val guavaVersion = "29.0-jre" - const val ciVersion = "1.5.0-RC.2-RELEASE" + const val ciVersion = "1.14.0-SNAPSHOT" const val commonIo = "2.7" const val jjwtVersion = "0.9.0" const val ciAuthVersion = "1.8.0-RC.3-RELEASE" diff --git a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt index 54eb1869d..2d5371490 100644 --- a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt +++ b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt @@ -24,12 +24,24 @@ data class TTbsDaySummaryEntity( @Field("project_id") var projectId: String? = null, + /** + * 项目名称 + */ + @Field("project_name") + var projectName: String? = null, + /** * 方案id */ @Field("plan_id") var planId: String?, + /** + * 方案名称 + */ + @Field("plan_name") + var planName: String? = null, + /** * 方案创建人 */ From 0cd76bdb7fecb96d9ecf2590cb6eef44fb7daeaf Mon Sep 17 00:00:00 2001 From: RJ Date: Wed, 20 Dec 2023 13:08:18 +0800 Subject: [PATCH 03/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/turbo/job/TBSDaySummaryJob.kt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index d0cf9c14e..24bd3ca03 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -3,6 +3,7 @@ package com.tencent.devops.turbo.job import com.tencent.devops.common.client.Client import com.tencent.devops.common.util.DateTimeUtils import com.tencent.devops.common.util.JsonUtil +import com.tencent.devops.project.api.service.ServiceProjectResource import com.tencent.devops.project.pojo.ProjectVO import com.tencent.devops.turbo.dao.repository.TbsDaySummaryRepository import com.tencent.devops.turbo.dao.repository.TurboEngineConfigRepository @@ -70,9 +71,16 @@ class TBSDaySummaryJob @Autowired constructor( val projectIdSet = summaryEntityList.map { it.projectId!! }.toSet() val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) if (notInProjectMapKeySet.isEmpty()) { -// val projectVO = client.get(ServiceProjectResource::class.java).get("") // TODO 换成批量 -// val projectVOMap = listOf().associateBy { it.projectId } - val resultMap = mapOf() + val resultMap = mutableMapOf() + for (it in notInProjectMapKeySet) { + // TODO 换成批量 + val projectVOResult = client.get(ServiceProjectResource::class.java).get(it) + if (projectVOResult.isNotOk() || projectVOResult.data == null) { + logger.error("ServiceProjectResource#get request is not ok! project id: $it") + continue + } + resultMap[it] = projectVOResult.data!! + } projectVOMap.putAll(resultMap) } @@ -82,7 +90,7 @@ class TBSDaySummaryJob @Autowired constructor( logger.info("turboPlanRepository.findByIdIn result size: ${turboPlanList.size}") if (turboPlanList.isEmpty()) { logger.warn("turboPlanList is empty! continue") - return@forEach + continue } turboPlanList.forEach { @@ -100,7 +108,9 @@ class TBSDaySummaryJob @Autowired constructor( } tbsDaySummaryRepository.saveAll(summaryEntityList) + logger.info("save summary entity size: ${summaryEntityList.size}") } + logger.info("TBS day summary job execution completed!") } /** From 8db325d6e8eb2259f5a07b7e108b4985f6b35608 Mon Sep 17 00:00:00 2001 From: RJ Date: Wed, 20 Dec 2023 15:17:19 +0800 Subject: [PATCH 04/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/turbo/job/TBSDaySummaryJob.kt | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 24bd3ca03..5c64801e2 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -70,18 +70,10 @@ class TBSDaySummaryJob @Autowired constructor( // 取出项目ID集合用于获取项目组织架构信息 val projectIdSet = summaryEntityList.map { it.projectId!! }.toSet() val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) - if (notInProjectMapKeySet.isEmpty()) { - val resultMap = mutableMapOf() - for (it in notInProjectMapKeySet) { - // TODO 换成批量 - val projectVOResult = client.get(ServiceProjectResource::class.java).get(it) - if (projectVOResult.isNotOk() || projectVOResult.data == null) { - logger.error("ServiceProjectResource#get request is not ok! project id: $it") - continue - } - resultMap[it] = projectVOResult.data!! - } - projectVOMap.putAll(resultMap) + // 获取项目信息清单 + val projectVOList = this.getProjectVOListByProjectIds(notInProjectMapKeySet.toList()) + if (projectVOList.isNotEmpty()) { + projectVOMap.putAll(projectVOList.associateBy { it.projectId }) } val planIdsList = planIdEntityMap.keys.chunked(PAGE_SIZE) @@ -151,4 +143,20 @@ class TBSDaySummaryJob @Autowired constructor( } return summaryEntities } + + /** + * 根据项目id获取项目信息 + */ + private fun getProjectVOListByProjectIds(projectIds: List): List { + var list = emptyList() + if (projectIds.isNotEmpty()) { + val result = client.get(ServiceProjectResource::class.java).listByProjectCodeList(projectIds) + if (result.isNotOk() || result.data == null) { + logger.error("ServiceProjectResource#get request is failed!") + return list + } + list = result.data!! + } + return list + } } From 494000d996d2cc2f022d88d60e6c90c9bd984a5c Mon Sep 17 00:00:00 2001 From: RJ Date: Thu, 21 Dec 2023 10:12:39 +0800 Subject: [PATCH 05/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/turbo/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/turbo/build.gradle.kts b/src/backend/turbo/build.gradle.kts index 77ab6a607..3f5834edb 100644 --- a/src/backend/turbo/build.gradle.kts +++ b/src/backend/turbo/build.gradle.kts @@ -22,6 +22,7 @@ allprojects { it.exclude(group = "org.slf4j", module = "log4j-over-slf4j") it.exclude(group = "org.slf4j", module = "slf4j-log4j12") it.exclude(group = "org.slf4j", module = "slf4j-nop") + it.exclude(group = "org.bouncycastle", module = "bcutil-jdk15on") if (project.name.startsWith("boot-")) { when (com.tencent.devops.enums.AssemblyMode.ofValueOrDefault(property)) { com.tencent.devops.enums.AssemblyMode.CONSUL -> { From 88ebcdb71db83c4c29d41973144104310562d81f Mon Sep 17 00:00:00 2001 From: RJ Date: Thu, 21 Dec 2023 11:53:36 +0800 Subject: [PATCH 06/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/turbo/build.gradle.kts | 10 +++++++++- src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/backend/turbo/build.gradle.kts b/src/backend/turbo/build.gradle.kts index 3f5834edb..fdbacb62b 100644 --- a/src/backend/turbo/build.gradle.kts +++ b/src/backend/turbo/build.gradle.kts @@ -23,6 +23,7 @@ allprojects { it.exclude(group = "org.slf4j", module = "slf4j-log4j12") it.exclude(group = "org.slf4j", module = "slf4j-nop") it.exclude(group = "org.bouncycastle", module = "bcutil-jdk15on") + it.exclude(group = "org.bouncycastle", module = "bcpkix-jdk15on") if (project.name.startsWith("boot-")) { when (com.tencent.devops.enums.AssemblyMode.ofValueOrDefault(property)) { com.tencent.devops.enums.AssemblyMode.CONSUL -> { @@ -43,7 +44,6 @@ allprojects { dependencies { dependency("javax.ws.rs:javax.ws.rs-api:${Versions.jaxrsVersion}") dependency("com.github.ulisesbocchio:jasypt-spring-boot-starter:${Versions.jasyptVersion}") - dependency("org.bouncycastle:bcprov-jdk15on:${Versions.bouncyCastleVersion}") dependency("com.google.guava:guava:${Versions.guavaVersion}") dependency("io.jsonwebtoken:jjwt:${Versions.jjwtVersion}") dependency("commons-io:commons-io:${Versions.commonIo}") @@ -56,6 +56,14 @@ allprojects { entry("micrometer-registry-prometheus") } dependency("com.squareup.okhttp3:okhttp:${Versions.okHttpVersion}") + dependencySet("org.bouncycastle:${Versions.bouncyCastleVersion}") { + entry("bcprov-jdk15on") + entry("bcprov-ext-jdk15on") + } + dependencySet("org.jetbrains.kotlin:${Versions.Kotlin}") { + entry("kotlin-stdlib-jdk8") + entry("kotlin-reflect") + } } } } diff --git a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt index ecec9c37d..81565d646 100644 --- a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt @@ -10,4 +10,5 @@ object Versions { const val ciAuthVersion = "1.8.0-RC.3-RELEASE" const val micrometerVersion = "1.6.6" const val okHttpVersion = "4.9.0" + const val Kotlin = "1.6.0" } From e201871bf0755658c45875011be9af1187eb38c2 Mon Sep 17 00:00:00 2001 From: RJ Date: Thu, 21 Dec 2023 15:12:06 +0800 Subject: [PATCH 07/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/turbo/build.gradle.kts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backend/turbo/build.gradle.kts b/src/backend/turbo/build.gradle.kts index fdbacb62b..3e0457d86 100644 --- a/src/backend/turbo/build.gradle.kts +++ b/src/backend/turbo/build.gradle.kts @@ -1,7 +1,8 @@ import com.tencent.devops.utils.findPropertyOrEmpty plugins { - id("com.tencent.devops.boot") version "0.0.7" + id("com.tencent.devops.boot") version "0.0.6" id("org.owasp.dependencycheck") version "7.1.0.1" + kotlin("jvm") version "1.6.0" } allprojects { @@ -63,6 +64,9 @@ allprojects { dependencySet("org.jetbrains.kotlin:${Versions.Kotlin}") { entry("kotlin-stdlib-jdk8") entry("kotlin-reflect") + entry("kotlin-stdlib") + entry("kotlin-stdlib-jdk7") + entry("kotlin-stdlib-common") } } } From 09f7b6840187d6f7da629493e2d0023438052144 Mon Sep 17 00:00:00 2001 From: RJ Date: Thu, 21 Dec 2023 16:52:22 +0800 Subject: [PATCH 08/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../turbo/controller/UserCustomScheduleTaskController.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt index 1f1f3da9a..7c4dca7ed 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt @@ -22,9 +22,9 @@ class UserCustomScheduleTaskController @Autowired constructor( projectId: String, customScheduleJobModel: CustomScheduleJobModel ): Boolean { - if (!turboAuthService.validatePlatformMember(projectId, user)) { - throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) - } +// if (!turboAuthService.validatePlatformMember(projectId, user)) { +// throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) +// } return customScheduleJobService.customScheduledJobAdd(customScheduleJobModel) } From c04c175e16caf1372d35a000eac37e9769441178 Mon Sep 17 00:00:00 2001 From: RJ Date: Thu, 21 Dec 2023 17:27:42 +0800 Subject: [PATCH 09/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../turbo/api/IUserCustomScheduleTaskController.kt | 4 ++-- .../controller/UserCustomScheduleTaskController.kt | 8 ++++---- .../turbo/service/CustomScheduleJobService.kt | 14 +++++++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt index 75cd38502..d8cb01d73 100644 --- a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt +++ b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt @@ -34,7 +34,7 @@ interface IUserCustomScheduleTaskController { @ApiParam(value = "编译加速模式信息", required = true) @RequestBody customScheduleJobModel: CustomScheduleJobModel - ): Boolean + ): Result @ApiOperation("触发定时任务执行") @GetMapping( @@ -51,5 +51,5 @@ interface IUserCustomScheduleTaskController { @ApiParam(value = "任务名称", required = true) @RequestParam(value = "jobName") jobName: String - ): String? + ): Result } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt index 7c4dca7ed..90263d255 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt @@ -21,17 +21,17 @@ class UserCustomScheduleTaskController @Autowired constructor( user: String, projectId: String, customScheduleJobModel: CustomScheduleJobModel - ): Boolean { + ): Result { // if (!turboAuthService.validatePlatformMember(projectId, user)) { // throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) // } - return customScheduleJobService.customScheduledJobAdd(customScheduleJobModel) + return Result.success(customScheduleJobService.customScheduledJobAdd(customScheduleJobModel)) } - override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): String? { + override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): Result { if (!turboAuthService.getAuthResult(projectId, user)) { throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) } - return customScheduleJobService.trigger(jobName) + return Result.success(customScheduleJobService.trigger(jobName)!!) } } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt index 139802c7f..3451b3f55 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt @@ -9,14 +9,17 @@ import org.quartz.JobBuilder import org.quartz.JobDataMap import org.quartz.JobKey import org.quartz.Scheduler +import org.quartz.SchedulerException import org.quartz.TriggerBuilder import org.quartz.TriggerKey import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired +import org.springframework.scheduling.quartz.SchedulerFactoryBean import org.springframework.stereotype.Service @Service class CustomScheduleJobService @Autowired constructor( + private val schedulerFactoryBean: SchedulerFactoryBean, private val scheduler: Scheduler ) { companion object { @@ -61,7 +64,7 @@ class CustomScheduleJobService @Autowired constructor( val triggerKey = TriggerKey.triggerKey(triggerName, triggerGroup) try { scheduler.scheduleJob(jobDetail, trigger) - } catch (e: Exception) { + } catch (e: SchedulerException) { logger.warn("schedule $triggerName job fail with scheduler exception!") trigger = trigger.triggerBuilder.withIdentity(triggerKey).withSchedule( CronScheduleBuilder.cronSchedule(cronExpression) @@ -78,13 +81,14 @@ class CustomScheduleJobService @Autowired constructor( /** * 触发job立即执行 */ - fun trigger(jobName: String): String? { + fun trigger(jobName: String): String { return try { + val defaultScheduler = schedulerFactoryBean.scheduler val jobKey = JobKey.jobKey(jobName, jobGroup) - scheduler.triggerJob(jobKey) - "trigger job [] successfully" + defaultScheduler.triggerJob(jobKey) + "trigger job [$jobName] successfully" } catch (e: Exception) { - e.message + e.message.toString() } } } From b818f17c961762f910a3076f34339ec0d2e703d9 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 11:29:58 +0800 Subject: [PATCH 10/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../turbo/api/IUserCustomScheduleTaskController.kt | 5 +++-- .../turbo/controller/UserCustomScheduleTaskController.kt | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt index d8cb01d73..c7557f8b2 100644 --- a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt +++ b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt @@ -3,6 +3,7 @@ package com.tencent.devops.turbo.api import com.tencent.devops.common.util.constants.AUTH_HEADER_DEVOPS_PROJECT_ID import com.tencent.devops.common.util.constants.AUTH_HEADER_DEVOPS_USER_ID import com.tencent.devops.turbo.pojo.CustomScheduleJobModel +import com.tencent.devops.api.pojo.Response import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -34,7 +35,7 @@ interface IUserCustomScheduleTaskController { @ApiParam(value = "编译加速模式信息", required = true) @RequestBody customScheduleJobModel: CustomScheduleJobModel - ): Result + ): Response @ApiOperation("触发定时任务执行") @GetMapping( @@ -51,5 +52,5 @@ interface IUserCustomScheduleTaskController { @ApiParam(value = "任务名称", required = true) @RequestParam(value = "jobName") jobName: String - ): Result + ): Response } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt index 90263d255..e675fb97f 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt @@ -1,5 +1,6 @@ package com.tencent.devops.turbo.controller +import com.tencent.devops.api.pojo.Response import com.tencent.devops.common.api.exception.TurboException import com.tencent.devops.common.api.exception.code.IS_NOT_ADMIN_MEMBER import com.tencent.devops.common.util.constants.NO_ADMIN_MEMBER_MESSAGE @@ -21,17 +22,17 @@ class UserCustomScheduleTaskController @Autowired constructor( user: String, projectId: String, customScheduleJobModel: CustomScheduleJobModel - ): Result { + ): Response { // if (!turboAuthService.validatePlatformMember(projectId, user)) { // throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) // } - return Result.success(customScheduleJobService.customScheduledJobAdd(customScheduleJobModel)) + return Response.success(customScheduleJobService.customScheduledJobAdd(customScheduleJobModel)) } - override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): Result { + override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): Response { if (!turboAuthService.getAuthResult(projectId, user)) { throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) } - return Result.success(customScheduleJobService.trigger(jobName)!!) + return Response.success(customScheduleJobService.trigger(jobName)) } } From f3b77bb70dbe89c3934cb7e883d3a2548966ba5b Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 11:40:34 +0800 Subject: [PATCH 11/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index 5ad796e12..0f741e93c 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -131,7 +131,7 @@ object TBSSdkApi { val templatePath = properties.urlTemplate!!.replace("{engine}", engineCode).replace("{resource_type}", resourceName) var url = "${properties.rootPath}/$templatePath" - if (customPath.isNullOrBlank()) { + if (!customPath.isNullOrBlank()) { url = url.plus(customPath) } if (pathParam.isNotEmpty()) { From 89c6843e857db1efbfa90850e791e3fb8e9297da Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 12:09:57 +0800 Subject: [PATCH 12/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 5c64801e2..16d762f2b 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -68,7 +68,12 @@ class TBSDaySummaryJob @Autowired constructor( val planIdEntityMap = summaryEntityList.associateBy { it.planId } // 取出项目ID集合用于获取项目组织架构信息 - val projectIdSet = summaryEntityList.map { it.projectId!! }.toSet() + val projectIdSet = try { + summaryEntityList.map { it.projectId!! }.toSet() + } catch (e: Exception) { + logger.warn("summaryEntityList.map { it.projectId!! } error: ${e.message}") + emptySet() + } val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) // 获取项目信息清单 val projectVOList = this.getProjectVOListByProjectIds(notInProjectMapKeySet.toList()) From df1d81929e866f8ef6077e64eb6c6642c46d611b Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 12:32:17 +0800 Subject: [PATCH 13/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 1 + .../src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 16d762f2b..e4d128611 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -50,6 +50,7 @@ class TBSDaySummaryJob @Autowired constructor( val engineConfigEntities = turboEngineConfigRepository.findAll() engineConfigEntities.forEach { engineConfig -> + logger.info("query engineConfig: ${engineConfig.engineCode}") val daySummaryDtoList = TBSSdkApi.queryTbsDaySummary( engineCode = engineConfig.engineCode, queryParam = mapOf( diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index 0f741e93c..b1b466fd3 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -147,6 +147,7 @@ object TBSSdkApi { ) url = url.trimEnd('&') } + logger.info(">>>request url: $url") val request = when (method) { HttpMethod.GET.name -> { buildGet(url, headers) @@ -227,7 +228,7 @@ object TBSSdkApi { ) } else { tbsCommonRequest( - engineCode = "distcc", + engineCode = engineCode, resourceName = "summary", queryParam = queryParam ) From 5b7f22df449eced2e879cc5f361349aa93e39db0 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 14:27:44 +0800 Subject: [PATCH 14/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/IUserCustomScheduleTaskController.kt | 18 ++++++++++++++++++ .../UserCustomScheduleTaskController.kt | 13 ++++++++++--- .../devops/turbo/job/TBSDaySummaryJob.kt | 15 ++++++++++----- .../turbo/service/CustomScheduleJobService.kt | 18 ++++++++++++++++++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt index c7557f8b2..954a92ef2 100644 --- a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt +++ b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt @@ -8,6 +8,7 @@ import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam import org.springframework.http.MediaType +import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody @@ -37,6 +38,23 @@ interface IUserCustomScheduleTaskController { customScheduleJobModel: CustomScheduleJobModel ): Response + @ApiOperation("删除计划任务") + @DeleteMapping( + "/deleteScheduleJob", + produces = [MediaType.APPLICATION_JSON_VALUE] + ) + fun deleteScheduleJob( + @ApiParam(value = "用户信息", required = true) + @RequestHeader(AUTH_HEADER_DEVOPS_USER_ID) + user: String, + @ApiParam(value = "项目id", required = true) + @RequestHeader(AUTH_HEADER_DEVOPS_PROJECT_ID) + projectId: String, + @ApiParam(value = "任务名称", required = true) + @RequestParam(value = "jobName") + jobName: String + ): Response + @ApiOperation("触发定时任务执行") @GetMapping( "/trigger", diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt index e675fb97f..a7d8431b3 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt @@ -23,12 +23,19 @@ class UserCustomScheduleTaskController @Autowired constructor( projectId: String, customScheduleJobModel: CustomScheduleJobModel ): Response { -// if (!turboAuthService.validatePlatformMember(projectId, user)) { -// throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) -// } + if (!turboAuthService.getAuthResult(projectId, user)) { + throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) + } return Response.success(customScheduleJobService.customScheduledJobAdd(customScheduleJobModel)) } + override fun deleteScheduleJob(user: String, projectId: String, jobName: String): Response { + if (!turboAuthService.getAuthResult(projectId, user)) { + throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) + } + return Response.success(customScheduleJobService.customScheduledJobDel(jobName)) + } + override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): Response { if (!turboAuthService.getAuthResult(projectId, user)) { throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index e4d128611..0927d7275 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -51,12 +51,17 @@ class TBSDaySummaryJob @Autowired constructor( val engineConfigEntities = turboEngineConfigRepository.findAll() engineConfigEntities.forEach { engineConfig -> logger.info("query engineConfig: ${engineConfig.engineCode}") - val daySummaryDtoList = TBSSdkApi.queryTbsDaySummary( - engineCode = engineConfig.engineCode, - queryParam = mapOf( - "day" to statisticsDateStr + val daySummaryDtoList = try { + TBSSdkApi.queryTbsDaySummary( + engineCode = engineConfig.engineCode, + queryParam = mapOf( + "day" to statisticsDateStr + ) ) - ) + } catch (e: Exception) { + logger.error("queryTbsDaySummary error: ${e.message}") + return@forEach + } logger.info("daySummaryDtoList size: ${daySummaryDtoList.size}") if (daySummaryDtoList.isEmpty()) { diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt index 3451b3f55..8da6672e4 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/CustomScheduleJobService.kt @@ -78,6 +78,24 @@ class CustomScheduleJobService @Autowired constructor( return true } + /** + * 删除自定义计划任务 + */ + fun customScheduledJobDel(jobName: String): Boolean { + logger.info("delete job $jobName") + val jobKey = JobKey.jobKey(jobName, jobGroup) + try { + scheduler.deleteJob(jobKey) + } catch (e: SchedulerException) { + logger.warn("delete $jobName job fail with scheduler exception!") + return false + } catch (e: Exception) { + logger.error("delete $jobName job fail with exception!") + return false + } + return true + } + /** * 触发job立即执行 */ From f810f9fd1e8abbcdfc1d58833a550451600bc446 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 14:55:50 +0800 Subject: [PATCH 15/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/turbo/dao/repository/TbsDaySummaryRepository.kt | 7 ++++++- .../com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 3 +++ .../main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt index 86aeb6482..c60b7b307 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt @@ -5,4 +5,9 @@ import org.springframework.data.mongodb.repository.MongoRepository import org.springframework.stereotype.Repository @Repository -interface TbsDaySummaryRepository : MongoRepository +interface TbsDaySummaryRepository : MongoRepository { + /** + * 根据日期删除 + */ + fun removeAllByDay(day: String) +} diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 0927d7275..361997040 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -46,6 +46,9 @@ class TBSDaySummaryJob @Autowired constructor( DateTimeUtils.localDate2DateStr(statLocalDate) } + // 清理待统计的数据，防止重复统计 + tbsDaySummaryRepository.removeAllByDay(statisticsDateStr) + val projectVOMap = mutableMapOf() val engineConfigEntities = turboEngineConfigRepository.findAll() diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index b1b466fd3..9903e619c 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -233,6 +233,7 @@ object TBSSdkApi { queryParam = queryParam ) } + logger.info(">>> $engineCode response: $responseStr") val response = JsonUtil.to(responseStr, object : TypeReference>>() {}) if (response.code != 0 || !response.result) { throw TurboException(errorCode = TURBO_THIRDPARTY_SYSTEM_FAIL, errorMessage = "fail to invoke request: " From 26af80279825685487a4a88e60386505b7ab0ea5 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 15:22:39 +0800 Subject: [PATCH 16/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 361997040..2fb8a0b06 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -138,6 +138,7 @@ class TBSDaySummaryJob @Autowired constructor( planId = stringArr[0] engineCode = if (stringArr[1] == "cc") "disttask-cc" else if (stringArr[1] == "ue4") "disttask-ue4" else stringArr[1] + logger.info("planIdAndEngineCode: $planIdAndEngineCode, engineCode: $engineCode") } else { planId = planIdAndEngineCode engineCode = "distcc" From af1cf05a4a74e61f5ece3117795ac7b7454b375f Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 17:07:41 +0800 Subject: [PATCH 17/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/turbo/job/TBSDaySummaryJob.kt | 28 +++++++++---------- .../com/tencent/devops/turbo/sdk/TBSSdkApi.kt | 1 - 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 2fb8a0b06..b87e91e87 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -76,20 +76,6 @@ class TBSDaySummaryJob @Autowired constructor( val summaryEntityList = this.dto2SummaryEntityList(daySummaryList = daySummaryDtoList) val planIdEntityMap = summaryEntityList.associateBy { it.planId } - // 取出项目ID集合用于获取项目组织架构信息 - val projectIdSet = try { - summaryEntityList.map { it.projectId!! }.toSet() - } catch (e: Exception) { - logger.warn("summaryEntityList.map { it.projectId!! } error: ${e.message}") - emptySet() - } - val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) - // 获取项目信息清单 - val projectVOList = this.getProjectVOListByProjectIds(notInProjectMapKeySet.toList()) - if (projectVOList.isNotEmpty()) { - projectVOMap.putAll(projectVOList.associateBy { it.projectId }) - } - val planIdsList = planIdEntityMap.keys.chunked(PAGE_SIZE) for (planIds in planIdsList) { val turboPlanList = turboPlanRepository.findByIdIn(planIds) @@ -104,6 +90,19 @@ class TBSDaySummaryJob @Autowired constructor( summaryEntity?.planCreator = it.createdBy summaryEntity?.planName = it.planName summaryEntity?.projectId = it.projectId + } + + // 取出项目ID集合用于获取项目组织架构信息 + val projectIdSet = turboPlanList.map { it.projectId }.toSet() + val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) + + // 获取项目信息清单 + val projectVOList = this.getProjectVOListByProjectIds(notInProjectMapKeySet.toList()) + if (projectVOList.isNotEmpty()) { + projectVOMap.putAll(projectVOList.associateBy { it.projectId }) + } + turboPlanList.forEach { + val summaryEntity = planIdEntityMap[it.id] summaryEntity?.projectName = projectVOMap[it.projectId]?.projectName summaryEntity?.bgName = projectVOMap[it.projectId]?.bgName summaryEntity?.bgId = projectVOMap[it.projectId]?.bgId?.toInt() @@ -138,7 +137,6 @@ class TBSDaySummaryJob @Autowired constructor( planId = stringArr[0] engineCode = if (stringArr[1] == "cc") "disttask-cc" else if (stringArr[1] == "ue4") "disttask-ue4" else stringArr[1] - logger.info("planIdAndEngineCode: $planIdAndEngineCode, engineCode: $engineCode") } else { planId = planIdAndEngineCode engineCode = "distcc" diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index 9903e619c..b1b466fd3 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -233,7 +233,6 @@ object TBSSdkApi { queryParam = queryParam ) } - logger.info(">>> $engineCode response: $responseStr") val response = JsonUtil.to(responseStr, object : TypeReference>>() {}) if (response.code != 0 || !response.result) { throw TurboException(errorCode = TURBO_THIRDPARTY_SYSTEM_FAIL, errorMessage = "fail to invoke request: " From d147324d2909df1d5474e858d3f778d9d0a09098 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 17:37:08 +0800 Subject: [PATCH 18/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/turbo/biz-turbo/build.gradle.kts | 4 ++-- src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/turbo/biz-turbo/build.gradle.kts b/src/backend/turbo/biz-turbo/build.gradle.kts index 8be240620..03a30a8f7 100644 --- a/src/backend/turbo/biz-turbo/build.gradle.kts +++ b/src/backend/turbo/biz-turbo/build.gradle.kts @@ -16,10 +16,10 @@ dependencies { api("com.tencent.bk.devops.ci.common:common-api:${Versions.ciVersion}"){ isTransitive = false } - api("com.tencent.bk.devops.ci.auth:api-auth:${Versions.ciAuthVersion}"){ + api("com.tencent.bk.devops.ci.auth:api-auth:${Versions.ciVersion}"){ isTransitive = false } - api("com.tencent.bk.devops.ci.common:common-auth-api:${Versions.ciAuthVersion}"){ + api("com.tencent.bk.devops.ci.common:common-auth-api:${Versions.ciVersion}"){ isTransitive = false } } diff --git a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt index 81565d646..f94480e9f 100644 --- a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt @@ -7,7 +7,6 @@ object Versions { const val ciVersion = "1.14.0-SNAPSHOT" const val commonIo = "2.7" const val jjwtVersion = "0.9.0" - const val ciAuthVersion = "1.8.0-RC.3-RELEASE" const val micrometerVersion = "1.6.6" const val okHttpVersion = "4.9.0" const val Kotlin = "1.6.0" From 691240ee6c89389187fd9e561d904df31ac1b4b8 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 18:41:02 +0800 Subject: [PATCH 19/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/turbo/job/TBSDaySummaryJob.kt | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index b87e91e87..54c053fa7 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -74,22 +74,20 @@ class TBSDaySummaryJob @Autowired constructor( // 把TBS的接口数据整理成entity val summaryEntityList = this.dto2SummaryEntityList(daySummaryList = daySummaryDtoList) - val planIdEntityMap = summaryEntityList.associateBy { it.planId } - - val planIdsList = planIdEntityMap.keys.chunked(PAGE_SIZE) - for (planIds in planIdsList) { - val turboPlanList = turboPlanRepository.findByIdIn(planIds) - logger.info("turboPlanRepository.findByIdIn result size: ${turboPlanList.size}") - if (turboPlanList.isEmpty()) { - logger.warn("turboPlanList is empty! continue") - continue - } - - turboPlanList.forEach { - val summaryEntity = planIdEntityMap[it.id] - summaryEntity?.planCreator = it.createdBy - summaryEntity?.planName = it.planName - summaryEntity?.projectId = it.projectId + val summaryListList = summaryEntityList.chunked(PAGE_SIZE) + for (summaryList in summaryListList) { + + // 根据planId批量获取方案信息 + val planIds = summaryList.map { it.planId }.toSet() + val turboPlanList = turboPlanRepository.findByIdIn(planIds.toList()) + val planEntityMap = turboPlanList.associateBy { it.id } + + // 赋值plan信息和项目id + for (summaryEntity in summaryList) { + val planEntity = planEntityMap[summaryEntity.planId] + summaryEntity.planCreator = planEntity?.createdBy + summaryEntity.planName = planEntity?.planName + summaryEntity.projectId = planEntity?.projectId } // 取出项目ID集合用于获取项目组织架构信息 @@ -101,14 +99,14 @@ class TBSDaySummaryJob @Autowired constructor( if (projectVOList.isNotEmpty()) { projectVOMap.putAll(projectVOList.associateBy { it.projectId }) } - turboPlanList.forEach { - val summaryEntity = planIdEntityMap[it.id] - summaryEntity?.projectName = projectVOMap[it.projectId]?.projectName - summaryEntity?.bgName = projectVOMap[it.projectId]?.bgName - summaryEntity?.bgId = projectVOMap[it.projectId]?.bgId?.toInt() - summaryEntity?.deptName = projectVOMap[it.projectId]?.deptName - summaryEntity?.deptId = projectVOMap[it.projectId]?.deptId?.toInt() - summaryEntity?.productId = projectVOMap[it.projectId]?.productId + + for (it in summaryList) { + it.projectName = projectVOMap[it.projectId]?.projectName + it.bgName = projectVOMap[it.projectId]?.bgName + it.bgId = projectVOMap[it.projectId]?.bgId?.toInt() + it.deptName = projectVOMap[it.projectId]?.deptName + it.deptId = projectVOMap[it.projectId]?.deptId?.toInt() + it.productId = projectVOMap[it.projectId]?.productId } } From 3e677a4de8b8d0c16034db1136ebfb25cabaae35 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 19:52:14 +0800 Subject: [PATCH 20/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 54c053fa7..1bca919ac 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -80,6 +80,7 @@ class TBSDaySummaryJob @Autowired constructor( // 根据planId批量获取方案信息 val planIds = summaryList.map { it.planId }.toSet() val turboPlanList = turboPlanRepository.findByIdIn(planIds.toList()) + logger.info("turboPlanRepository.findByIdIn result size: ${turboPlanList.size}") val planEntityMap = turboPlanList.associateBy { it.id } // 赋值plan信息和项目id @@ -167,6 +168,7 @@ class TBSDaySummaryJob @Autowired constructor( return list } list = result.data!! + logger.info(">>> ProjectVOs ${JsonUtil.toJson(list)}") } return list } From 3f4c1a5f4d4a699a86c7ca9855df3588bb5a424d Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 20:00:23 +0800 Subject: [PATCH 21/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 1bca919ac..4a154c8f2 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -98,7 +98,7 @@ class TBSDaySummaryJob @Autowired constructor( // 获取项目信息清单 val projectVOList = this.getProjectVOListByProjectIds(notInProjectMapKeySet.toList()) if (projectVOList.isNotEmpty()) { - projectVOMap.putAll(projectVOList.associateBy { it.projectId }) + projectVOMap.putAll(projectVOList.associateBy { it.englishName }) } for (it in summaryList) { From d0592c15444cceb3956c87f7d3811faf0e17cd68 Mon Sep 17 00:00:00 2001 From: RJ Date: Mon, 25 Dec 2023 20:12:50 +0800 Subject: [PATCH 22/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index 4a154c8f2..f32328188 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -168,7 +168,6 @@ class TBSDaySummaryJob @Autowired constructor( return list } list = result.data!! - logger.info(">>> ProjectVOs ${JsonUtil.toJson(list)}") } return list } From 6ecbaee20800dae0790002bd911089762d9719c5 Mon Sep 17 00:00:00 2001 From: RJ Date: Tue, 26 Dec 2023 09:49:48 +0800 Subject: [PATCH 23/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/turbo/job/TBSDaySummaryJob.kt | 4 +++- .../kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt | 1 - .../tencent/devops/turbo/model/TTbsDaySummaryEntity.kt | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt index f32328188..98f65bb2f 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt @@ -96,7 +96,7 @@ class TBSDaySummaryJob @Autowired constructor( val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys) // 获取项目信息清单 - val projectVOList = this.getProjectVOListByProjectIds(notInProjectMapKeySet.toList()) + val projectVOList = this.getProjectVOListByProjectIds(projectIds = notInProjectMapKeySet.toList()) if (projectVOList.isNotEmpty()) { projectVOMap.putAll(projectVOList.associateBy { it.englishName }) } @@ -107,6 +107,8 @@ class TBSDaySummaryJob @Autowired constructor( it.bgId = projectVOMap[it.projectId]?.bgId?.toInt() it.deptName = projectVOMap[it.projectId]?.deptName it.deptId = projectVOMap[it.projectId]?.deptId?.toInt() + it.centerName = projectVOMap[it.projectId]?.centerName + it.centerId = projectVOMap[it.projectId]?.centerId?.toInt() it.productId = projectVOMap[it.projectId]?.productId } } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index b1b466fd3..658fa08e6 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -147,7 +147,6 @@ object TBSSdkApi { ) url = url.trimEnd('&') } - logger.info(">>>request url: $url") val request = when (method) { HttpMethod.GET.name -> { buildGet(url, headers) diff --git a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt index 2d5371490..55f263739 100644 --- a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt +++ b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/TTbsDaySummaryEntity.kt @@ -1,6 +1,7 @@ package com.tencent.devops.turbo.model import org.springframework.data.annotation.Id +import org.springframework.data.mongodb.core.index.Indexed import org.springframework.data.mongodb.core.mapping.Document import org.springframework.data.mongodb.core.mapping.Field import java.time.LocalDateTime @@ -16,6 +17,7 @@ data class TTbsDaySummaryEntity( * 统计数据归属日期 */ @Field + @Indexed(background = true) var day: String?, /** @@ -101,6 +103,14 @@ data class TTbsDaySummaryEntity( @Field("dept_id") var deptId: Int? = null, + /** + * 中心信息 + */ + @Field("center_name") + var centerName: String? = null, + @Field("center_id") + var centerId: Int? = null, + /** * 项目所属组织架构 */ From 0a9f387bb7b52496db32ebba039b8c7e5278f55f Mon Sep 17 00:00:00 2001 From: RJ Date: Tue, 26 Dec 2023 14:47:40 +0800 Subject: [PATCH 24/24] =?UTF-8?q?feat=EF=BC=9A=E7=BC=96=E8=AF=91=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E8=BF=90=E8=90=A5=E6=95=B0=E6=8D=AE=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=20#156?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt index f94480e9f..0917e535a 100644 --- a/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/turbo/buildSrc/src/main/kotlin/Versions.kt @@ -4,7 +4,7 @@ object Versions { const val bouncyCastleVersion = "1.70" const val swaggerVersion = "1.6.2" const val guavaVersion = "29.0-jre" - const val ciVersion = "1.14.0-SNAPSHOT" + const val ciVersion = "1.14.0-rc.1" const val commonIo = "2.7" const val jjwtVersion = "0.9.0" const val micrometerVersion = "1.6.6"