From 44da2e2a0221c643da2d833fcf0f895306bf4457 Mon Sep 17 00:00:00 2001 From: emptyOVO Date: Wed, 13 Nov 2024 16:54:02 +0800 Subject: [PATCH] fix: exception handle, default configure value --- .../DolphinScheduleEngine.java | 36 +++++---- .../DolphinScheduleUtils.java | 81 ++++++++++++------- .../DolphinScheduleContainerTestEnv.java | 66 ++++++++------- .../DolphinScheduleEngineTest.java | 1 + .../main/resources/application-dev.properties | 4 +- .../resources/application-prod.properties | 4 +- .../resources/application-test.properties | 4 +- 7 files changed, 118 insertions(+), 78 deletions(-) diff --git a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngine.java b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngine.java index 5506c12630..d6a27cf29d 100644 --- a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngine.java +++ b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngine.java @@ -65,10 +65,10 @@ public class DolphinScheduleEngine implements ScheduleEngine { @Value("${default.admin.password:inlong}") private String password; - @Value("${inlong.schedule.dolphinscheduler.url:}") + @Value("${inlong.schedule.dolphinscheduler.url:http://127.0.0.1:12345/dolphinscheduler}") private String dolphinUrl; - @Value("${inlong.schedule.dolphinscheduler.token:}") + @Value("${inlong.schedule.dolphinscheduler.token:default_token_value}") private String token; private long projectCode; @@ -90,7 +90,8 @@ public DolphinScheduleEngine(String host, int port, String username, String pass this.dsUtils = new DolphinScheduleUtils(); this.scheduledProcessMap = new ConcurrentHashMap<>(); } catch (Exception e) { - throw new DolphinScheduleException("Failed to init dolphin scheduler ", e); + LOGGER.error("Failed to init dolphin scheduler: ", e); + throw new DolphinScheduleException(String.format("Failed to init dolphin scheduler: %s", e.getMessage())); } } @@ -101,7 +102,8 @@ public DolphinScheduleEngine() { this.dsUtils = new DolphinScheduleUtils(); this.scheduledProcessMap = new ConcurrentHashMap<>(); } catch (Exception e) { - throw new DolphinScheduleException("Failed to init dolphin scheduler ", e); + LOGGER.error("Failed to init dolphin scheduler: ", e); + throw new DolphinScheduleException(String.format("Failed to init dolphin scheduler: %s", e.getMessage())); } } @@ -111,8 +113,7 @@ public DolphinScheduleEngine() { */ @Override public void start() { - LOGGER.info("Starting dolphin scheduler engine"); - LOGGER.info("Checking project exists..."); + LOGGER.info("Starting dolphin scheduler engine, Checking project exists..."); long code = dsUtils.checkAndGetUniqueId(projectBaseUrl, token, DS_DEFAULT_PROJECT_NAME); if (code != 0) { LOGGER.info("Project exists, project code: {}", code); @@ -141,8 +142,8 @@ public boolean handleRegister(ScheduleInfo scheduleInfo) { String processName = scheduleInfo.getInlongGroupId() + DS_DEFAULT_PROCESS_NAME; String processDesc = DS_DEFAULT_PROCESS_DESC + scheduleInfo.getInlongGroupId(); - LOGGER.info("Dolphin Scheduler handle register begin for {}", scheduleInfo.getInlongGroupId()); - LOGGER.info("Checking process definition id uniqueness..."); + LOGGER.info("Dolphin Scheduler handle register begin for {}, Checking process definition id uniqueness...", + scheduleInfo.getInlongGroupId()); try { long processDefCode = dsUtils.checkAndGetUniqueId(processDefUrl, token, processName); @@ -180,7 +181,9 @@ public boolean handleRegister(ScheduleInfo scheduleInfo) { scheduledProcessMap.putIfAbsent(processDefCode, processName); return online; } catch (Exception e) { - throw new DolphinScheduleException("Failed to handle unregister dolphin scheduler", e); + LOGGER.error("Failed to handle unregister dolphin scheduler: ", e); + throw new DolphinScheduleException( + String.format("Failed to handle unregister dolphin scheduler: %s", e.getMessage())); } } @@ -194,8 +197,8 @@ public boolean handleUnregister(String groupId) { String processName = groupId + DS_DEFAULT_PROCESS_NAME; String processDefUrl = projectBaseUrl + "/" + projectCode + DS_PROCESS_URL; - LOGGER.info("Dolphin Scheduler handle Unregister begin for {}", groupId); - LOGGER.info("Checking process definition id uniqueness..."); + LOGGER.info("Dolphin Scheduler handle Unregister begin for {}, Checking process definition id uniqueness...", + groupId); try { long processDefCode = dsUtils.checkAndGetUniqueId(processDefUrl, token, processName); if (processDefCode != 0 || scheduledProcessMap.containsKey(processDefCode)) { @@ -211,7 +214,9 @@ public boolean handleUnregister(String groupId) { LOGGER.info("Un-registered dolphin schedule info for {}", groupId); return !scheduledProcessMap.containsKey(processDefCode); } catch (Exception e) { - throw new DolphinScheduleException("Failed to handle unregister dolphin scheduler", e); + LOGGER.error("Failed to handle unregister dolphin scheduler: ", e); + throw new DolphinScheduleException( + String.format("Failed to handle unregister dolphin scheduler: %s", e.getMessage())); } } @@ -226,7 +231,9 @@ public boolean handleUpdate(ScheduleInfo scheduleInfo) { try { return handleUnregister(scheduleInfo.getInlongGroupId()) && handleRegister(scheduleInfo); } catch (Exception e) { - throw new DolphinScheduleException("Failed to handle update dolphin scheduler", e); + LOGGER.error("Failed to handle update dolphin scheduler: ", e); + throw new DolphinScheduleException( + String.format("Failed to handle update dolphin scheduler: %s", e.getMessage())); } } @@ -256,7 +263,8 @@ public void stop() { LOGGER.info("Dolphin scheduler engine stopped"); } catch (Exception e) { - throw new DolphinScheduleException("Failed to stop dolphin scheduler", e); + LOGGER.error("Failed to stop dolphin scheduler: ", e); + throw new DolphinScheduleException(String.format("Failed to stop dolphin scheduler: %s", e.getMessage())); } } diff --git a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleUtils.java b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleUtils.java index 7e263bac9a..5397799c11 100644 --- a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleUtils.java +++ b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleUtils.java @@ -152,8 +152,9 @@ public long checkAndGetUniqueId(String url, String token, String searchVal) { return 0; } catch (IOException e) { - LOGGER.error("Unexpected wrong in check id uniqueness", e); - return 0; + LOGGER.error("Unexpected wrong in check id uniqueness: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in check id uniqueness: %s", e.getMessage())); } } @@ -184,8 +185,9 @@ public long creatNewProject(String url, String token, String projectName, String } return 0; } catch (IOException e) { - LOGGER.error("Unexpected wrong in creating new project", e); - return 0; + LOGGER.error("Unexpected wrong in creating new project: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in creating new project: %s", e.getMessage())); } } @@ -207,8 +209,9 @@ public Map queryAllProcessDef(String url, String token) { LOGGER.info("Query all process definition success, processes info: {}", processDef); return processDef; } catch (IOException e) { - LOGGER.error("Unexpected wrong in generating task code", e); - return null; + LOGGER.error("Unexpected wrong in query process definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in query process definition: %s", e.getMessage())); } } @@ -235,8 +238,9 @@ public long genTaskCode(String url, String token) { } return 0; } catch (IOException e) { - LOGGER.error("Unexpected wrong in generating task code", e); - return 0; + LOGGER.error("Unexpected wrong in generating task code: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in generating task code: %s", e.getMessage())); } } @@ -291,8 +295,9 @@ public long createProcessDef(String url, String token, String name, String desc, } return 0; } catch (IOException e) { - LOGGER.error("Unexpected wrong in creating process definition", e); - return 0; + LOGGER.error("Unexpected wrong in creating process definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in creating process definition: %s", e.getMessage())); } } @@ -318,8 +323,9 @@ public boolean releaseProcessDef(String processDefUrl, long processDefCode, Stri } return false; } catch (IOException e) { - LOGGER.error("Unexpected wrong in release process definition", e); - return false; + LOGGER.error("Unexpected wrong in release process definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in release process definition: %s", e.getMessage())); } } @@ -347,7 +353,8 @@ public int createScheduleForProcessDef(String url, long processDefCode, String t } else if (scheduleInfo.getScheduleType() == 1) { crontab = scheduleInfo.getCrontabExpression(); } else { - throw new IllegalArgumentException("Unsupported schedule type: " + scheduleInfo.getScheduleType()); + LOGGER.error("Unsupported schedule type: {}", scheduleInfo.getScheduleType()); + throw new DolphinScheduleException("Unsupported schedule type: " + scheduleInfo.getScheduleType()); } DScheduleInfo dScheduleInfo = new DScheduleInfo(); @@ -371,8 +378,9 @@ public int createScheduleForProcessDef(String url, long processDefCode, String t return data.get(DS_ID).getAsInt(); } } catch (IOException e) { - LOGGER.error("Unexpected wrong in creating schedule for process definition", e); - return 0; + LOGGER.error("Unexpected wrong in creating schedule for process definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in creating schedule for process definition: %s", e.getMessage())); } return 0; } @@ -394,8 +402,9 @@ public boolean onlineScheduleForProcessDef(String scheduleUrl, int scheduleId, S return response.get(DS_RESPONSE_DATA).getAsBoolean(); } } catch (IOException e) { - LOGGER.error("Unexpected wrong in online process definition", e); - return false; + LOGGER.error("Unexpected wrong in online process definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in online process definition: %s", e.getMessage())); } return false; } @@ -413,7 +422,9 @@ public void deleteProcessDef(String processDefUrl, String token, long processDef try { executeHttpRequest(url, HTTP_DELETE, new HashMap<>(), header); } catch (IOException e) { - LOGGER.error("Unexpected wrong in deleting process definition", e); + LOGGER.error("Unexpected wrong in deleting process definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in deleting process definition: %s", e.getMessage())); } } @@ -430,7 +441,9 @@ public void deleteProject(String projectBaseUrl, String token, long projectCode) try { executeHttpRequest(url, HTTP_DELETE, new HashMap<>(), header); } catch (IOException e) { - LOGGER.error("Unexpected wrong in deleting project definition", e); + LOGGER.error("Unexpected wrong in deleting project definition: ", e); + throw new DolphinScheduleException( + String.format("Unexpected wrong in deleting project definition: %s", e.getMessage())); } } @@ -517,7 +530,8 @@ private JsonObject executeHttpRequest(String url, String method, Map loginParams = new HashMap<>(); loginParams.put(DS_USERNAME, DS_DEFAULT_USERNAME); loginParams.put(DS_PASSWORD, DS_DEFAULT_PASSWORD); - JsonObject loginResponse = executeHttpRequest(DS_URL + DS_LOGIN_URL, loginParams, new HashMap<>()); - if (loginResponse.get("success").getAsBoolean()) { - String tokenGenUrl = DS_URL + DS_TOKEN_URL + DS_TOKEN_GEN_URL; - Map tokenParams = new HashMap<>(); - tokenParams.put(DS_USERID, String.valueOf(DS_DEFAULT_USERID)); - - LocalDateTime now = LocalDateTime.now(); - LocalDateTime tomorrow = now.plusDays(1); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DS_EXPIRE_TIME_FORMAT); - String expireTime = tomorrow.format(formatter); - tokenParams.put(DS_EXPIRE_TIME, expireTime); - - Map cookies = new HashMap<>(); - cookies.put(DS_COOKIE_SC_TYPE, loginResponse.get(DS_RESPONSE_DATA) - .getAsJsonObject().get(DS_COOKIE_SC_TYPE).getAsString()); - cookies.put(DS_COOKIE_SESSION_ID, loginResponse.get(DS_RESPONSE_DATA) - .getAsJsonObject().get(DS_COOKIE_SESSION_ID).getAsString()); - - JsonObject tokenGenResponse = executeHttpRequest(tokenGenUrl, tokenParams, cookies); - - String accessTokenUrl = DS_URL + DS_TOKEN_URL; - tokenParams.put(DS_RESPONSE_TOKEN, tokenGenResponse.get(DS_RESPONSE_DATA).getAsString()); - JsonObject result = executeHttpRequest(accessTokenUrl, tokenParams, cookies); - String token = result.get(DS_RESPONSE_DATA).getAsJsonObject().get(DS_RESPONSE_TOKEN).getAsString(); - DS_LOG.info("login and generate token success, token: {}", token); - return token; + try { + JsonObject loginResponse = executeHttpRequest(DS_URL + DS_LOGIN_URL, loginParams, new HashMap<>()); + if (loginResponse.get("success").getAsBoolean()) { + String tokenGenUrl = DS_URL + DS_TOKEN_URL + DS_TOKEN_GEN_URL; + Map tokenParams = new HashMap<>(); + tokenParams.put(DS_USERID, String.valueOf(DS_DEFAULT_USERID)); + + LocalDateTime now = LocalDateTime.now(); + LocalDateTime tomorrow = now.plusDays(1); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DS_EXPIRE_TIME_FORMAT); + String expireTime = tomorrow.format(formatter); + tokenParams.put(DS_EXPIRE_TIME, expireTime); + + Map cookies = new HashMap<>(); + cookies.put(DS_COOKIE_SC_TYPE, loginResponse.get(DS_RESPONSE_DATA) + .getAsJsonObject().get(DS_COOKIE_SC_TYPE).getAsString()); + cookies.put(DS_COOKIE_SESSION_ID, loginResponse.get(DS_RESPONSE_DATA) + .getAsJsonObject().get(DS_COOKIE_SESSION_ID).getAsString()); + + JsonObject tokenGenResponse = executeHttpRequest(tokenGenUrl, tokenParams, cookies); + + String accessTokenUrl = DS_URL + DS_TOKEN_URL; + tokenParams.put(DS_RESPONSE_TOKEN, tokenGenResponse.get(DS_RESPONSE_DATA).getAsString()); + JsonObject result = executeHttpRequest(accessTokenUrl, tokenParams, cookies); + String token = result.get(DS_RESPONSE_DATA).getAsJsonObject().get(DS_RESPONSE_TOKEN).getAsString(); + DS_LOG.info("login and generate token success, token: {}", token); + return token; + } + return null; + } catch (Exception e) { + DS_LOG.error("login and generate token fail: ", e); + throw new DolphinScheduleException(String.format("login and generate token fail: %s", e.getMessage())); } - DS_LOG.error("login and generate token fail"); - return null; } public static void envShutDown() { @@ -167,7 +172,8 @@ private static JsonObject executeHttpRequest(String url, Map que String responseBody = response.body().string(); return JsonParser.parseString(responseBody).getAsJsonObject(); } else { - throw new IOException("Unexpected http response error " + response); + DS_LOG.error("Unexpected http response error: {}", response); + throw new DolphinScheduleException("Unexpected http response error " + response); } } } diff --git a/inlong-manager/manager-schedule/src/test/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngineTest.java b/inlong-manager/manager-schedule/src/test/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngineTest.java index 0364e1ccc7..f27155c4bf 100644 --- a/inlong-manager/manager-schedule/src/test/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngineTest.java +++ b/inlong-manager/manager-schedule/src/test/java/org/apache/inlong/manager/schedule/dolphinscheduler/DolphinScheduleEngineTest.java @@ -33,6 +33,7 @@ import static org.apache.inlong.manager.schedule.dolphinscheduler.DolphinSchedulerContainerEnvConstants.INLONG_DS_TEST_USERNAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; + @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class DolphinScheduleEngineTest extends DolphinScheduleContainerTestEnv { diff --git a/inlong-manager/manager-web/src/main/resources/application-dev.properties b/inlong-manager/manager-web/src/main/resources/application-dev.properties index 06baf34649..f0a9f767e0 100644 --- a/inlong-manager/manager-web/src/main/resources/application-dev.properties +++ b/inlong-manager/manager-web/src/main/resources/application-dev.properties @@ -108,5 +108,5 @@ default.module.id=1 inlong.schedule.engine=none # DolphinScheduler related config -# inlong.schedule.dolphinscheduler.url= -# inlong.schedule.dolphinscheduler.token= \ No newline at end of file +inlong.schedule.dolphinscheduler.url=http://127.0.0.1:12345/dolphinscheduler +inlong.schedule.dolphinscheduler.token=default_token_value \ No newline at end of file diff --git a/inlong-manager/manager-web/src/main/resources/application-prod.properties b/inlong-manager/manager-web/src/main/resources/application-prod.properties index 32bdef5259..af0ffb5353 100644 --- a/inlong-manager/manager-web/src/main/resources/application-prod.properties +++ b/inlong-manager/manager-web/src/main/resources/application-prod.properties @@ -100,5 +100,5 @@ cls.manager.endpoint=127.0.0.1 inlong.schedule.engine=none # DolphinScheduler related config -# inlong.schedule.dolphinscheduler.url= -# inlong.schedule.dolphinscheduler.token= \ No newline at end of file +inlong.schedule.dolphinscheduler.url=http://127.0.0.1:12345/dolphinscheduler +inlong.schedule.dolphinscheduler.token=default_token_value \ No newline at end of file diff --git a/inlong-manager/manager-web/src/main/resources/application-test.properties b/inlong-manager/manager-web/src/main/resources/application-test.properties index 7a2b92442c..a8a6b638f9 100644 --- a/inlong-manager/manager-web/src/main/resources/application-test.properties +++ b/inlong-manager/manager-web/src/main/resources/application-test.properties @@ -101,5 +101,5 @@ cls.manager.endpoint=127.0.0.1 inlong.schedule.engine=none # DolphinScheduler related config -# inlong.schedule.dolphinscheduler.url= -# inlong.schedule.dolphinscheduler.token= \ No newline at end of file +inlong.schedule.dolphinscheduler.url=http://127.0.0.1:12345/dolphinscheduler +inlong.schedule.dolphinscheduler.token=default_token_value \ No newline at end of file