From 34379970ab600499902ca707526c186db2c6cff0 Mon Sep 17 00:00:00 2001 From: dmitrys Date: Sat, 19 Oct 2024 21:21:44 +0200 Subject: [PATCH] #73 Add support for additional ENV parameters to be stored with analysis and sent to execution engine --- datanode/pom.xml | 2 +- .../analysis/AnalysisController.java | 2 +- .../datanode/converter/GsonConverter.java | 43 +++++++++++++++++++ .../converter/StringMapConverter.java | 25 +++++++++++ .../dto/analysis/AnalysisRequestDTO.java | 3 ++ .../datanode/model/analysis/Analysis.java | 8 ++++ .../datanode/service/AnalysisService.java | 3 ++ .../V20241019000000__analysis-env.sql | 1 + 8 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/GsonConverter.java create mode 100644 datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/StringMapConverter.java create mode 100644 datanode/src/main/resources/db/migration/V20241019000000__analysis-env.sql diff --git a/datanode/pom.xml b/datanode/pom.xml index 438f8fd4..52d55127 100644 --- a/datanode/pom.xml +++ b/datanode/pom.xml @@ -80,7 +80,7 @@ com.odysseusinc.arachne execution-engine-commons - 2.3 + 2.6.0 arachne-sys-settings diff --git a/datanode/src/main/java/com/odysseusinc/arachne/datanode/controller/analysis/AnalysisController.java b/datanode/src/main/java/com/odysseusinc/arachne/datanode/controller/analysis/AnalysisController.java index 96b3e281..ca6c0940 100644 --- a/datanode/src/main/java/com/odysseusinc/arachne/datanode/controller/analysis/AnalysisController.java +++ b/datanode/src/main/java/com/odysseusinc/arachne/datanode/controller/analysis/AnalysisController.java @@ -92,7 +92,7 @@ public UploadDTO uploadFiles(Principal principal, @RequestPart("file") List execute(Principal principal, String id, @Valid @RequestBody AnalysisRequestDTO request) { User user = userService.getUser(principal); return analysisService.run(user, request, id); diff --git a/datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/GsonConverter.java b/datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/GsonConverter.java new file mode 100644 index 00000000..d75fbf99 --- /dev/null +++ b/datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/GsonConverter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Odysseus Data Services, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.odysseusinc.arachne.datanode.converter; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import lombok.extern.slf4j.Slf4j; + +import javax.persistence.AttributeConverter; +import java.lang.reflect.Type; + +@Slf4j +public abstract class GsonConverter implements AttributeConverter { + private static final Gson GSON = new Gson(); + + private final Type type; + + public GsonConverter(TypeToken type) { + this.type = type.getType(); + } + + @Override + public String convertToDatabaseColumn(T value) { + return GSON.toJson(value, type); + } + + @Override + public T convertToEntityAttribute(String field) { + return GSON.fromJson(field, type); + } +} diff --git a/datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/StringMapConverter.java b/datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/StringMapConverter.java new file mode 100644 index 00000000..dcc2a4b1 --- /dev/null +++ b/datanode/src/main/java/com/odysseusinc/arachne/datanode/converter/StringMapConverter.java @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Odysseus Data Services, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.odysseusinc.arachne.datanode.converter; + +import com.google.gson.reflect.TypeToken; + +import java.util.Map; + +public class StringMapConverter extends GsonConverter> { + public StringMapConverter() { + super(new TypeToken>(){}); + } +} diff --git a/datanode/src/main/java/com/odysseusinc/arachne/datanode/dto/analysis/AnalysisRequestDTO.java b/datanode/src/main/java/com/odysseusinc/arachne/datanode/dto/analysis/AnalysisRequestDTO.java index 00a49e7b..4f8180f2 100644 --- a/datanode/src/main/java/com/odysseusinc/arachne/datanode/dto/analysis/AnalysisRequestDTO.java +++ b/datanode/src/main/java/com/odysseusinc/arachne/datanode/dto/analysis/AnalysisRequestDTO.java @@ -20,6 +20,7 @@ import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; +import java.util.Map; @Getter @Setter @@ -42,4 +43,6 @@ public class AnalysisRequestDTO { private String environmentId; private String dockerImage; + + private Map parameters; } diff --git a/datanode/src/main/java/com/odysseusinc/arachne/datanode/model/analysis/Analysis.java b/datanode/src/main/java/com/odysseusinc/arachne/datanode/model/analysis/Analysis.java index c51b41cd..7ad3e2f3 100644 --- a/datanode/src/main/java/com/odysseusinc/arachne/datanode/model/analysis/Analysis.java +++ b/datanode/src/main/java/com/odysseusinc/arachne/datanode/model/analysis/Analysis.java @@ -14,6 +14,8 @@ */ package com.odysseusinc.arachne.datanode.model.analysis; +import com.odysseusinc.arachne.datanode.converter.GsonConverter; +import com.odysseusinc.arachne.datanode.converter.StringMapConverter; import com.odysseusinc.arachne.datanode.environment.EnvironmentDescriptor; import com.odysseusinc.arachne.datanode.model.datasource.DataSource; import com.odysseusinc.arachne.execution_engine_common.api.v1.dto.AnalysisResultStatusDTO; @@ -24,6 +26,7 @@ import javax.persistence.AttributeOverrides; import javax.persistence.CascadeType; import javax.persistence.Column; +import javax.persistence.Convert; import javax.persistence.Embedded; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -39,6 +42,7 @@ import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.List; +import java.util.Map; @Getter @Setter @@ -117,5 +121,9 @@ public class Analysis { @JoinColumn(name = "actual_environment_id") private EnvironmentDescriptor actualEnvironment; + @Column(name = "parameters") + @Convert(converter = StringMapConverter.class) + private Map parameters; + } diff --git a/datanode/src/main/java/com/odysseusinc/arachne/datanode/service/AnalysisService.java b/datanode/src/main/java/com/odysseusinc/arachne/datanode/service/AnalysisService.java index fc616947..6661c1a3 100644 --- a/datanode/src/main/java/com/odysseusinc/arachne/datanode/service/AnalysisService.java +++ b/datanode/src/main/java/com/odysseusinc/arachne/datanode/service/AnalysisService.java @@ -287,6 +287,7 @@ public AnalysisDTO get(Long id) { dto.setTitle(analysis.getTitle()); dto.setStudy(analysis.getStudyTitle()); dto.setExecutableFileName(analysis.getExecutableFileName()); + dto.setParameters(analysis.getParameters()); dto.setFiles(UploadService.scan(sourcedir, UploadService.toRelativePath(sourcedir))); String environmentId = Optional.ofNullable(analysis.getActualEnvironment()).map(EnvironmentDescriptor::getDescriptorId).orElseGet(() -> Optional.ofNullable(analysis.getEnvironment()).map(EnvironmentDescriptor::getDescriptorId).orElse(null) @@ -337,6 +338,7 @@ private Analysis save(com.odysseusinc.arachne.datanode.dto.analysis.AnalysisRequ Analysis analysis = new Analysis(); analysis.setExecutableFileName(dto.getExecutableFileName()); + analysis.setParameters(dto.getParameters()); analysis.setSourceFolder(sourceFolder); // This is not used but we need to have something because not null in DB analysis.setAnalysisFolder(AnalysisUtils.createUniqueDir(filesStorePath).getAbsolutePath()); @@ -391,6 +393,7 @@ private AnalysisRequestDTO toDto(Analysis analysis) { dto.setDataSource(dataSourceService.toUnsecuredDto(analysis.getDataSource())); dto.setId(analysis.getId()); dto.setExecutableFileName(analysis.getExecutableFileName()); + dto.setParameters(analysis.getParameters());; dto.setRequestedDescriptorId(Optional.ofNullable(analysis.getEnvironment()).map(EnvironmentDescriptor::getDescriptorId).orElse(null)); dto.setDockerImage(analysis.getDockerImage()); dto.setUpdateStatusCallback(analysis.getUpdateStatusCallback()); diff --git a/datanode/src/main/resources/db/migration/V20241019000000__analysis-env.sql b/datanode/src/main/resources/db/migration/V20241019000000__analysis-env.sql new file mode 100644 index 00000000..c6744811 --- /dev/null +++ b/datanode/src/main/resources/db/migration/V20241019000000__analysis-env.sql @@ -0,0 +1 @@ +ALTER TABLE analyses ADD COLUMN parameters VARCHAR;