diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f577c30..d2d7497f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [4.7.0] - 2024-10-29 + +### Added +- User information to environment of process running the tool [#28](https://github.com/ncsa/datawolf/issues/28) +- Ability for extra environment variables (used by IN-CORE) [#35](https://github.com/ncsa/datawolf/issues/35) + +### Changed +- IN-CORE Dataset DAO and FileStorage implementation to use latest API [#29](https://github.com/ncsa/datawolf/issues/29) +- Kubernetes executor prints exception [#23](https://github.com/ncsa/datawolf/issues/23) +- Upgrade hsqldb to 2.7.3 [#27](https://github.com/ncsa/datawolf/issues/27) +- Custom properties to include more configuration variables [#33](https://github.com/ncsa/datawolf/issues/33) + ## [4.6.0] - 2023-02-15 ### Added diff --git a/charts/datawolf/Chart.yaml b/charts/datawolf/Chart.yaml index 66b5f1c2..228443fd 100644 --- a/charts/datawolf/Chart.yaml +++ b/charts/datawolf/Chart.yaml @@ -13,7 +13,7 @@ version: 1.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 4.6.0 +appVersion: 4.7.0 # List of people that maintain this helm chart. maintainers: @@ -38,6 +38,5 @@ dependencies: # annotations for artifact.io annotations: artifacthub.io/changes: | - - add ability to set dataset permission (public/private) - - fix Chart.yaml - - fix ingress (deploy at /datawolf) + - User information to environment of process running the tool + - Ability for extra environment variables (used by IN-CORE) diff --git a/charts/datawolf/templates/deployment.yaml b/charts/datawolf/templates/deployment.yaml index d9c3cacb..9b5f5ec8 100644 --- a/charts/datawolf/templates/deployment.yaml +++ b/charts/datawolf/templates/deployment.yaml @@ -94,6 +94,9 @@ spec: value: {{ .Values.jobs.cpu | quote }} - name: KUBERNETES_MEMORY value: {{ .Values.jobs.memory | quote }} + {{- if .Values.extraEnvVars }} + {{ .Values.extraEnvVars | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: {{ include "datawolf.fullname" . }} mountPath: /home/datawolf/data diff --git a/charts/datawolf/values.yaml b/charts/datawolf/values.yaml index 3c4838e5..50cd593a 100644 --- a/charts/datawolf/values.yaml +++ b/charts/datawolf/values.yaml @@ -33,6 +33,8 @@ jobs: # default memory in GB per job memory: 4.0 +extraEnvVars: {} + serviceAccount: # Specifies whether a service account should be created create: true diff --git a/datawolf-core/pom.xml b/datawolf-core/pom.xml index 77c79abd..ae1ac656 100644 --- a/datawolf-core/pom.xml +++ b/datawolf-core/pom.xml @@ -5,7 +5,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 datawolf-core diff --git a/datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Executor.java b/datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Executor.java index b3fcc77c..961e60cb 100644 --- a/datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Executor.java +++ b/datawolf-core/src/main/java/edu/illinois/ncsa/datawolf/Executor.java @@ -36,6 +36,8 @@ public abstract class Executor { private static Logger logger = LoggerFactory.getLogger(Executor.class); + protected static String DATAWOLF_USER = "DATAWOLF_USER"; + private StringBuilder log = new StringBuilder(); private LogFile logfile = new LogFile(); private int lastsave = 0; diff --git a/datawolf-doc/doc/manual/index.html b/datawolf-doc/doc/manual/index.html index 9b5056ee..7eca2dfe 100644 --- a/datawolf-doc/doc/manual/index.html +++ b/datawolf-doc/doc/manual/index.html @@ -247,18 +247,18 @@ https://opensource.ncsa.illinois.edu/projects/artifacts.php?key=WOLF -By default, the latest release is selected in the page (currently 4.6.0). To get early access to development releases, check the box **Show also prereleases.** +By default, the latest release is selected in the page (currently 4.7.0). To get early access to development releases, check the box **Show also prereleases.** * Click on **Version** - * Select **4.6.0** - * Under **Files** select **datawolf-webapp-all-4.6.0-bin.zip** + * Select **4.7.0** + * Under **Files** select **datawolf-webapp-all-4.7.0-bin.zip** * Click **I Accept** to accept the License. This will give you the latest stable build that includes both the Data Wolf Server and the Web Editor. You can also find links to the javacode there as well as the manual. The link to the source code can be found at the end of this document. ### Installation and Setup -To install the files necessary for the Server and Editor, find where you downloaded Data Wolf and unzip it somewhere. This will create a folder called **datawolf-webapp-all-4.6.0**. In the next few sections, we'll discuss some of the important files that come with the installation you just unzipped so you can tailor your setup to meet your needs. If you wish to skip this, you can go directly to the section **Running Data Wolf Server and Editor**. +To install the files necessary for the Server and Editor, find where you downloaded Data Wolf and unzip it somewhere. This will create a folder called **datawolf-webapp-all-4.7.0**. In the next few sections, we'll discuss some of the important files that come with the installation you just unzipped so you can tailor your setup to meet your needs. If you wish to skip this, you can go directly to the section **Running Data Wolf Server and Editor**. #### Data Wolf properties @@ -443,7 +443,7 @@ #### Launch Scripts -If you go back to the folder **Data Wolf-webapp-all-4.6.0** you will see a sub-folder called **bin**, open this. Inside you will find two scripts, **datawolf-service** and **datawolf-service.bat**. The latter is intended for running Data Wolf on a Windows machine and the former is for running on Mac & Linux. As with the previous section, knowledge of this file is not required unless you are interested in configuring the Data Wolf Server and Editor beyond the default settings. We will show snippets of the file **datawolf-service** and discuss what each section is configuring. +If you go back to the folder **Data Wolf-webapp-all-4.7.0** you will see a sub-folder called **bin**, open this. Inside you will find two scripts, **datawolf-service** and **datawolf-service.bat**. The latter is intended for running Data Wolf on a Windows machine and the former is for running on Mac & Linux. As with the previous section, knowledge of this file is not required unless you are interested in configuring the Data Wolf Server and Editor beyond the default settings. We will show snippets of the file **datawolf-service** and discuss what each section is configuring. ``` # port for the jetty server diff --git a/datawolf-doc/pom.xml b/datawolf-doc/pom.xml index f427a849..8630096c 100644 --- a/datawolf-doc/pom.xml +++ b/datawolf-doc/pom.xml @@ -3,7 +3,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 datawolf-doc diff --git a/datawolf-domain/pom.xml b/datawolf-domain/pom.xml index 47e76a6e..a0389162 100644 --- a/datawolf-domain/pom.xml +++ b/datawolf-domain/pom.xml @@ -5,7 +5,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 datawolf-domain diff --git a/datawolf-editor/pom.xml b/datawolf-editor/pom.xml index d5a19f4d..4ee16aec 100644 --- a/datawolf-editor/pom.xml +++ b/datawolf-editor/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 war datawolf-editor diff --git a/datawolf-executor-commandline/pom.xml b/datawolf-executor-commandline/pom.xml index 1ddd446b..e380f728 100644 --- a/datawolf-executor-commandline/pom.xml +++ b/datawolf-executor-commandline/pom.xml @@ -4,7 +4,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-executor-commandline diff --git a/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java b/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java index 27189e16..afbf110f 100644 --- a/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java +++ b/datawolf-executor-commandline/src/main/java/edu/illinois/ncsa/datawolf/executor/commandline/CommandLineExecutor.java @@ -108,6 +108,12 @@ public void execute(File cwd) throws AbortException, FailedException { env.putAll(impl.getEnv()); } + // Add user to the environment in case a tool needs this information + if(execution.getCreator() != null) { + String user = execution.getCreator().getEmail(); + env.put(DATAWOLF_USER, user); + } + // find the app to execute command.add(findApp(impl.getExecutable().trim(), cwd)); @@ -170,9 +176,14 @@ public void execute(File cwd) throws AbortException, FailedException { throw (new FailedException("Could not get input file.", e)); } } else { - // Create a folder for the datasets File inputFolder = new File(filename); + if (inputFolder.exists() && inputFolder.getAbsolutePath().startsWith(System.getProperty("java.io.tmpdir"))) { + // For single file, a tmp file got created above; however in this case, we need + // a temporary folder to store the files + inputFolder.delete(); + } + if (!inputFolder.mkdirs()) { throw (new FailedException("Could not create folder for input files")); } @@ -251,6 +262,7 @@ public void execute(File cwd) throws AbortException, FailedException { sb.append(" "); } println("Executing : " + sb.toString()); + logger.debug("Executing : " + sb.toString()); // create the process builder ProcessBuilder pb = new ProcessBuilder(command); @@ -369,11 +381,11 @@ public void execute(File cwd) throws AbortException, FailedException { ds.setTitle(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle()); ds.setCreator(execution.getCreator()); + ds = datasetDao.save(ds); + ByteArrayInputStream bais = new ByteArrayInputStream(stdout.toString().getBytes("UTF-8")); FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle(), bais, execution.getCreator(), ds); - ds = datasetDao.save(ds); - execution.setDataset(step.getOutputs().get(impl.getCaptureStdOut()), ds.getId()); saveExecution = true; } catch (IOException exc) { @@ -385,11 +397,11 @@ public void execute(File cwd) throws AbortException, FailedException { Dataset ds = new Dataset(); ds.setTitle(step.getTool().getOutput(impl.getCaptureStdErr()).getTitle()); ds.setCreator(execution.getCreator()); + ds = datasetDao.save(ds); ByteArrayInputStream bais = new ByteArrayInputStream(stderr.toString().getBytes("UTF-8")); FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdErr()).getTitle(), bais, execution.getCreator(), ds); - ds = datasetDao.save(ds); execution.setDataset(step.getOutputs().get(impl.getCaptureStdErr()), ds.getId()); saveExecution = true; @@ -419,15 +431,15 @@ public boolean accept(File pathname) { for (File file : files) { logger.debug("adding files to a dataset: " + file); FileInputStream fis = new FileInputStream(file); - fileStorage.storeFile(file.getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(file.getName(), fis, execution.getCreator(), ds); fis.close(); } } else { FileInputStream fis = new FileInputStream(entry.getValue()); - fileStorage.storeFile(new File(entry.getValue()).getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(new File(entry.getValue()).getName(), fis, execution.getCreator(), ds); } - ds = datasetDao.save(ds); +// ds = datasetDao.save(ds); execution.setDataset(step.getOutputs().get(entry.getKey()), ds.getId()); saveExecution = true; diff --git a/datawolf-executor-hpc/pom.xml b/datawolf-executor-hpc/pom.xml index 0352ad81..08c84846 100644 --- a/datawolf-executor-hpc/pom.xml +++ b/datawolf-executor-hpc/pom.xml @@ -4,7 +4,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-executor-hpc @@ -27,7 +27,7 @@ com.jcraft jsch - 0.1.48 + 0.1.54 diff --git a/datawolf-executor-java-tool/pom.xml b/datawolf-executor-java-tool/pom.xml index f0d7076d..1435892c 100644 --- a/datawolf-executor-java-tool/pom.xml +++ b/datawolf-executor-java-tool/pom.xml @@ -4,7 +4,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-executor-java-tool diff --git a/datawolf-executor-java/pom.xml b/datawolf-executor-java/pom.xml index e08a0bf1..d3413af3 100644 --- a/datawolf-executor-java/pom.xml +++ b/datawolf-executor-java/pom.xml @@ -5,7 +5,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-executor-java diff --git a/datawolf-executor-kubernetes/pom.xml b/datawolf-executor-kubernetes/pom.xml index f9bb119c..ef1bbcdd 100644 --- a/datawolf-executor-kubernetes/pom.xml +++ b/datawolf-executor-kubernetes/pom.xml @@ -3,7 +3,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 datawolf-executor-kubernetes diff --git a/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java b/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java index c3a8f0f9..34c1ce44 100644 --- a/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java +++ b/datawolf-executor-kubernetes/src/main/java/edu/illinois/ncsa/datawolf/executor/kubernetes/KubernetesExecutor.java @@ -159,10 +159,17 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException { // Create a folder for the datasets File inputFolder = new File(filename); + if (inputFolder.exists()) { + // For single file, a tmp file got created above; however in this case, we need + // a temporary folder to store the files + inputFolder.delete(); + } + if (!inputFolder.mkdirs()) { throw (new FailedException("Could not create folder for input files")); } + int duplicate = 1; for (FileDescriptor fd : ds.getFileDescriptors()) { String localFileName = fd.getFilename(); @@ -285,8 +292,25 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException { container.args(command); // add any environment variables if (!impl.getEnv().isEmpty()) { - // TODO implement - //container.addEnvItem(); + Map environment = impl.getEnv(); + + for (Map.Entry entry : environment.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + V1EnvVar envVar = new V1EnvVar(); + envVar.setName(key); + envVar.setValue(value); + container.addEnvItem(envVar); + } + } + + // Add user to the environment in case a tool needs this information + if(execution.getCreator() != null) { + String user = execution.getCreator().getEmail(); + V1EnvVar envVar = new V1EnvVar(); + envVar.setName(DATAWOLF_USER); + envVar.setValue(user); + container.addEnvItem(envVar); } // add resource limits @@ -318,7 +342,7 @@ public State submitRemoteJob(File cwd) throws AbortException, FailedException { throw e; } catch (FailedException e) { // Job could not be submitted, set state to waiting to try again - logger.info("Job not submitted because the job scheduler appears to be down, will try again shortly..."); + logger.info("Job not submitted because the job scheduler appears to be down, will try again shortly...", e); return State.WAITING; // throw e; } catch (Throwable e) { @@ -384,12 +408,11 @@ public State checkRemoteJob() throws FailedException { Dataset ds = new Dataset(); ds.setTitle(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle()); ds.setCreator(execution.getCreator()); + ds = datasetDao.save(ds); ByteArrayInputStream bais = new ByteArrayInputStream(lastlog.getBytes("UTF-8")); FileDescriptor fd = fileStorage.storeFile(step.getTool().getOutput(impl.getCaptureStdOut()).getTitle(), bais, execution.getCreator(), ds); - ds = datasetDao.save(ds); - execution.setDataset(step.getOutputs().get(impl.getCaptureStdOut()), ds.getId()); saveExecution = true; } @@ -419,15 +442,15 @@ public boolean accept(File pathname) { for (File file : files) { logger.debug("adding files to a dataset: " + file); FileInputStream fis = new FileInputStream(file); - fileStorage.storeFile(file.getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(file.getName(), fis, execution.getCreator(), ds); fis.close(); } } else { FileInputStream fis = new FileInputStream(entry.getValue()); - fileStorage.storeFile(new File(entry.getValue()).getName(), fis, ds.getCreator(), ds); + fileStorage.storeFile(new File(entry.getValue()).getName(), fis, execution.getCreator(), ds); } - ds = datasetDao.save(ds); +// ds = datasetDao.save(ds); execution.setDataset(step.getOutputs().get(entry.getKey()), ds.getId()); saveExecution = true; diff --git a/datawolf-jpa/pom.xml b/datawolf-jpa/pom.xml index 675f30aa..0ef0c6d3 100644 --- a/datawolf-jpa/pom.xml +++ b/datawolf-jpa/pom.xml @@ -4,7 +4,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-jpa jar diff --git a/datawolf-provenance/pom.xml b/datawolf-provenance/pom.xml index 1d1114a4..2d69dbfb 100644 --- a/datawolf-provenance/pom.xml +++ b/datawolf-provenance/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 war datawolf-provenance diff --git a/datawolf-service-client/pom.xml b/datawolf-service-client/pom.xml index 39f59f72..2e746c45 100644 --- a/datawolf-service-client/pom.xml +++ b/datawolf-service-client/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 datawolf-service-client diff --git a/datawolf-service/pom.xml b/datawolf-service/pom.xml index 91ff9841..c639849a 100644 --- a/datawolf-service/pom.xml +++ b/datawolf-service/pom.xml @@ -4,7 +4,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-service diff --git a/datawolf-tool-example/pom.xml b/datawolf-tool-example/pom.xml index f472d28a..088de345 100644 --- a/datawolf-tool-example/pom.xml +++ b/datawolf-tool-example/pom.xml @@ -5,7 +5,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf-tool-example diff --git a/datawolf-webapp-all/pom.xml b/datawolf-webapp-all/pom.xml index 0f450814..00e26f9a 100644 --- a/datawolf-webapp-all/pom.xml +++ b/datawolf-webapp-all/pom.xml @@ -3,7 +3,7 @@ 4.0.0 datawolf - 4.6.0 + 4.7.0 edu.illinois.ncsa war @@ -40,6 +40,7 @@ hsqldb ${hsql.version} runtime + jdk8 mysql diff --git a/datawolf-webapp/pom.xml b/datawolf-webapp/pom.xml index 6323cee6..8d3a0736 100644 --- a/datawolf-webapp/pom.xml +++ b/datawolf-webapp/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 war datawolf-webapp @@ -28,6 +28,7 @@ hsqldb ${hsql.version} runtime + jdk8 mysql diff --git a/docker-compose.yml b/docker-compose.yml index 72baf2d5..0303163a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,11 @@ services: - ENGINE_PAGESIZE=${ENGINE_PAGESIZE:-250} - EXECUTOR_DEBUG=${EXECUTOR_DEBUG:-false} - DATASET_PERMISSIONS=${DATASET_PERMISSIONS:-private} + - DATASET_DAO=${DATASET_DAO:-edu.illinois.ncsa.jpa.dao.DatasetJPADao} + - FILE_STORAGE=${FILE_STORAGE:-edu.illinois.ncsa.domain.impl.FileStorageDisk} + - INCORE_USER=${INCORE_USER:-incrtst} + - INCORE_GROUP=${INCORE_GROUP:-incore_user} + - INCORE_SERVER=${INCORE_SERVER:-http://localhost:8080} - LOG=${LOG} ports: - 8888:8888 diff --git a/docker/custom.properties b/docker/custom.properties index 36d15fab..b517991c 100644 --- a/docker/custom.properties +++ b/docker/custom.properties @@ -1,5 +1,8 @@ + +# Initial accounts for administrator(s) initialAdmins=${DATAWOLF_ADMINS} +# Database configuration hibernate.hikari.dataSourceClassName=${DB_CLASS_NAME} hibernate.dialect=${DB_DIALECT} hibernate.hikari.dataSource.url=${DB_SOURCE_URL} @@ -8,18 +11,31 @@ hibernate.hikari.idleTimeout=${DB_IDLE_TIMEOUT} hibernate.hikari.dataSource.user=${DB_USER} hibernate.hikari.dataSource.password=${DB_PASSWORD} +# Engine configuration engine.storeLogs=${ENGINE_STORELOGS} engine.timeout=${ENGINE_TIMEOUT} engine.extraLocalExecutor=${ENGINE_EXTRALOCALEXECUTOR} engine.localExecutorThreads=${ENGINE_LOCALEXECUTORTHREADS} engine.pageSize=${ENGINE_PAGESIZE} +executor.debug=${EXECUTOR_DEBUG} +#Kubernetes configuration kubernetes.namespace=${KUBERNETES_NAMESPACE} kubernetes.pvc=${KUBERNETES_PVC} kubernetes.data=${KUBERNETES_DATA} kubernetes.cpu=${KUBERNETES_CPU} kubernetes.memory=${KUBERNETES_MEMORY} -executor.debug=${EXECUTOR_DEBUG} +# DAO Configurations +dataset.dao=${DATASET_DAO} + +# File storage configuration +filestorage=${FILE_STORAGE} + +# IN-CORE specific configuration +incore.server=${INCORE_SERVER} +incore.group=${INCORE_GROUP} +incore.user=${INCORE_USER} +# Dataset permissions dataset.permissions=${DATASET_PERMISSIONS} diff --git a/file-service-client/pom.xml b/file-service-client/pom.xml index d6da580e..bd2346c0 100644 --- a/file-service-client/pom.xml +++ b/file-service-client/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 file-service-client diff --git a/file-service/pom.xml b/file-service/pom.xml index 36f8965a..9af4f7d1 100644 --- a/file-service/pom.xml +++ b/file-service/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 jar file-service diff --git a/file-webapp/pom.xml b/file-webapp/pom.xml index 9093f05f..a8a6e606 100644 --- a/file-webapp/pom.xml +++ b/file-webapp/pom.xml @@ -4,7 +4,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 war file-webapp @@ -57,6 +57,7 @@ hsqldb ${hsql.version} runtime + jdk8 mysql diff --git a/gondola/pom.xml b/gondola/pom.xml index 0168eb87..4445624d 100644 --- a/gondola/pom.xml +++ b/gondola/pom.xml @@ -5,7 +5,7 @@ datawolf edu.illinois.ncsa - 4.6.0 + 4.7.0 gondola jar diff --git a/ncsa-common-clowder/pom.xml b/ncsa-common-clowder/pom.xml index f2ccf082..65174fe8 100644 --- a/ncsa-common-clowder/pom.xml +++ b/ncsa-common-clowder/pom.xml @@ -4,7 +4,7 @@ ncsa-common-clowder edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf @@ -45,7 +45,7 @@ com.google.code.gson gson - 2.3 + 2.8.9 junit diff --git a/ncsa-common-domain/pom.xml b/ncsa-common-domain/pom.xml index 700602ad..81b36335 100644 --- a/ncsa-common-domain/pom.xml +++ b/ncsa-common-domain/pom.xml @@ -5,7 +5,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 ncsa-common-domain diff --git a/ncsa-common-incore/pom.xml b/ncsa-common-incore/pom.xml index 93194045..00b75181 100644 --- a/ncsa-common-incore/pom.xml +++ b/ncsa-common-incore/pom.xml @@ -3,7 +3,7 @@ ncsa-common-incore edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf @@ -43,7 +43,7 @@ com.google.code.gson gson - 2.3 + 2.8.9 junit diff --git a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/IncoreDataset.java b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/IncoreDataset.java index 94e480ca..4f7fbe20 100644 --- a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/IncoreDataset.java +++ b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/IncoreDataset.java @@ -9,6 +9,10 @@ public class IncoreDataset { + // Auth API + public static final String X_AUTH_USERINFO = "x-auth-userinfo"; + public static final String X_AUTH_USERGROUP = "x-auth-usergroup"; + // Datasets API public static final String DATASETS_ENDPOINT = "data/api/datasets"; public static final String PARENT_DATASET = "parentdataset"; @@ -53,6 +57,7 @@ public static Dataset getDataset(JsonObject datasetProperties, Person creator) { dataset.setDescription(description); dataset.setTitle(title); + for (int index = 0; index < fileDescriptors.size(); index++) { JsonObject fileDescriptor = fileDescriptors.get(index).getAsJsonObject(); diff --git a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/AbstractIncoreDao.java b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/AbstractIncoreDao.java index c2f62bf6..24b2416a 100644 --- a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/AbstractIncoreDao.java +++ b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/AbstractIncoreDao.java @@ -12,6 +12,14 @@ public abstract class AbstractIncoreDao implements I @Named("incore.server") private String server; + @Inject + @Named("incore.group") + private String group; + + @Inject + @Named("incore.user") + private String incoreUser; + /** * IN-CORE Service endpoint * @@ -25,4 +33,20 @@ public String getServer() { return this.server; } + /** + * Primary IN-CORE user group + * @return + */ + public String getGroup() { + return this.group; + } + + /** + * DataWolf User that can access data on IN-CORE services + * @return + */ + public String getIncoreUser() { + return this.incoreUser; + } + } diff --git a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/IncoreDatasetDao.java b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/IncoreDatasetDao.java index 4bcf4228..8db06582 100644 --- a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/IncoreDatasetDao.java +++ b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/dao/IncoreDatasetDao.java @@ -1,11 +1,17 @@ package edu.illinois.ncsa.incore.dao; +import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import javax.inject.Inject; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.ClientProtocolException; @@ -24,7 +30,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.google.gson.JsonPrimitive; import edu.illinois.ncsa.domain.Dataset; import edu.illinois.ncsa.domain.Person; @@ -42,40 +47,44 @@ public class IncoreDatasetDao extends AbstractIncoreDao impleme @Override public Dataset save(Dataset entity) { + // Uncomment for local testing +// String bearerToken = getToken(); // Check if this is a new entity (e.g. still has UUID) if (entity.getId().contains("-")) { - + // The description should contain the type and format to store // TODO is there a better way? String description = entity.getDescription(); String[] datasetInfo = description.split(","); - String schema = "Unknown"; String type = "Unknown"; String format = "Unknown"; - if (datasetInfo.length == 3) { - schema = datasetInfo[0]; - type = datasetInfo[1]; - format = datasetInfo[2]; + if (datasetInfo.length == 2) { + type = datasetInfo[0]; + format = datasetInfo[1]; + logger.debug("Saving data with dataType = " + type + ", format = " + format); + } else { + // Assume this is something like stdout which didn't define anything + logger.debug("Saving plain text file with type = incore:stdout"); + type = "incore:stdout"; + format = "text"; } JsonObject jsonObject = new JsonObject(); - jsonObject.addProperty("schema", schema); - jsonObject.addProperty("type", type); + jsonObject.addProperty("dataType", type); jsonObject.addProperty("title", entity.getTitle()); jsonObject.addProperty("sourceDataset", ""); jsonObject.addProperty("format", format); - JsonArray spaces = new JsonArray(); - String creatorId = null; + String creatorEmail = null; if (entity.getCreator() != null) { creatorId = entity.getCreator().getId(); - spaces.add(new JsonPrimitive(creatorId)); + creatorEmail = entity.getCreator().getEmail(); } - jsonObject.add("spaces", spaces); String incoreEndpoint = getServer(); String requestUrl = incoreEndpoint; + HttpClientBuilder builder = HttpClientBuilder.create(); HttpClient httpclient = builder.build(); @@ -86,8 +95,14 @@ public Dataset save(Dataset entity) { requestUrl += IncoreDataset.DATASETS_ENDPOINT; HttpPost httpPost = new HttpPost(requestUrl); + if (creatorId != null) { - httpPost.setHeader("X-Credential-Username", creatorId); + // This will only work for internal communication, for testing, we add the auth token + String creatorGroup = "[\""+ this.getGroup() + "\"]"; + httpPost.setHeader(IncoreDataset.X_AUTH_USERINFO, "{\"preferred_username\": \"" + creatorEmail + "\"}"); + httpPost.setHeader(IncoreDataset.X_AUTH_USERGROUP, "{\"groups\": " + creatorGroup + "}" ); + // Uncomment for local testing +// httpPost.setHeader("Authorization", bearerToken); } MultipartEntityBuilder params = MultipartEntityBuilder.create(); @@ -128,6 +143,9 @@ public void delete(Dataset entity) {} @Override public Dataset findOne(String id) { + // Uncomment for local testing +// String bearerToken = getToken(); + String incoreEndpoint = getServer(); String requestUrl = incoreEndpoint; @@ -142,6 +160,20 @@ public Dataset findOne(String id) { requestUrl += IncoreDataset.DATASETS_ENDPOINT + "/" + id; HttpGet httpGet = new HttpGet(requestUrl); + // This is a workaround for data that is stored outside of DataWolf + // DataWolf assumes a level of trust that only authenticated/authorized users could reach here + // This works fine with local storage/data; however, when accessing an external source, the user + // info, which we don't have, might be needed. + // A potential solution could be to add the requester info to the method + String user = this.getIncoreUser(); + String creatorGroup = "[\""+ this.getGroup() + "\"]"; + + httpGet.setHeader(IncoreDataset.X_AUTH_USERINFO, "{\"preferred_username\": \"" + user + "\"}"); + httpGet.setHeader(IncoreDataset.X_AUTH_USERGROUP, "{\"groups\": " + creatorGroup + "}" ); + + // Uncomment for local testing +// httpGet.setHeader("Authorization", bearerToken); + response = httpclient.execute(httpGet); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { @@ -238,9 +270,16 @@ public List findAll(int page, int size) { String incoreEndpoint = getServer(); String requestUrl = incoreEndpoint; + + // TODO if this method is needed, we will need to get this from the request + String creatorEmail = ""; try { requestUrl += IncoreDataset.DATASETS_ENDPOINT; HttpGet httpGet = new HttpGet(requestUrl); + String creatorGroup = "[\""+ this.getGroup() + "\"]"; + + httpGet.setHeader(IncoreDataset.X_AUTH_USERINFO, "{\"preferred_username\": \"" + creatorEmail + "\"}"); + httpGet.setHeader(IncoreDataset.X_AUTH_USERGROUP, "{\"groups\": " + creatorGroup + "}" ); ResponseHandler responseHandler = new BasicResponseHandler(); try { @@ -360,8 +399,63 @@ public List findByCreatorEmail(String email, int page, int size) { @Override public List findByCreatorEmailAndDeleted(String email, boolean deleted, int page, int size) { - // TODO Auto-generated method stub - return null; + // Uncomment for local testing +// String bearerToken = getToken(); + + List results = new ArrayList(); + + String responseStr = null; + HttpClientBuilder builder = HttpClientBuilder.create(); + HttpClient httpclient = builder.build(); + + String incoreEndpoint = getServer(); + String requestUrl = incoreEndpoint; + try { + requestUrl += IncoreDataset.DATASETS_ENDPOINT + "?skip="+page + "&limit="+size; + HttpGet httpGet = new HttpGet(requestUrl); + + String creatorGroup = "[\""+ this.getGroup() + "\"]"; + httpGet.setHeader(IncoreDataset.X_AUTH_USERINFO, "{\"preferred_username\": \"" + email + "\"}"); + httpGet.setHeader(IncoreDataset.X_AUTH_USERGROUP, "{\"groups\": " + creatorGroup + "}" ); + // Uncomment for local testing +// httpGet.setHeader("Authorization", bearerToken); + ResponseHandler responseHandler = new BasicResponseHandler(); + + try { + responseStr = httpclient.execute(httpGet, responseHandler); + JsonElement jsonElement = new JsonParser().parse(responseStr); + JsonArray jsonArray = jsonElement.getAsJsonArray(); + + Dataset dataset = null; + for (int index = 0; index < jsonArray.size(); index++) { + + JsonObject datasetProperties = jsonArray.get(index).getAsJsonObject(); + + if (datasetProperties != null) { + Person creator = null; + if (datasetProperties.has(IncoreDataset.CREATOR)) { + String creatorId = datasetProperties.get(IncoreDataset.CREATOR).getAsString(); + if (creatorId != null) { + creator = personDao.findByEmail(creatorId); + } + } + dataset = IncoreDataset.getDataset(datasetProperties, creator); + results.add(dataset); + } + + } + } catch (Exception e) { + logger.error("HTTP Get failed.", e); + } + + } finally { + try { + ((CloseableHttpClient) httpclient).close(); + } catch (IOException e) { + logger.warn("Error closing http client", e); + } + } + return results; } @Override @@ -376,4 +470,27 @@ public List findByCreatorEmailAndTitleLikeAndDeleted(String email, Stri return null; } + // Helper method for testing with a local instance + public String getToken() { + String server = getServer(); + // pyincore uses the server URL without ending in / when creating the token name + if(server.endsWith("/")) { + server = server.substring(0, server.length() - 1); + } + + String tokenFile = "." + DigestUtils.sha256Hex(server) + "_token"; + String bearerToken = null; + try { + String userHome = System.getProperty("user.home"); + Path path = Paths.get(userHome + "/.incore/" + tokenFile); + if (Files.exists(path)) { + byte[] encoded = Files.readAllBytes(path); + bearerToken = new String(encoded, StandardCharsets.UTF_8); + } + } catch(IOException e) { + logger.error("Error getting token", e); + } + return bearerToken; + } + } diff --git a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/impl/IncoreFileStorage.java b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/impl/IncoreFileStorage.java index d38a53e9..4a4949e7 100644 --- a/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/impl/IncoreFileStorage.java +++ b/ncsa-common-incore/src/main/java/edu/illinois/ncsa/incore/impl/IncoreFileStorage.java @@ -8,7 +8,12 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -45,6 +50,10 @@ public class IncoreFileStorage implements FileStorage { @Named("incore.server") private String server; + @Inject + @Named("incore.group") + private String group; + @Inject(optional = true) private AccountDao accountDao; @@ -77,24 +86,9 @@ public FileDescriptor storeFile(String id, String filename, InputStream is) thro @Override public FileDescriptor storeFile(String id, String filename, InputStream is, Person creator, Dataset ds) throws IOException { - // TODO we should clean up this temporary file - // Write file to temp file to remove source dataset id - File cwd = File.createTempFile("cib", ".dir"); //$NON-NLS-1$ //$NON-NLS-2$ - cwd.delete(); - if (!cwd.mkdirs()) { - throw (new IOException("Error creating temp directory.")); - } - - File output = new File(cwd, filename); - String datasetId = null; - try (final BufferedReader br = new BufferedReader(new InputStreamReader(is)); BufferedWriter bw = new BufferedWriter(new FileWriter(output));) { - datasetId = br.readLine(); - String line = null; - - while ((line = br.readLine()) != null) { - bw.write(line + "\n"); - } - } + logger.debug("Store the file on the IN-CORE service"); + // Uncomment this for testing locally +// String bearerToken = getToken(); String incoreEndpoint = server; if (!incoreEndpoint.endsWith("/")) { @@ -106,43 +100,52 @@ public FileDescriptor storeFile(String id, String filename, InputStream is, Pers HttpClientBuilder builder = HttpClientBuilder.create(); HttpClient httpclient = builder.build(); - // Hack to update the source dataset - JsonObject jsonObject = new JsonObject(); - jsonObject.addProperty("property name", "sourceDataset"); - jsonObject.addProperty("property value", datasetId); - - MultipartEntityBuilder paramBuilder = MultipartEntityBuilder.create(); - paramBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); - paramBuilder.addTextBody("update", jsonObject.toString()); - - HttpPut httpPut = new HttpPut(requestUrl); - httpPut.setEntity(paramBuilder.build()); - - if (creator != null) { - httpPut.setHeader("X-Credential-Username", creator.getId()); - } - + // This was an old Hack to update the source dataset, but doesn't work with the current pyincore + // This might be something to handle on the frontend when visualizing the outputs and updating the parent from + // the execution +// JsonObject jsonObject = new JsonObject(); +// jsonObject.addProperty("property name", "sourceDataset"); +// jsonObject.addProperty("property value", datasetId); +// +// MultipartEntityBuilder paramBuilder = MultipartEntityBuilder.create(); +// paramBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); +// paramBuilder.addTextBody("update", jsonObject.toString()); +// +// HttpPut httpPut = new HttpPut(requestUrl); +// httpPut.setEntity(paramBuilder.build()); +// +// if (creator != null) { +// httpPut.setHeader("X-Credential-Username", creator.getId()); +// } +// HttpResponse response = null; ResponseHandler responseHandler = new BasicResponseHandler(); - response = httpclient.execute(httpPut); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - logger.warn("failed to update parent dataset - " + response); - } +// response = httpclient.execute(httpPut); +// if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { +// logger.warn("failed to update parent dataset - " + response); +// } // Add the file to the dataset requestUrl = incoreEndpoint; requestUrl += IncoreDataset.DATASETS_ENDPOINT + "/" + ds.getId() + "/" + IncoreDataset.DATASET_FILES; - paramBuilder = MultipartEntityBuilder.create(); + MultipartEntityBuilder paramBuilder = MultipartEntityBuilder.create(); paramBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); - paramBuilder.addTextBody(IncoreDataset.PARENT_DATASET, jsonObject.toString()); - paramBuilder.addBinaryBody(IncoreDataset.DATASET_FILE, output, ContentType.DEFAULT_BINARY, filename); +// paramBuilder.addTextBody(IncoreDataset.PARENT_DATASET, jsonObject.toString()); + paramBuilder.addBinaryBody(IncoreDataset.DATASET_FILE, is, ContentType.DEFAULT_BINARY, filename); HttpPost httpPost = new HttpPost(requestUrl); httpPost.setEntity(paramBuilder.build()); if (creator != null) { - httpPost.setHeader("X-Credential-Username", creator.getId()); + // Uncomment this for testing locally +// httpPost.setHeader("Authorization", bearerToken); + + String creatorGroup = "[\""+ this.group + "\"]"; + String creatorId = creator.getEmail(); + httpPost.setHeader(IncoreDataset.X_AUTH_USERINFO, "{\"preferred_username\": \"" + creatorId + "\"}"); + httpPost.setHeader(IncoreDataset.X_AUTH_USERGROUP, "{\"groups\": " + creatorGroup + "}" ); + } response = httpclient.execute(httpPost); @@ -165,6 +168,8 @@ public FileDescriptor storeFile(String id, String filename, InputStream is, Pers @Override public InputStream readFile(FileDescriptor fd) throws IOException { + // Uncomment for local testing +// String bearerToken = getToken(); String incoreEndpoint = server; if (!incoreEndpoint.endsWith("/")) { incoreEndpoint += "/"; @@ -174,7 +179,15 @@ public InputStream readFile(FileDescriptor fd) throws IOException { HttpClientBuilder builder = HttpClientBuilder.create(); HttpClient httpclient = builder.build(); + + String creatorId = ""; + String creatorGroup = "[\""+ this.group + "\"]"; + HttpGet httpGet = new HttpGet(requestUrl); + // Uncomment for local testing +// httpGet.setHeader("Authorization", bearerToken); + httpGet.setHeader(IncoreDataset.X_AUTH_USERINFO, "{\"preferred_username\": \"" + creatorId + "\"}"); + httpGet.setHeader(IncoreDataset.X_AUTH_USERGROUP, "{\"groups\": " + creatorGroup + "}" ); HttpResponse response = httpclient.execute(httpGet); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { @@ -198,4 +211,27 @@ public boolean deleteFile(FileDescriptor fd, Person creator) { return false; } + // Helper method for testing with a local instance + public String getToken() { + String server = this.server; + // pyincore uses the server URL without ending in / when creating the token name + if(server.endsWith("/")) { + server = server.substring(0, server.length() - 1); + } + + String tokenFile = "." + DigestUtils.sha256Hex(server) + "_token"; + String bearerToken = null; + try { + String userHome = System.getProperty("user.home"); + Path path = Paths.get(userHome + "/.incore/" + tokenFile); + if (Files.exists(path)) { + byte[] encoded = Files.readAllBytes(path); + bearerToken = new String(encoded, StandardCharsets.UTF_8); + } + } catch(IOException e) { + logger.error("Error getting token", e); + } + return bearerToken; + } + } diff --git a/ncsa-common-jpa/pom.xml b/ncsa-common-jpa/pom.xml index 4eac1d29..9c0bd9e0 100644 --- a/ncsa-common-jpa/pom.xml +++ b/ncsa-common-jpa/pom.xml @@ -5,7 +5,7 @@ edu.illinois.ncsa datawolf - 4.6.0 + 4.7.0 @@ -73,6 +73,7 @@ hsqldb ${hsql.version} test + jdk8 diff --git a/pom.xml b/pom.xml index 287c3acf..515542b5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 edu.illinois.ncsa - 4.6.0 + 4.7.0 datawolf pom @@ -11,7 +11,7 @@ UTF-8 1.0.0.Final 4.3.5.Final - 2.3.2 + 2.7.3 5.1.34 42.4.0 4.5.13 @@ -159,6 +159,7 @@ hsqldb ${hsql.version} test + jdk8 junit