From 8f743ea987464d820d1fd63528a51d9d8dcd9165 Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Tue, 23 Jan 2024 13:36:11 -0600 Subject: [PATCH 1/4] Added raster geopackage checker --- CHANGELOG.md | 5 +++- .../data/controllers/DatasetController.java | 23 +++++++++++++---- .../service/data/utils/GeotoolsUtils.java | 25 +++++++++++++++---- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 817765fb..f19c12c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added +- GeoPakage handling in services [#205](https://github.com/IN-CORE/incore-services/issues/205) +- Raster GeoPackage checker for not supporting it [#258](https://github.com/IN-CORE/incore-services/issues/258) + ### Changed - Use Java models to represent semantics [#239](https://github.com/IN-CORE/incore-services/issues/239) @@ -24,7 +28,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [1.21.0] - 2023-10-11 ### Added -- Geopakage handling in services [#205](https://github.com/IN-CORE/incore-services/issues/205) - Template dataset generation (CSV and Shapefile) API to semantics-service [#214](https://github.com/IN-CORE/incore-services/issues/214) - Owner item added to the dataset, hazard, and dfr3 object [#92](https://github.com/IN-CORE/incore-services/issues/92) - Attenuation model Sadigh et al. 1997 [#208](https://github.com/IN-CORE/incore-services/issues/208) diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java index 652fd1ab..c3122093 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java @@ -893,12 +893,25 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req } File tmpFile = new File(FilenameUtils.concat(DATA_REPO_FOLDER, dataFDs.get(0).getDataURL())); - // check if geopackage only has a single layer and the layer name is the same as file name - if (!GeotoolsUtils.isGpkgSingleLayer(tmpFile)) { + // check if geopackage only has a single layer + // and the layer name is the same as file name + // and it is not the raster data since incore-service doesn't support it yet + int isGpkgFit = GeotoolsUtils.isGpkgFitToService(tmpFile); + if (isGpkgFit != 0) { FileUtils.removeFilesFromFileDescriptor(dataset.getFileDescriptors()); - logger.debug("The geopackage has to have a single layer, and layer name should be the same as file name."); - throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, - "Geopackage is not a single layer or layer name is not the same as file name."); + if (isGpkgFit == 1) { + logger.debug("The geopackage has not vector layer or contains the raster layer that is not being supported yet."); + throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, + "The geopackage has no vector layer or contains the raster layer that is not being supported yet."); + } else if (isGpkgFit == 2) { + logger.debug("The geopackage has to have a single layer."); + throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, + "The geopackage has to have a single layer."); + } else if (isGpkgFit == 3) { + logger.debug("The geopackage's'layer name should be the same as file name."); + throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, + "The geopackage's'layer name should be the same as file name."); + } } // check if geopackage has guid diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java index 00fe6123..e92e2354 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java @@ -44,6 +44,7 @@ import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.AttributeDescriptor; +import org.opengis.feature.type.Name; import org.opengis.filter.Filter; import org.opengis.parameter.GeneralParameterValue; import org.opengis.parameter.ParameterValueGroup; @@ -969,33 +970,47 @@ public static boolean doesGuidExist(SimpleFeatureCollection inputFeatures) { } /** - * check if geopackage has single layer and layer name is the same as the file name + * check if geopackage has single layer + * and layer name is the same as the file name + * and layer is not raster + * since incore-services doesn't support raster geopackage yet * * @param inFile * @return + * 0: okay + * 1: data is raster or no vector layer + * 2: multiple vector layer + * 3: name not matching * @throws IOException */ - public static boolean isGpkgSingleLayer(File inFile) throws IOException { - Boolean output = false; + public static int isGpkgFitToService(File inFile) throws IOException { + int output = 0; try { HashMap map = new HashMap<>(); map.put(GeoPkgDataStoreFactory.DBTYPE.key, "geopkg"); map.put(GeoPkgDataStoreFactory.DATABASE.key, inFile.getAbsoluteFile()); DataStore dataStore = DataStoreFinder.getDataStore(map); + if (dataStore == null) { throw new IOException("Unable to open geopackage file"); } // get all layer names in input geopackage file + // if the layerNames list is more than one, it means there are multiple vector layer + // if the layerNames list empty then, there is no vector layer or it could be a raster data String[] layerNames = dataStore.getTypeNames(); if (layerNames.length == 1) { // check if the layername is the same as file name String layerName = layerNames[0]; String fileName = inFile.getName().split("\\.")[0]; - if (layerName.equals(fileName)) { - output = true; + if (!layerName.equals(fileName)) { + return 3; } + } else if (layerNames.length == 0) { + return 1; + } else if (layerNames.length > 1) { + return 2; } } catch (IOException e) { throw new IOException("Unable to open geopackage file."); From ca4e40968437dad9cc1a29bb139f84892d216b02 Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Tue, 30 Jan 2024 11:12:15 -0600 Subject: [PATCH 2/4] Updated geopackage checking variable to be more descriptive --- .../java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java diff --git a/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java b/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java new file mode 100644 index 00000000..0f43cfb0 --- /dev/null +++ b/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java @@ -0,0 +1,2 @@ +package edu.illinois.ncsa.incore.common.utils;public class GeoUtils { +} From f4cd57555ecf017f80ca4687c681593789d12d88 Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Tue, 30 Jan 2024 12:04:01 -0600 Subject: [PATCH 3/4] Updated geopackage checking variable to be more descriptive --- .../service/data/controllers/DatasetController.java | 11 ++++++----- .../ncsa/incore/service/data/utils/GeotoolsUtils.java | 11 ++++++----- .../illinois/ncsa/incore/common/utils/GeoUtils.java | 10 +++++++++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java index c3122093..89fb2cb1 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java @@ -27,6 +27,7 @@ import edu.illinois.ncsa.incore.common.utils.UserInfoUtils; import edu.illinois.ncsa.incore.common.utils.AllocationUtils; import edu.illinois.ncsa.incore.common.AllocationConstants; +import edu.illinois.ncsa.incore.common.utils.GeoUtils; import edu.illinois.ncsa.incore.service.data.dao.IRepository; import edu.illinois.ncsa.incore.service.data.models.Dataset; import edu.illinois.ncsa.incore.service.data.models.FileDescriptor; @@ -896,18 +897,18 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req // check if geopackage only has a single layer // and the layer name is the same as file name // and it is not the raster data since incore-service doesn't support it yet - int isGpkgFit = GeotoolsUtils.isGpkgFitToService(tmpFile); - if (isGpkgFit != 0) { + GeoUtils.gpkgValidationResult isGpkgFit = GeotoolsUtils.isGpkgFitToService(tmpFile); + if (isGpkgFit != GeoUtils.gpkgValidationResult.VALID) { FileUtils.removeFilesFromFileDescriptor(dataset.getFileDescriptors()); - if (isGpkgFit == 1) { + if (isGpkgFit == GeoUtils.gpkgValidationResult.RASTER_OR_NO_VECTOR_LAYER) { logger.debug("The geopackage has not vector layer or contains the raster layer that is not being supported yet."); throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, "The geopackage has no vector layer or contains the raster layer that is not being supported yet."); - } else if (isGpkgFit == 2) { + } else if (isGpkgFit == GeoUtils.gpkgValidationResult.MULTIPLE_VECTOR_LAYERS) { logger.debug("The geopackage has to have a single layer."); throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, "The geopackage has to have a single layer."); - } else if (isGpkgFit == 3) { + } else if (isGpkgFit == GeoUtils.gpkgValidationResult.NAME_MISMATCH) { logger.debug("The geopackage's'layer name should be the same as file name."); throw new IncoreHTTPException(Response.Status.NOT_ACCEPTABLE, "The geopackage's'layer name should be the same as file name."); diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java index e92e2354..3f9ac16b 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java @@ -12,6 +12,7 @@ import com.opencsv.CSVReader; import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; +import edu.illinois.ncsa.incore.common.utils.GeoUtils; import edu.illinois.ncsa.incore.service.data.models.Dataset; import org.apache.commons.io.FilenameUtils; import org.apache.log4j.Logger; @@ -983,7 +984,7 @@ public static boolean doesGuidExist(SimpleFeatureCollection inputFeatures) { * 3: name not matching * @throws IOException */ - public static int isGpkgFitToService(File inFile) throws IOException { + public static GeoUtils.gpkgValidationResult isGpkgFitToService(File inFile) throws IOException { int output = 0; try { HashMap map = new HashMap<>(); @@ -1005,18 +1006,18 @@ public static int isGpkgFitToService(File inFile) throws IOException { String layerName = layerNames[0]; String fileName = inFile.getName().split("\\.")[0]; if (!layerName.equals(fileName)) { - return 3; + return GeoUtils.gpkgValidationResult.NAME_MISMATCH; } } else if (layerNames.length == 0) { - return 1; + return GeoUtils.gpkgValidationResult.RASTER_OR_NO_VECTOR_LAYER; } else if (layerNames.length > 1) { - return 2; + return GeoUtils.gpkgValidationResult.MULTIPLE_VECTOR_LAYERS; } } catch (IOException e) { throw new IOException("Unable to open geopackage file."); } - return output; + return GeoUtils.gpkgValidationResult.VALID; } } diff --git a/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java b/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java index 0f43cfb0..edb89208 100644 --- a/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java +++ b/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/utils/GeoUtils.java @@ -1,2 +1,10 @@ -package edu.illinois.ncsa.incore.common.utils;public class GeoUtils { +package edu.illinois.ncsa.incore.common.utils; + +public class GeoUtils { + public static enum gpkgValidationResult { + VALID, + RASTER_OR_NO_VECTOR_LAYER, + MULTIPLE_VECTOR_LAYERS, + NAME_MISMATCH + } } From 2363f963c4476dbf4266f179f19029aacda3baba Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Tue, 30 Jan 2024 12:07:30 -0600 Subject: [PATCH 4/4] changed description --- .../ncsa/incore/service/data/utils/GeotoolsUtils.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java index 3f9ac16b..aad6d8f5 100644 --- a/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java +++ b/server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java @@ -978,10 +978,6 @@ public static boolean doesGuidExist(SimpleFeatureCollection inputFeatures) { * * @param inFile * @return - * 0: okay - * 1: data is raster or no vector layer - * 2: multiple vector layer - * 3: name not matching * @throws IOException */ public static GeoUtils.gpkgValidationResult isGpkgFitToService(File inFile) throws IOException {