283 uploaded joined shapefile to geoserver has missing values (#284)

* Fixes #283 - changed join to create geopackage instead of shapefile * Changed file upload to be the geoPkgFile * pass in dataset id as the default join (#285) * Temp fix layername (#286) * pass in dataset id as the default join * one more fix * Changed log info to debug and updated CHANGELOG --------- Co-authored-by: Chen Wang <[email protected]>
IN-CORE · Mar 13, 2024 · 55c239e · 55c239e
1 parent 057c61c
commit 55c239e
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [Unreleased]
+
+### Changed
+
+- Join Shapefile to upload GeoPackage to GeoServer instead of Shapefile [#283](https://github.com/IN-CORE/incore-services/issues/283)
+
 ## [1.25.0] - 2024-02-21
 
 ### Changed

diff --git a/...ce/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java b/...ce/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java
@@ -938,8 +938,9 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
                 File joinedShapefile = null;
                 // todo: the join process for the network dataset should be added in here.
                 try {
-                    joinedShapefile = FileUtils.joinShpTable(dataset, repository, true);
-                    if (!GeoserverUtils.uploadShapefileToGeoserver(dataset.getId(), joinedShapefile)) {
+                    //joinedShapefile = FileUtils.joinShpTable(dataset, repository, true);
+                    geoPkgFile = FileUtils.joinShpTable(dataset, repository, true);
+                    if (!GeoserverUtils.uploadGpkgToGeoserver(dataset.getId(), geoPkgFile)) {
                         logger.error("Fail to upload geopackage file");
                         throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, "Fail to upload geopakcage file.");
                     }
@@ -952,7 +953,7 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
                     throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
                 }
                 // clean up
-                FileUtils.deleteTmpDir(joinedShapefile);
+                FileUtils.deleteTmpDir(geoPkgFile);
             } else {
                 try {
                     if (format.equalsIgnoreCase(FileUtils.FORMAT_NETWORK)) {

diff --git a/...a-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeoserverRestApi.java b/...a-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeoserverRestApi.java
@@ -29,7 +29,7 @@ public class GeoserverRestApi {
     public static final String GEOSERVER_USER = System.getenv("GEOSERVER_USER");
     public static final String GEOSERVER_PW = System.getenv("GEOSERVER_PW");
     public static final String GEOSERVER_WORKSPACE = System.getenv("GEOSERVER_WORKSPACE");
-    private static final Logger logger = Logger.getLogger(GeoserverUtils.class);
+    private static final Logger logger = Logger.getLogger(GeoserverRestApi.class);
     private final String geoserverUrl;
     private final String username;
     private final String password;
@@ -164,6 +164,7 @@ public boolean uploadToGeoserver(String store, File inFile, String inExt, Boolea
         String fileName = FilenameUtils.getBaseName(inFile.getName());
 
         // check if workspace exists
+        // TODO created never been used
         boolean created = createWorkspace(this.geoserverUrl, GEOSERVER_WORKSPACE);
         boolean published = false;
 
@@ -245,14 +246,16 @@ public static File zipShapefileInDirectory(File inFile, String fileName) {
      * @param fileForamt
      * @return
      */
-    public Boolean uploadToGeoserverWithRenaming(String fileName, String store, File inFile, String fileForamt) {
+    public Boolean uploadToGeoserverWithRenaming(String fileName, String store, File inFile, String fileFormat) {
         Boolean published = false;
         try {
             String restUrl = this.geoserverUrl + "/rest";
             String fileNameNoExt = fileName.split("\\.")[0];
-            int datastoreResponse = createDatastore(restUrl, GEOSERVER_WORKSPACE, store, inFile.getAbsolutePath(), fileForamt);
+            int datastoreResponse = createDatastore(restUrl, GEOSERVER_WORKSPACE, store, inFile.getAbsolutePath(), fileFormat);
+            logger.debug("Successfully created datastore for " + fileNameNoExt + " with response code " + datastoreResponse);
             int layerResponse = createLayer(restUrl, GEOSERVER_WORKSPACE, store, store, fileNameNoExt);
-            if (datastoreResponse == 201 && layerResponse == 201) {
+            logger.debug("Successfully created layer for " + fileNameNoExt + " with response code " + layerResponse);
+            if ((datastoreResponse == 201 || datastoreResponse == 200) && (layerResponse == 201 || layerResponse == 200)){
                 published = true;
             }
         } catch (IOException e) {
@@ -362,12 +365,12 @@ public int createLayer(String baseUrl, String workspaceName, String datastoreNam
      */
     public int sendHttpRequest(String url, Map<String, String> headers, byte[] data,
                                         Map<String, String> params, String method) throws IOException {
-        HttpURLConnection connection = createHttpConnection(url, method, headers);
-
         if (params != null && !params.isEmpty()) {
             url += getQueryString(params);
         }
 
+        HttpURLConnection connection = createHttpConnection(url, method, headers);
+
         try (OutputStream outputStream = connection.getOutputStream()) {
             outputStream.write(data);
         }

diff --git a/...data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java b/...data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java
@@ -14,6 +14,7 @@
 import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
 import edu.illinois.ncsa.incore.common.utils.GeoUtils;
 import edu.illinois.ncsa.incore.service.data.models.Dataset;
+import jakarta.ws.rs.core.Response;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.log4j.Logger;
 import org.geotools.coverage.grid.GridCoverage2D;
@@ -32,9 +33,6 @@
 import org.geotools.feature.FeatureIterator;
 import org.geotools.feature.simple.SimpleFeatureBuilder;
 import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
-import org.geotools.feature.visitor.UniqueVisitor;
-import org.geotools.filter.text.cql2.CQL;
-import org.geotools.filter.text.cql2.CQLException;
 import org.geotools.gce.arcgrid.ArcGridReader;
 import org.geotools.gce.geotiff.GeoTiffFormat;
 import org.geotools.gce.geotiff.GeoTiffWriteParams;
@@ -48,12 +46,10 @@
 import org.opengis.feature.simple.SimpleFeature;
 import org.opengis.feature.simple.SimpleFeatureType;
 import org.opengis.feature.type.AttributeDescriptor;
-import org.opengis.feature.type.GeometryDescriptor;
 import org.opengis.filter.Filter;
 import org.opengis.parameter.GeneralParameterValue;
 import org.opengis.parameter.ParameterValueGroup;
 
-import jakarta.ws.rs.core.Response;
 import java.io.*;
 import java.net.URL;
 import java.nio.file.Files;
@@ -289,20 +285,21 @@ public static File joinTableShapefile(Dataset dataset, List<File> shpfiles, File
         SimpleFeatureType csvFeatureType = createCsvFeatureType(csvFile.getPath(), shapeFileName);
         SimpleFeatureCollection csvFeatures = createCsvFeatureFromCsvType(csvFile.getPath(), csvFeatureType);
 
-        SimpleFeatureCollection joinedFeatures = performInnerJoin(csvFeatures, shapefileSource, UNI_ID_CSV);
+        DefaultFeatureCollection newCollection = (DefaultFeatureCollection) performInnerJoin(csvFeatures, shapefileSource, UNI_ID_CSV, dataset.getId());
 
         // to make an output to shapefile, use this
-        File outShapefile = outToShapefile(joinedFeatures, tempDir, outFileName, shapefileSource);
-        return outShapefile;
+//        File outShapefile = outToShapefile(joinedFeatures, tempDir, outFileName, shapefileSource);
+//        return outShapefile;
 
         // to make an output to file, use this
         // return outToFile(new File(tempDir + File.separator + outFileName), newSft, newCollection);
 
         // to make an output to geopackage, use this
-        //return outToGpkgFile(new File(tempDir + File.separator + dataset.getId() + "." + FileUtils.EXTENSION_GEOPACKAGE), newCollection);
+        return outToGpkgFile(new File(tempDir + File.separator + dataset.getId() + "." + FileUtils.EXTENSION_GEOPACKAGE), newCollection);
     }
 
-    public static File outToShapefile(SimpleFeatureCollection features, String outputDir, String outputFileName, SimpleFeatureSource inputShapefileSource) throws IOException {
+    public static File outToShapefile(SimpleFeatureCollection features, String outputDir, String outputFileName,
+                                      SimpleFeatureSource inputShapefileSource) throws IOException {
         File shapefileOutputFile = new File(outputDir, outputFileName);
         Map<String, Serializable> shapefileParams = Map.of(
             "url", shapefileOutputFile.toURI().toURL(),
@@ -323,7 +320,8 @@ public static File outToShapefile(SimpleFeatureCollection features, String outpu
             // write features to Shapefile
             Transaction transaction = new DefaultTransaction("create");
             String typeName = shapefileDataStore.getTypeNames()[0];
-            FeatureStore<SimpleFeatureType, SimpleFeature> featureStore = (FeatureStore<SimpleFeatureType, SimpleFeature>) shapefileDataStore.getFeatureSource(typeName);
+            FeatureStore<SimpleFeatureType, SimpleFeature> featureStore =
+                (FeatureStore<SimpleFeatureType, SimpleFeature>) shapefileDataStore.getFeatureSource(typeName);
             featureStore.addFeatures(features);
             transaction.commit();
             transaction.close();
@@ -414,16 +412,14 @@ public static SimpleFeatureType createCsvFeatureType(String csvFilePath, String
         }
     }
 
-    public static SimpleFeatureCollection performInnerJoin(
-        SimpleFeatureCollection csvFeatures,
-        SimpleFeatureSource shapefileSource,
-        String commonFieldName) throws IOException {
+    public static SimpleFeatureCollection performInnerJoin(SimpleFeatureCollection csvFeatures, SimpleFeatureSource shapefileSource,
+                                                           String commonFieldName, String datasetId) throws IOException {
 
         // create an index for the CSV features
         Map<Object, SimpleFeature> csvFeatureIndex = indexFeatures(csvFeatures, commonFieldName);
 
         // create joined feature type
-        SimpleFeatureType joinedFeatureType = createJoinedFeatureType(csvFeatures.getSchema(), shapefileSource.getSchema());
+        SimpleFeatureType joinedFeatureType = createJoinedFeatureType(csvFeatures.getSchema(), shapefileSource.getSchema(), datasetId);
 
         // create a DefaultFeatureCollection to store the joined features
         DefaultFeatureCollection joinedFeatures = new DefaultFeatureCollection(null, joinedFeatureType);
@@ -458,36 +454,48 @@ public static Map<Object, SimpleFeature> indexFeatures(SimpleFeatureCollection f
         return featureIndex;
     }
 
-    public static SimpleFeature createJoinedFeature(SimpleFeature csvFeature, SimpleFeature shapefileFeature, SimpleFeatureType joinedFeatureType) {
+    public static SimpleFeature createJoinedFeature(SimpleFeature csvFeature, SimpleFeature shapefileFeature,
+                                                    SimpleFeatureType joinedFeatureType) {
         SimpleFeatureBuilder builder = new SimpleFeatureBuilder(joinedFeatureType);
 
-        // copy all attributes from CSV feature
-        for (int i = 0; i < csvFeature.getAttributeCount(); i++) {
-            builder.add(csvFeature.getAttribute(i));
-        }
-
+        List<String> attributes = new LinkedList<String>();
         // copy all attributes from shapefile feature
         for (int i = 0; i < shapefileFeature.getAttributeCount(); i++) {
+            String localName = shapefileFeature.getFeatureType().getDescriptor(i).getLocalName();
+            attributes.add(localName);
             builder.add(shapefileFeature.getAttribute(i));
         }
 
+        // copy all attributes from CSV feature that are not in the original shapefile
+        for (int i = 0; i < csvFeature.getAttributeCount(); i++) {
+            String localName = csvFeature.getFeatureType().getDescriptor(i).getLocalName();
+            if (!attributes.contains(localName)) {
+                builder.add(csvFeature.getAttribute(i));
+            }
+        }
+
         SimpleFeature joinedFeature = builder.buildFeature(null);
 
         return joinedFeature;
     }
 
-    public static SimpleFeatureType createJoinedFeatureType(SimpleFeatureType csvType, SimpleFeatureType shapefileType) {
+    public static SimpleFeatureType createJoinedFeatureType(SimpleFeatureType csvType, SimpleFeatureType shapefileType, String datasetId) {
         SimpleFeatureTypeBuilder builder = new SimpleFeatureTypeBuilder();
-        builder.setName("JoinedFeatureType");
-
-        addAttributesWithoutDuplicate(builder, csvType);
-        addAttributesWithoutDuplicate(builder, shapefileType, "guid");
-        GeometryDescriptor geometryDescriptor = shapefileType.getGeometryDescriptor();
-        if (geometryDescriptor != null) {
-            builder.setCRS(geometryDescriptor.getCoordinateReferenceSystem());
-            builder.add("geometry", geometryDescriptor.getType().getBinding());
+        builder.init(shapefileType);
+        if (!datasetId.equals("")) {
+            builder.setName(datasetId);
+        } else {
+            builder.setName("JoinedFeatureType");
         }
 
+        addAttributesWithoutDuplicate(builder, csvType, "guid");
+//        addAttributesWithoutDuplicate(builder, shapefileType, "guid");
+//        GeometryDescriptor geometryDescriptor = shapefileType.getGeometryDescriptor();
+//        if (geometryDescriptor != null) {
+//            builder.setCRS(geometryDescriptor.getCoordinateReferenceSystem());
+//            builder.add("geometry", geometryDescriptor.getType().getBinding());
+//        }
+
         SimpleFeatureType builtFeatureType = builder.buildFeatureType();
 
         return builtFeatureType;
@@ -497,7 +505,8 @@ public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builde
         addAttributesWithoutDuplicate(builder, sourceType, null);
     }
 
-    public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builder, SimpleFeatureType sourceType, String excludeAttribute) {
+    public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builder, SimpleFeatureType sourceType,
+                                                     String excludeAttribute) {
         Set<String> addedAttributes = new HashSet<>();
 
         for (int i = 0; i < sourceType.getAttributeCount(); i++) {
@@ -1127,7 +1136,7 @@ public static boolean doesGuidExist(SimpleFeatureCollection inputFeatures) {
      * @return
      * @throws IOException
      */
-    public static GeoUtils.gpkgValidationResult  isGpkgFitToService(File inFile) throws IOException {
+    public static GeoUtils.gpkgValidationResult isGpkgFitToService(File inFile) throws IOException {
         int output = 0;
         try {
             HashMap<String, Object> map = new HashMap<>();
@@ -1149,7 +1158,7 @@ public static GeoUtils.gpkgValidationResult  isGpkgFitToService(File inFile) thr
                 String layerName = layerNames[0];
                 String fileName = inFile.getName().split("\\.")[0];
                 if (!layerName.equals(fileName)) {
-                     return GeoUtils.gpkgValidationResult.NAME_MISMATCH;
+                    return GeoUtils.gpkgValidationResult.NAME_MISMATCH;
                 }
             } else if (layerNames.length == 0) {
                 return GeoUtils.gpkgValidationResult.RASTER_OR_NO_VECTOR_LAYER;