Merge pull request #276 from IN-CORE/release-1.25.0

Release 1.25.0

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.25.0] - 2024-02-21
+
+### Changed
+- Join between CSV table and shapefile has been updated based on Geotools update [#272](https://github.com/IN-CORE/incore-services/issues/272)
+
+### Added 
+- Add retrofit information to the mapping set class [#252](https://github.com/IN-CORE/incore-services/issues/252)
+
+### Fixed
+- Fix broken Semantics Type embedded in data viewer [#273](https://github.com/IN-CORE/incore-services/issues/273)
 
 ## [1.24.0] - 2024-02-07
 

server/build.gradle

@@ -48,7 +48,7 @@ apply plugin: 'war'
 // This will not apply to the war files we build unless we put it within project loop
 // at the end of this file. This is only for the docker build to pull a version
 war {
-    archiveVersion = '1.24.0'
+    archiveVersion = '1.25.0'
 }
 
 apply plugin: 'org.gretty'

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java

@@ -71,7 +71,7 @@
 @OpenAPIDefinition(
     info = @Info(
         description = "IN-CORE Data Service for creating and accessing datasets",
-        version = "1.24.0",
+        version = "1.25.0",
         title = "IN-CORE v2 Data Service API",
         contact = @Contact(
             name = "IN-CORE Dev Team",
@@ -935,14 +935,15 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
 
         if (enableGeoserver && isGeoserver) {
             if (isJoin) {
+                File joinedShapefile = null;
                 // todo: the join process for the network dataset should be added in here.
                 try {
-                    geoPkgFile = FileUtils.joinShpTable(dataset, repository, true);
-                    if (!GeoserverUtils.uploadGpkgToGeoserver(dataset.getId(), geoPkgFile)) {
+                    joinedShapefile = FileUtils.joinShpTable(dataset, repository, true);
+                    if (!GeoserverUtils.uploadShapefileToGeoserver(dataset.getId(), joinedShapefile)) {
                         logger.error("Fail to upload geopackage file");
                         throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, "Fail to upload geopakcage file.");
                     }
-                    // GeoserverUtils.uploadShpZipToGeoserver(dataset.getId(), zipFile);
+//                     GeoserverUtils.uploadShpZipToGeoserver(dataset.getId(), zipFile);
                 } catch (IOException e) {
                     logger.error("Error making temp directory in joining process ", e);
                     throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, "Error making temp directory in joining process.");
@@ -951,7 +952,7 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
                     throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
                 }
                 // clean up
-                FileUtils.deleteTmpDir(geoPkgFile);
+                FileUtils.deleteTmpDir(joinedShapefile);
             } else {
                 try {
                     if (format.equalsIgnoreCase(FileUtils.FORMAT_NETWORK)) {

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/FileUtils.java

@@ -697,7 +697,7 @@ public static File loadZipdataFromRepository(String inId) throws IOException {
     public static File joinShpTable(Dataset dataset, IRepository repository, boolean isRename) throws IncoreHTTPException, IOException {
         List<FileDescriptor> csvFDs = dataset.getFileDescriptors();
         File csvFile = null;
-        File geoPkgFile = null;
+        File shapeFile = null;
 
         for (int i = 0; i < csvFDs.size(); i++) {
             FileDescriptor csvFd = csvFDs.get(i);
@@ -740,9 +740,9 @@ public static File joinShpTable(Dataset dataset, IRepository repository, boolean
 
         }
 
-        geoPkgFile = GeotoolsUtils.joinTableShapefile(dataset, shpfiles, csvFile, isRename);
+        shapeFile = GeotoolsUtils.joinTableShapefile(dataset, shpfiles, csvFile, isRename);
 
-        return geoPkgFile;
+        return shapeFile;
     }
 
     /**

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeoserverRestApi.java

@@ -12,18 +12,16 @@
 import org.apache.commons.io.FilenameUtils;
 import org.apache.log4j.Logger;
 
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 public class GeoserverRestApi {
     public static final String DEFAULT_CRS = "EPSG:4326";
@@ -111,7 +109,7 @@ public boolean postFileToGeoserver(String apiUrl, File inData, String contType)
         try {
             HttpURLConnection connection = createHttpConnection(apiUrl, contType);
 
-            // Write Shapefile data to request body
+            // write shapefile data to request body
             try (DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
                  FileInputStream fis = new FileInputStream(inData)) {
                 byte[] buffer = new byte[1024];
@@ -174,7 +172,15 @@ public boolean uploadToGeoserver(String store, File inFile, String inExt, Boolea
                 published =  this.uploadToGeoserverWithRenaming(fileName, store, inFile, "shapefile");
             } else {
                 String restUrl = this.geoserverUrl + "/rest/workspaces/" + GEOSERVER_WORKSPACE + "/datastores/" + store + "/file.shp";
-                published = this.postFileToGeoserver(restUrl, inFile, "zip");
+                // check if the file extension is shapefile or zip file
+                String ext = FilenameUtils.getExtension(inFile.getName());
+                if (ext.equalsIgnoreCase("shp")) {
+                    File zipFile = zipShapefileInDirectory(inFile, fileName);
+                    published = this.postFileToGeoserver(restUrl, zipFile, "zip");
+                } else {
+                    // if the file is already a zip file, just use it
+                    published = this.postFileToGeoserver(restUrl, inFile, "zip");
+                }
             }
         } else if (inExt.equalsIgnoreCase("gpkg")) {
             if (renameLayer) {
@@ -193,6 +199,42 @@ public boolean uploadToGeoserver(String store, File inFile, String inExt, Boolea
         return published;
     }
 
+    /**
+     * zip shapefile in directory
+     *
+     * @param inFile
+     * @param fileName
+     * @return
+     */
+    public static File zipShapefileInDirectory(File inFile, String fileName) {
+        // create a zip file with the shapefile
+        File zipFile = new File(inFile.getParent(), fileName + ".zip");
+        try (FileOutputStream fos = new FileOutputStream(zipFile);
+             ZipOutputStream zos = new ZipOutputStream(fos)) {
+            File[] files = inFile.getParentFile().listFiles((dir, name) -> name.startsWith(fileName));
+            for (File file : files) {
+                // zip only shapefile related files that are .shp, .shx, .dbf, .prj
+                if (file.getName().endsWith(".shp") || file.getName().endsWith(".shx") || file.getName().endsWith(".dbf") || file.getName().endsWith(".prj")) {
+                    try (FileInputStream fis = new FileInputStream(file)) {
+                        ZipEntry zipEntry = new ZipEntry(file.getName());
+                        zos.putNextEntry(zipEntry);
+                        byte[] bytes = new byte[1024];
+                        int length;
+                        while ((length = fis.read(bytes)) >= 0) {
+                            zos.write(bytes, 0, length);
+                        }
+                        zos.closeEntry();
+                    }
+                }
+            }
+        } catch (FileNotFoundException e) {
+            throw new RuntimeException(e);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return zipFile;
+    }
+
     /**
      * upload file to geoserver with renaming
      * this method is for the case when the file name is different from published layer name

server/data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeoserverUtils.java

@@ -81,6 +81,12 @@ public static boolean datasetUploadToGeoserver(Dataset dataset, IRepository repo
         return published;
     }
 
+    public static boolean uploadShapefileToGeoserver(String datasetId, File file) {
+        GeoserverRestApi gsApi = GeoserverRestApi.createGeoserverApi();
+        boolean published = gsApi.uploadToGeoserver(datasetId, file, "shp", false);
+        return published;
+    }
+
     public static boolean networkDatasetUploadToGeoserver(Dataset dataset, IRepository repository) throws IOException, URISyntaxException {
         String datasetId = dataset.getId();
         boolean link_published = false;