support postgis

adapter/adapter-postgis/README.md

@@ -0,0 +1,6 @@
+# Postgis适配说明
+Point类在com.hufudb.openhufu.data.storage.Point中定义
+
+在proto传输过程中，Point类型以字符串类型Point(x y)形式传递
+
+在生成SQL语句时，需要将Point类型翻译为ST_GeomFromText('Point(x y)', 4326)，从而表示为geometry类型对象

adapter/adapter-postgis/pom.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>openhufu-adapter</artifactId>
+    <groupId>com.hufudb.openhufu</groupId>
+    <version>1.0.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>openhufu-adapter-postgis</artifactId>
+
+  <properties>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.hufudb.openhufu</groupId>
+      <artifactId>openhufu-owner</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>net.postgis</groupId>
+      <artifactId>postgis-jdbc</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>report-aggregate</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>report-aggregate</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapter.java

@@ -0,0 +1,32 @@
+package com.hufudb.openhufu.owner.adapter.postgis;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.sql.SQLException;
+import java.sql.ResultSet;
+import com.hufudb.openhufu.owner.adapter.AdapterTypeConverter;
+import com.hufudb.openhufu.owner.adapter.jdbc.JDBCAdapter;
+import com.hufudb.openhufu.data.storage.DataSet;
+import com.hufudb.openhufu.data.storage.EmptyDataSet;
+import com.hufudb.openhufu.expression.Translator;
+import com.hufudb.openhufu.data.schema.Schema;
+
+
+public class PostgisAdapter extends JDBCAdapter {
+  PostgisAdapter(String catalog, Connection connection, Statement statement,
+      AdapterTypeConverter converter, Translator translator) {
+    super(catalog, connection, statement, converter, translator);
+  }
+
+  @Override
+  protected DataSet executeSQL(String sql, Schema schema) {
+    try {
+      ResultSet rs = statement.executeQuery(sql);
+      LOG.info("Execute {}", sql);
+      return new PostgisResultDataSet(schema, rs);
+    } catch (SQLException e) {
+      LOG.error("Fail to execute SQL [{}]: {}", sql, e.getMessage());
+      return EmptyDataSet.INSTANCE;
+    }
+  }
+}

adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisAdapterFactory.java

@@ -0,0 +1,43 @@
+package com.hufudb.openhufu.owner.adapter.postgis;
+
+import com.hufudb.openhufu.common.enums.DataSourceType;
+import com.hufudb.openhufu.owner.adapter.AdapterConfig;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import com.hufudb.openhufu.owner.adapter.AdapterFactory;
+import com.hufudb.openhufu.expression.BasicTranslator;
+import com.hufudb.openhufu.owner.adapter.Adapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PostgisAdapterFactory implements AdapterFactory {
+
+  static final Logger LOG = LoggerFactory.getLogger(PostgisAdapterFactory.class);
+
+  public PostgisAdapterFactory() {
+    try {
+      Class.forName("org.postgresql.Driver");
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Override
+  public Adapter create(AdapterConfig config) {
+    try {
+      Connection connection = DriverManager.getConnection(config.url, config.user, config.passwd);
+      Statement statement = connection.createStatement();
+      return new PostgisAdapter(config.catalog, connection, statement, new PostgisTypeConverter(),
+          new BasicTranslator(getType().getType()));
+    } catch (Exception e) {
+      LOG.error("Fail to create csv adapter: {}", config.url, e);
+      return null;
+    }
+  }
+
+  @Override
+  public DataSourceType getType() {
+    return DataSourceType.POSTGIS;
+  }
+}

adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisResultDataSet.java

@@ -0,0 +1,91 @@
+package com.hufudb.openhufu.owner.adapter.postgis;
+
+import java.sql.ResultSet;
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.hufudb.openhufu.data.schema.Schema;
+import com.hufudb.openhufu.data.storage.ResultDataSet;
+import com.hufudb.openhufu.proto.OpenHuFuData.ColumnDesc;
+import org.postgresql.util.PGobject;
+
+/**
+ * PostGIS extension of ResultDataSet
+ */
+public class PostgisResultDataSet extends ResultDataSet {
+
+  public PostgisResultDataSet(Schema schema, ResultSet result) {
+    super(schema, result);
+  }
+
+  @Override
+  protected List<Getter> generateGetters() {
+    ImmutableList.Builder<Getter> builder = ImmutableList.builder();
+    int i = 1;
+    for (ColumnDesc col : schema.getColumnDescs()) {
+      final int idx = i;
+      switch (col.getType()) {
+        case BLOB:
+          builder.add(() -> {
+            return result.getBytes(idx);
+          });
+          break;
+        case BOOLEAN:
+          builder.add(() -> {
+            return result.getBoolean(idx);
+          });
+          break;
+        case BYTE:
+        case SHORT:
+        case INT:
+          builder.add(() -> {
+            return result.getInt(idx);
+          });
+          break;
+        case DATE:
+          builder.add(() -> {
+            return result.getDate(idx);
+          });
+          break;
+        case TIME:
+          builder.add(() -> {
+            return result.getTime(idx);
+          });
+          break;
+        case TIMESTAMP:
+          builder.add(() -> {
+            return result.getTimestamp(idx);
+          });
+          break;
+        case LONG:
+          builder.add(() -> {
+            return result.getLong(idx);
+          });
+          break;
+        case STRING:
+          builder.add(() -> {
+            return result.getString(idx);
+          });
+          break;
+        case DOUBLE:
+          builder.add(() -> {
+            return result.getDouble(idx);
+          });
+          break;
+        case FLOAT:
+          builder.add(() -> {
+            return result.getFloat(idx);
+          });
+          break;
+        case GEOMETRY:
+          builder.add(() -> {
+            return PostgisUtils.fromPGPoint(((PGobject) (result.getObject(idx))));
+          });
+          break;
+        default:
+          break;
+      }
+      ++i;
+    }
+    return builder.build();
+  }
+}

adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisTypeConverter.java

@@ -0,0 +1,48 @@
+package com.hufudb.openhufu.owner.adapter.postgis;
+
+import com.hufudb.openhufu.owner.adapter.AdapterTypeConverter;
+import com.hufudb.openhufu.proto.OpenHuFuData.ColumnType;
+
+public class PostgisTypeConverter extends AdapterTypeConverter {
+  @Override
+  public ColumnType convert(int type, String typeName) {
+    switch (typeName) {
+      case "real":
+      case "float4":
+      return ColumnType.FLOAT;
+      case "float8":
+      case "double precision":
+      case "numeric":
+      return ColumnType.DOUBLE;
+      case "TINYINT":
+      return ColumnType.BYTE;
+      case "SMALLINT":
+      return ColumnType.SHORT;
+      case "int2":
+      case "int4":
+      return ColumnType.INT;
+      case "oid":
+      case "int8":
+      return ColumnType.LONG;
+      case "varchar":
+      case "char":
+      case "bpchar":
+      case "text":
+      case "name":
+      return ColumnType.STRING;
+      case "bit":
+      case "bool":
+      return ColumnType.BOOLEAN;
+      case "date":
+      return ColumnType.DATE;
+      case "time":
+      return ColumnType.TIME;
+      case "timestamp":
+      return ColumnType.TIMESTAMP;
+      case "geometry":
+      return ColumnType.GEOMETRY;
+      default:
+      return ColumnType.STRING;
+    }
+  }
+}

adapter/adapter-postgis/src/main/java/com/hufudb/openhufu/owner/adapter/postgis/PostgisUtils.java

@@ -0,0 +1,23 @@
+package com.hufudb.openhufu.owner.adapter.postgis;
+
+import java.sql.SQLException;
+import org.postgresql.util.PGobject;
+import org.postgis.GeometryBuilder;
+import com.hufudb.openhufu.data.storage.Point;
+
+public class PostgisUtils {
+  private PostgisUtils() {}
+
+  /**
+   * convert PostGIS Point to Pair in java
+   */
+  public static Point fromPGPoint(PGobject pgpoint) {
+    try {
+      org.postgis.Point p = GeometryBuilder.geomFromString(pgpoint.getValue()).getPoint(0);
+      return new Point(p.x, p.y);
+    } catch (SQLException e) {
+      e.printStackTrace();
+      return null;
+    }
+  }
+}

adapter/adapter-postgis/src/main/resources/META-INF/services/com.hufudb.openhufu.owner.adapter.AdapterFactory

@@ -0,0 +1 @@
+com.hufudb.openhufu.owner.adapter.postgis.PostgisAdapterFactory

adapter/pom.xml

@@ -15,6 +15,7 @@
 
   <modules>
     <module>adapter-csv</module>
+    <module>adapter-postgis</module>
   </modules>
 
   <dependencies>

benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialBenchmarkTest.java → benchmark/src/test/java/com/hufudb/openhufu/benchmark/OpenHuFuSpatialCSVTest.java

@@ -31,20 +31,20 @@
 
 import static org.junit.Assert.assertEquals;
 
-public class OpenHuFuSpatialBenchmarkTest {
+public class OpenHuFuSpatialCSVTest {
   private static final Logger LOG = LoggerFactory.getLogger(OpenHuFuBenchmark.class);
   private static final OpenHuFuUser user = new OpenHuFuUser();
 
   @BeforeClass
   public static void setUp() throws IOException {
     LinkedTreeMap userConfigs = new Gson().fromJson(Files.newBufferedReader(
-                    Path.of(OpenHuFuBenchmark.class.getClassLoader().getResource("spatial-user-configs.json")
+                    Path.of(OpenHuFuBenchmark.class.getClassLoader().getResource("spatial-csv-configs.json")
                             .getPath())),
             LinkedTreeMap.class);
     List<String> endpoints = (List<String>) userConfigs.get("owners");
     List<GlobalTableConfig> globalTableConfigs = new Gson().fromJson(new Gson().toJson(userConfigs.get("tables")),
             new TypeToken<ArrayList<GlobalTableConfig>>() {}.getType());
-    LOG.info("Init benchmark of OpenHuFuSpatial...");
+    LOG.info("Init benchmark of OpenHuFuSpatialCSV...");
     for (String endpoint : endpoints) {
       user.addOwner(endpoint, null);
     }