Implement a simple sql template to support PostgreSQL (#49)

This commit adds the support for PostgreSQL DB
 
Signed-off-by: Paolo Di Tommaso <[email protected]>

build.gradle

@@ -46,6 +46,7 @@ dependencies {
     runtimeOnly 'com.h2database:h2:1.4.200'
     runtimeOnly 'mysql:mysql-connector-java:8.0.28'
     runtimeOnly 'org.xerial:sqlite-jdbc:3.42.0.0'
+    runtimeOnly("org.postgresql:postgresql:42.7.3")
     implementation 'info.picocli:picocli:4.6.3'
     annotationProcessor 'info.picocli:picocli-codegen:4.6.3'
 
@@ -62,6 +63,8 @@ dependencies {
     testImplementation "org.testcontainers:testcontainers:1.20.1"
     testImplementation "org.testcontainers:mysql:1.16.3"
     testImplementation "org.testcontainers:spock:1.16.3"
+    testImplementation 'org.testcontainers:postgresql:1.16.3'
+    testImplementation("org.postgresql:postgresql:42.7.3")
 
     // Dummy library containing some migration files among their resources for testing purposes
     testImplementation files("libs/jar-with-resources.jar")

src/main/java/io/seqera/migtool/App.java

@@ -5,9 +5,8 @@
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
-
-import static io.seqera.migtool.Helper.driverFromUrl;
 import static io.seqera.migtool.Helper.dialectFromUrl;
+import static io.seqera.migtool.Helper.driverFromUrl;
 
 /**
  * Mig tool main launcher

src/main/java/io/seqera/migtool/Helper.java

@@ -216,6 +216,8 @@ static public String driverFromUrl(String url) {
             return "org.h2.Driver";
         if( "sqlite".equals(dialect))
             return "org.sqlite.JDBC";
+        if( "postgresql".equals(dialect))
+            return "org.postgresql.Driver";
         return null;
     }
 }

src/main/java/io/seqera/migtool/MigTool.java

@@ -14,13 +14,21 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Timestamp;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import groovy.lang.Binding;
 import groovy.lang.Closure;
 import groovy.lang.GroovyShell;
 import groovy.sql.Sql;
+import io.seqera.migtool.template.SqlTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,7 +50,7 @@ public class MigTool {
 
     static final String MIGTOOL_TABLE = "MIGTOOL_HISTORY";
 
-    static final String[] DIALECTS = {"h2", "mysql", "mariadb","sqlite"};
+    static final String[] DIALECTS = {"h2", "mysql", "mariadb","sqlite","postgresql"};
 
     String driver;
     String url;
@@ -54,6 +62,7 @@ public class MigTool {
     Pattern pattern;
     String schema;
     String catalog;
+    SqlTemplate template = SqlTemplate.defaultTemplate();
 
     private final List<MigRecord> migrationEntries;
     private final List<MigRecord> patchEntries;
@@ -87,6 +96,7 @@ public MigTool withPassword(String password) {
 
     public MigTool withDialect(String dialect) {
         this.dialect = dialect;
+        this.template = SqlTemplate.from(dialect);
         return this;
     }
 
@@ -350,7 +360,7 @@ protected void apply() {
 
     protected void checkRank(MigRecord entry) {
         try(Connection conn=getConnection(); Statement stm = conn.createStatement()) {
-            ResultSet rs = stm.executeQuery("select max(`rank`) from "+MIGTOOL_TABLE);
+            ResultSet rs = stm.executeQuery(template.selectMaxRank(MIGTOOL_TABLE));
             int last = rs.next() ? rs.getInt(1) : 0;
             int expected = last+1;
             if( entry.rank != expected) {
@@ -434,7 +444,7 @@ private int migrate(MigRecord entry) throws SQLException {
         int delta = (int)(System.currentTimeMillis()-now);
 
         // save the current migration
-        final String insertSql = "insert into "+MIGTOOL_TABLE+" (`rank`,`script`,`checksum`,`created_on`,`execution_time`) values (?,?,?,?,?)";
+        final String insertSql = template.insetMigration(MIGTOOL_TABLE);
         try (Connection conn=getConnection(); PreparedStatement insert = conn.prepareStatement(insertSql)) {
             insert.setInt(1, entry.rank);
             insert.setString(2, entry.script);
@@ -447,7 +457,7 @@ private int migrate(MigRecord entry) throws SQLException {
     }
 
     protected boolean checkMigrated(MigRecord entry) {
-        String sql = "select `id`, `checksum`, `script` from " + MIGTOOL_TABLE + " where `rank` = ? and `script` = ?";
+        final String sql = template.selectMigration(MIGTOOL_TABLE);
 
         try (Connection conn=getConnection(); PreparedStatement stm = conn.prepareStatement(sql)) {
             stm.setInt(1, entry.rank);

src/main/java/io/seqera/migtool/template/DefaultSqlTemplate.java

@@ -0,0 +1,23 @@
+package io.seqera.migtool.template;
+
+/**
+ * Default SQL template for migtool SQL statements
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+class DefaultSqlTemplate extends SqlTemplate {
+    @Override
+    public String selectMaxRank(String table) {
+        return "select max(`rank`) from " + table;
+    }
+
+    @Override
+    public String insetMigration(String table) {
+        return "insert into "+table+" (`rank`,`script`,`checksum`,`created_on`,`execution_time`) values (?,?,?,?,?)";
+    }
+
+    @Override
+    public String selectMigration(String table) {
+        return "select `id`, `checksum`, `script` from "+table+ " where `rank` = ? and `script` = ?";
+    }
+}

src/main/java/io/seqera/migtool/template/PostgreSqlTemplate.java

@@ -0,0 +1,25 @@
+package io.seqera.migtool.template;
+
+/**
+ * PostreSQL dialect implementation
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+class PostgreSqlTemplate extends SqlTemplate {
+
+    @Override
+    public String selectMaxRank(String table) {
+        return "select max(rank) from " + table;
+    }
+
+    @Override
+    public String insetMigration(String table) {
+        return "insert into "+table+" (rank,script,checksum,created_on,execution_time) values (?,?,?,?,?)";
+    }
+
+    @Override
+    public String selectMigration(String table) {
+        return "select id, checksum, script from "+table+ " where rank = ? and script = ?";
+    }
+
+}

src/main/java/io/seqera/migtool/template/SqlTemplate.java

@@ -0,0 +1,28 @@
+package io.seqera.migtool.template;
+
+/**
+ * Implements a simple template pattern to provide specialised
+ * version of required SQL statements depending on the specified SQL "dialect"
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+public abstract class SqlTemplate {
+
+    abstract public String selectMaxRank(String table);
+
+    abstract public String insetMigration(String table);
+
+    abstract public String selectMigration(String table);
+
+    static public SqlTemplate from(String dialect) {
+        if( "postgresql".equals(dialect) )
+            return new PostgreSqlTemplate();
+        else
+            return new DefaultSqlTemplate();
+    }
+
+    public static SqlTemplate defaultTemplate() {
+        return new DefaultSqlTemplate();
+    }
+
+}

src/main/resources/schema/postgresql.sql

@@ -0,0 +1,9 @@
+create table if not exists MIGTOOL_HISTORY
+(
+    id             INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    rank           INTEGER                  NOT NULL,
+    script         VARCHAR(250)             NOT NULL,
+    checksum       VARCHAR(64)              NOT NULL,
+    created_on     timestamp                NOT NULL,
+    execution_time INTEGER
+);

src/test/groovy/io/seqera/migtool/MigToolTest.groovy

@@ -4,8 +4,9 @@
 package io.seqera.migtool
 
 import java.nio.file.Files
-import io.seqera.migtool.resources.ClassFromJarWithResources
 
+import io.seqera.migtool.template.SqlTemplate
+import io.seqera.migtool.resources.ClassFromJarWithResources
 import spock.lang.Specification
 
 class MigToolTest extends Specification {
@@ -657,4 +658,30 @@ class MigToolTest extends Specification {
         thrown(IllegalArgumentException)
     }
 
+    def 'should validate tool parameters' () {
+        when:
+        def tool = new MigTool()
+                .withDriver('org.h2.Driver')
+                .withDialect('h2')
+                .withUrl('jdbc:h2:mem:test15;DB_CLOSE_DELAY=-1')
+                .withUser('sa')
+                .withPassword('')
+                .withLocations("/foo/bar")
+        then:
+        tool.driver == 'org.h2.Driver'
+        tool.dialect == 'h2'
+        tool.url == 'jdbc:h2:mem:test15;DB_CLOSE_DELAY=-1'
+        tool.user == 'sa'
+        tool.password == ''
+        tool.locations == "/foo/bar"
+        tool.template.class == SqlTemplate.defaultTemplate().class
+
+        when:
+        tool = new MigTool()
+                .withDialect('postgresql')
+        then:
+        tool.dialect == 'postgresql'
+        tool.template.class == SqlTemplate.from('postgresql').class
+    }
+
 }

src/test/groovy/io/seqera/migtool/PostgreSqlTest.groovy

@@ -0,0 +1,154 @@
+package io.seqera.migtool
+
+import org.postgresql.util.PSQLException
+import org.testcontainers.containers.PostgreSQLContainer
+import spock.lang.Specification
+/**
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+class PostgreSqlTest extends Specification {
+
+    private static final int PORT = 3306
+
+
+    static PostgreSQLContainer container
+
+    static {
+        container = new PostgreSQLContainer("postgres:16-alpine")
+        // start it -- note: it's stopped automatically
+        // https://www.testcontainers.org/test_framework_integration/manual_lifecycle_control/
+        container.start()
+    }
+
+    def 'should do something'  () {
+        given:
+        def tool = new MigTool()
+                .withDriver('org.postgresql.Driver')
+                .withDialect('postgresql')
+                .withUrl(container.getJdbcUrl())
+                .withUser(container.getUsername())
+                .withPassword(container.getPassword())
+                .withLocations('file:src/test/resources/migrate-db/postgresql')
+
+        when:
+        tool.run()
+
+        then:
+        tool.existTable(tool.getConnection(), 'organization')
+        tool.existTable(tool.getConnection(), 'license')
+        !tool.existTable(tool.getConnection(), 'foo')
+
+    }
+
+    def 'should run a successful Groovy script' () {
+        given:
+        def tool = new MigTool()
+                .withDriver('org.postgresql.Driver')
+                .withDialect('postgresql')
+                .withUrl(container.getJdbcUrl())
+                .withUser(container.getUsername())
+                .withPassword(container.getPassword())
+                .withLocations('file:src/test/resources/migrate-db/postgresql')
+
+        and: 'set up the initial tables'
+        tool.run()
+
+        when: 'run a script that inserts some data'
+        def insertScript = '''
+            import java.sql.Timestamp
+            
+            def now = new Timestamp(System.currentTimeMillis())
+            def newOrgs = [
+                ["1", "C", "C", "[email protected]", now, now],
+                ["2", "C", "C", "[email protected]", now, now],            
+            ]
+        
+            newOrgs.each { o ->
+                sql.executeInsert(
+                   "INSERT INTO organization(id, company, contact, email, date_created, last_updated) VALUES (?, ?, ?, ?, ?, ?)",
+                   o.toArray()
+                )
+            }
+        '''
+        def insertRecord = new MigRecord(rank: 2, script: 'V02__insert-data.groovy', checksum: 'checksum2', statements: [insertScript])
+        tool.runGroovyMigration(insertRecord)
+
+        then: 'the script ran successfully'
+        noExceptionThrown()
+
+        when: 'run another script to check whether the data is present'
+        def checkScript = '''
+            def expectedOrgIds = ["1", "2"]
+            
+            def orgs = sql.rows("SELECT * FROM organization")
+            orgs.each { o ->
+                assert o.id in expectedOrgIds
+            } 
+        '''
+        def checkRecord = new MigRecord(rank: 3, script: 'V03__check-data.groovy', checksum: 'checksum3', statements: [checkScript])
+        tool.runGroovyMigration(checkRecord)
+
+        then: 'the script ran successfully (the new records are present)'
+        noExceptionThrown()
+    }
+
+    def 'should run a failing Groovy script' () {
+        given:
+        def tool = new MigTool()
+                .withDriver('org.postgresql.Driver')
+                .withDialect('postgresql')
+                .withUrl(container.getJdbcUrl())
+                .withUser(container.getUsername())
+                .withPassword(container.getPassword())
+                .withLocations('file:src/test/resources/migrate-db/postgresql')
+
+        and: 'set up the initial tables'
+        tool.run()
+
+        when: 'run a script that inserts some data, but fails at some point'
+        def insertScript = '''
+            import java.sql.Timestamp
+            
+            def now = new Timestamp(System.currentTimeMillis())
+            def newOrgs = [
+                ["3", "C", "C", "[email protected]", now, now],
+                ["4", "C", "C", "[email protected]", now, now],            
+                ["3", "C", "C", "[email protected]", now, now], // Duplicated id: will fail           
+            ]
+        
+            newOrgs.each { o ->
+                sql.executeInsert(
+                   "INSERT INTO organization(id, company, contact, email, date_created, last_updated) VALUES (?, ?, ?, ?, ?, ?)",
+                   o.toArray()
+                )
+            }
+        '''
+        def insertRecord = new MigRecord(rank: 2, script: 'V02__insert-data.groovy', checksum: 'checksum2', statements: [insertScript])
+        tool.runGroovyMigration(insertRecord)
+
+        then: 'an exception is thrown'
+        def e = thrown(IllegalStateException)
+        e.message.startsWith('GROOVY MIGRATION FAILED')
+
+        and: 'the root cause is present and the stack trace contains the expected offending line number'
+        e.cause.class == PSQLException
+        e.cause.stackTrace.any { t -> t.toString() ==~ /.+\.groovy:\d+.+/ }
+
+        when: 'run another script to check whether the data is present'
+        def checkScript = '''
+            def expectedMissingOrgIds = ["3", "4"]
+            
+            def orgs = sql.rows("SELECT * FROM organization")
+            orgs.each { o ->
+                assert o.id !in expectedMissingOrgIds
+            } 
+        '''
+        def checkRecord = new MigRecord(rank: 3, script: 'V03__check-data.groovy', checksum: 'checksum3', statements: [checkScript])
+        tool.runGroovyMigration(checkRecord)
+
+        then: 'the script ran successfully (no records were persisted: the transaction rolled back)'
+        noExceptionThrown()
+    }
+
+}