From b1446a771de19bef443e27aeb8bd98cfd466be4f Mon Sep 17 00:00:00 2001 From: Dmitry Gusev Date: Tue, 11 Jul 2017 19:25:50 +0300 Subject: [PATCH] Version 3.2.5: With support for `sqlEscapeChar` + joda-time library on classpath --- README.md | 3 + build.gradle | 5 +- .../AbstractInsertUpdateRecordHandler.java | 22 +++-- .../java/com/anjlab/csv2db/Configuration.java | 70 ++++++++++++---- .../anjlab/csv2db/InsertRecordHandler.java | 15 ++-- .../com/anjlab/csv2db/MergeRecordHandler.java | 15 ++-- .../java/com/anjlab/csv2db/ImporterTest.java | 83 ++++++++++--------- .../anjlab/csv2db/ValueDefinitionTest.java | 12 +-- src/test/resources/test-config-with-map.json | 1 + 9 files changed, 135 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index d825bc8..e8ef3c5 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ CompanyName, CompanyNumber,RegAddress.CareOf,RegAddress.POBox,RegAddress.Address "password": "" }, "targetTable": "companies", + "sqlEscapeChar": "\"", "primaryKeys": [ "companies_house_id" ], @@ -204,6 +205,8 @@ The effect is that only the rows that don't already exist in the database will b `targetTable` is the name of target table in database. The table should exist before import. +`sqlEscapeChar` (optional) if specified then all table and column names will be escaped with this character, i.e. with `sqlEscapeChar: "\""` names of columns in generated SQL will be surrounded by quotes. Can be used when name of a column matches one of the SQL reserved words. + `primaryKeys` is the set of primary keys on the table. Only used in `MERGE` and `INSERTONLY` modes. All `primaryKeys` should be present in `columnMappings` section, which means that CSV should contain `primaryKeys` data. diff --git a/build.gradle b/build.gradle index 41f164a..4915abb 100644 --- a/build.gradle +++ b/build.gradle @@ -2,11 +2,9 @@ allprojects { apply plugin: 'eclipse' group = 'com.anjlab' - version = '3.2.4' + version = '3.2.5' } -import org.gradle.plugins.ide.eclipse.model.*; - apply plugin: 'java' apply plugin: 'maven' @@ -33,6 +31,7 @@ dependencies { compile 'org.apache.commons:commons-compress:1.10' runtime 'commons-codec:commons-codec:1.10' + runtime 'joda-time:joda-time:2.9.9' } repositories { diff --git a/src/main/java/com/anjlab/csv2db/AbstractInsertUpdateRecordHandler.java b/src/main/java/com/anjlab/csv2db/AbstractInsertUpdateRecordHandler.java index 6e304c4..767e438 100644 --- a/src/main/java/com/anjlab/csv2db/AbstractInsertUpdateRecordHandler.java +++ b/src/main/java/com/anjlab/csv2db/AbstractInsertUpdateRecordHandler.java @@ -1,5 +1,12 @@ package com.anjlab.csv2db; +import com.codahale.metrics.Timer; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import javax.script.ScriptEngine; +import javax.script.ScriptException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -11,15 +18,6 @@ import java.util.Map.Entry; import java.util.concurrent.Callable; -import javax.script.ScriptEngine; -import javax.script.ScriptException; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; - -import com.codahale.metrics.Timer; - public abstract class AbstractInsertUpdateRecordHandler extends AbstractRecordHandler { private Map selectStatements; @@ -66,9 +64,9 @@ private PreparedStatement getOrCreateSelectStatement(int batchSize) throws SQLEx { StringBuilder selectClause = new StringBuilder("SELECT ") - .append(StringUtils.join(getOrderedTableColumnNames(), ", ")) + .append(StringUtils.join(config.escapeSqlNames(getOrderedTableColumnNames()), ", ")) .append(" FROM ") - .append(config.getTargetTable()) + .append(config.escapeSqlName(config.getTargetTable())) .append(" WHERE ") .append(buildWhereClause()); @@ -105,7 +103,7 @@ protected StringBuilder buildWhereClause() whereClause.append(" AND "); } - whereClause.append(columnName).append(" = ?"); + whereClause.append(config.escapeSqlName(columnName)).append(" = ?"); } } diff --git a/src/main/java/com/anjlab/csv2db/Configuration.java b/src/main/java/com/anjlab/csv2db/Configuration.java index dfc6cb2..673beb6 100644 --- a/src/main/java/com/anjlab/csv2db/Configuration.java +++ b/src/main/java/com/anjlab/csv2db/Configuration.java @@ -1,5 +1,19 @@ package com.anjlab.csv2db; +import au.com.bytecode.opencsv.CSVParser; +import au.com.bytecode.opencsv.CSVReader; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.io.input.AutoCloseInputStream; + +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.ScriptException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -7,28 +21,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Options; -import org.apache.commons.io.input.AutoCloseInputStream; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import au.com.bytecode.opencsv.CSVParser; -import au.com.bytecode.opencsv.CSVReader; - public class Configuration { private static final String MODE = "mode"; @@ -128,6 +126,7 @@ public void setIgnoreLeadingWhiteSpace(boolean ignoreLeadingWhiteSpace) private String connectionUrl; private Map connectionProperties; private String targetTable; + private char sqlEscapeChar; private List primaryKeys; /** * Map keys are zero-based column indices in CSV file. @@ -351,6 +350,16 @@ public void setTargetTable(String targetTable) this.targetTable = targetTable; } + public char getSqlEscapeChar() + { + return sqlEscapeChar; + } + + public void setSqlEscapeChar(char sqlEscapeChar) + { + this.sqlEscapeChar = sqlEscapeChar; + } + public OperationMode getOperationMode() { return operationMode; @@ -558,4 +567,31 @@ public String joinPrimaryKeys(Map nameValues) } return builder.toString(); } + + public String escapeSqlName(String name) + { + if (sqlEscapeChar == 0) + { + return name; + } + + return sqlEscapeChar + name + sqlEscapeChar; + } + + public List escapeSqlNames(List names) + { + if (sqlEscapeChar == 0) + { + return names; + } + + ArrayList list = new ArrayList<>(names.size()); + + for (String name : names) + { + list.add(escapeSqlName(name)); + } + + return list; + } } diff --git a/src/main/java/com/anjlab/csv2db/InsertRecordHandler.java b/src/main/java/com/anjlab/csv2db/InsertRecordHandler.java index b90a7ce..50fd1e5 100644 --- a/src/main/java/com/anjlab/csv2db/InsertRecordHandler.java +++ b/src/main/java/com/anjlab/csv2db/InsertRecordHandler.java @@ -1,5 +1,9 @@ package com.anjlab.csv2db; +import com.codahale.metrics.Timer; + +import javax.script.ScriptEngine; +import javax.script.ScriptException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -7,11 +11,6 @@ import java.util.Map; import java.util.Set; -import javax.script.ScriptEngine; -import javax.script.ScriptException; - -import com.codahale.metrics.Timer; - public class InsertRecordHandler extends AbstractRecordHandler { private PreparedStatement insertStatement; @@ -35,7 +34,7 @@ public InsertRecordHandler( StringBuilder insertClause = new StringBuilder("INSERT INTO ") - .append(config.getTargetTable()) + .append(config.escapeSqlName(config.getTargetTable())) .append(" ("); StringBuilder valuesClause = new StringBuilder(); @@ -47,7 +46,7 @@ public InsertRecordHandler( insertClause.append(", "); valuesClause.append(", "); } - insertClause.append(targetTableColumnName); + insertClause.append(config.escapeSqlName(targetTableColumnName)); ValueDefinition definition = config.getInsertValues().get(targetTableColumnName); @@ -68,7 +67,7 @@ public InsertRecordHandler( insertClause.append(", "); valuesClause.append(", "); } - insertClause.append(targetTableColumnName); + insertClause.append(config.escapeSqlName(targetTableColumnName)); valuesClause.append("?"); } diff --git a/src/main/java/com/anjlab/csv2db/MergeRecordHandler.java b/src/main/java/com/anjlab/csv2db/MergeRecordHandler.java index 24bf5db..143708b 100644 --- a/src/main/java/com/anjlab/csv2db/MergeRecordHandler.java +++ b/src/main/java/com/anjlab/csv2db/MergeRecordHandler.java @@ -1,16 +1,15 @@ package com.anjlab.csv2db; -import static com.anjlab.csv2db.Import.runtimeException; +import com.codahale.metrics.Timer; +import javax.script.ScriptEngine; +import javax.script.ScriptException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -import javax.script.ScriptEngine; -import javax.script.ScriptException; - -import com.codahale.metrics.Timer; +import static com.anjlab.csv2db.Import.runtimeException; public class MergeRecordHandler extends AbstractInsertUpdateRecordHandler { @@ -41,7 +40,7 @@ public MergeRecordHandler( { setClause.append(", "); } - setClause.append(targetTableColumnName).append(" = "); + setClause.append(config.escapeSqlName(targetTableColumnName)).append(" = "); ValueDefinition definition = config.getUpdateValues().get(targetTableColumnName); @@ -61,12 +60,12 @@ public MergeRecordHandler( { setClause.append(", "); } - setClause.append(targetTableColumnName).append(" = ?"); + setClause.append(config.escapeSqlName(targetTableColumnName)).append(" = ?"); } StringBuilder updateClause = new StringBuilder("UPDATE ") - .append(config.getTargetTable()) + .append(config.escapeSqlName(config.getTargetTable())) .append(" SET ") .append(setClause) .append(" WHERE ") diff --git a/src/test/java/com/anjlab/csv2db/ImporterTest.java b/src/test/java/com/anjlab/csv2db/ImporterTest.java index 2ab285e..c40fb76 100644 --- a/src/test/java/com/anjlab/csv2db/ImporterTest.java +++ b/src/test/java/com/anjlab/csv2db/ImporterTest.java @@ -1,5 +1,10 @@ package com.anjlab.csv2db; +import com.anjlab.csv2db.Configuration.OperationMode; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Test; + import java.sql.Connection; import java.sql.Date; import java.sql.ResultSet; @@ -11,12 +16,6 @@ import java.util.Comparator; import java.util.List; -import org.apache.commons.lang3.StringUtils; -import org.junit.Assert; -import org.junit.Test; - -import com.anjlab.csv2db.Configuration.OperationMode; - public class ImporterTest { @Test @@ -31,7 +30,7 @@ public void testImport() throws Exception Connection connection = importer.createConnection(); - dropTableIfExists(connection); + dropTableIfExists(connection, "companies_house_records"); connection.createStatement() .executeUpdate( @@ -118,7 +117,7 @@ public void testMultithreadImport() throws Exception Connection connection = importer.createConnection(); - dropTableIfExists(connection); + dropTableIfExists(connection, "companies_house_records"); connection.createStatement() .executeUpdate( @@ -178,28 +177,38 @@ private List getExpectedDataset(boolean withDate) Date today = withDate ? new Date(cal.getTimeInMillis()) : null; return Arrays.asList( - new Object[]{"! LTD", "08209948", "METROHOUSE 57 PEPPER ROAD", "HUNSLET", today}, - new Object[]{"!BIG IMPACT GRAPHICS LIMITED", "07382019", "335 ROSDEN HOUSE", "372 OLD STREET", today}, - new Object[]{"!NFERNO LTD.", "04753368", "FIRST FLOOR THAVIES INN HOUSE 3-4", "HOLBORN CIRCUS", today}, - new Object[]{"!NSPIRED LTD", "SC421617", "12 BON ACCORD SQUARE", "", today}, - new Object[]{"!OBAC INSTALLATIONS LIMITED", "07527820", "DEVONSHIRE HOUSE", "60 GOSWELL ROAD", today}, - new Object[]{"!OBAC UK LIMITED", "07687209", "DEVONSHIRE HOUSE", "60 GOSWELL ROAD", today}, - new Object[]{"!ST MEDIA SOUTHAMPTON LTD", "07904170", "10 NORTHBROOK HOUSE", "FREE STREET, BISHOPS WALTHAM", today}, - new Object[]{"ALJOH B.V.", "SF000899", "ALEXANDER HINSHELWOOD BARR", "\"SHALIMAR\"", today}, - new Object[]{"ALLEGIS SERVICES (INDIA) PRIVATE LIMITED", "FC027847", "\\54, 1ST MAIN ROAD", "3RD PHASE", today}, - new Object[]{"APS DIRECT LIMITED", "05638208", "MANOR COURT CHAMBERS \\", "126 MANOR COURT ROAD", today}); + new Object[] { "! LTD", "08209948", "METROHOUSE 57 PEPPER ROAD", "HUNSLET", today }, + new Object[] { "!BIG IMPACT GRAPHICS LIMITED", "07382019", "335 ROSDEN HOUSE", "372 OLD STREET", today }, + new Object[] { "!NFERNO LTD.", "04753368", "FIRST FLOOR THAVIES INN HOUSE 3-4", "HOLBORN CIRCUS", today }, + new Object[] { "!NSPIRED LTD", "SC421617", "12 BON ACCORD SQUARE", "", today }, + new Object[] { "!OBAC INSTALLATIONS LIMITED", "07527820", "DEVONSHIRE HOUSE", "60 GOSWELL ROAD", today }, + new Object[] { "!OBAC UK LIMITED", "07687209", "DEVONSHIRE HOUSE", "60 GOSWELL ROAD", today }, + new Object[] { "!ST MEDIA SOUTHAMPTON LTD", "07904170", "10 NORTHBROOK HOUSE", "FREE STREET, BISHOPS WALTHAM", today }, + new Object[] { "ALJOH B.V.", "SF000899", "ALEXANDER HINSHELWOOD BARR", "\"SHALIMAR\"", today }, + new Object[] { "ALLEGIS SERVICES (INDIA) PRIVATE LIMITED", "FC027847", "\\54, 1ST MAIN ROAD", "3RD PHASE", today }, + new Object[] { "APS DIRECT LIMITED", "05638208", "MANOR COURT CHAMBERS \\", "126 MANOR COURT ROAD", today }); } - protected void assertRecordCount(Connection connection, List expectedData, boolean queryWithUpdateDate) throws SQLException + protected void assertRecordCount(Connection connection, List expectedData, boolean queryWithUpdateDate) + throws SQLException + { + assertRecordCount(connection, expectedData, queryWithUpdateDate, ""); + } + + protected void assertRecordCount(Connection connection, List expectedData, boolean queryWithUpdateDate, String sqlEscapeChar) + throws SQLException { String query; if (queryWithUpdateDate) { - query = "SELECT * FROM companies_house_records ORDER BY company_name, updated_at"; + query = "SELECT * FROM " + sqlEscapeChar + "companies_house_records" + sqlEscapeChar + + " ORDER BY " + sqlEscapeChar + "company_name" + sqlEscapeChar + + ", " + sqlEscapeChar + "updated_at" + sqlEscapeChar; } else { - query = "SELECT * FROM companies_house_records ORDER BY company_name"; + query = "SELECT * FROM " + sqlEscapeChar + "companies_house_records" + sqlEscapeChar + + " ORDER BY " + sqlEscapeChar + "company_name" + sqlEscapeChar; } ResultSet resultSet; resultSet = connection.createStatement() @@ -242,7 +251,7 @@ public void testImportWithScripting() throws Exception Connection connection = importer.createConnection(); - dropTableIfExists(connection); + dropTableIfExists(connection, "companies_house_records"); connection.createStatement() .executeUpdate( @@ -260,10 +269,10 @@ public void testImportWithScripting() throws Exception List expectedData = new ArrayList(); for (Object[] row : dataset) { - expectedData.add(new Object[]{ + expectedData.add(new Object[] { row[0].toString().toLowerCase(), row[1].toString(), - StringUtils.reverse(row[1].toString())}); + StringUtils.reverse(row[1].toString()) }); } assertRecordCount(connection, expectedData, false); @@ -282,15 +291,15 @@ public void testImportWithMap() throws Exception Connection connection = importer.createConnection(); - dropTableIfExists(connection); + dropTableIfExists(connection, "\"companies_house_records\""); connection.createStatement() .executeUpdate( - "create table companies_house_records (" + - "id timestamp not null," + - "company_name varchar(160)," + - "company_number varchar(8)," + - "generated_value varchar(8)" + + "create table \"companies_house_records\" (" + + "\"id\" timestamp not null," + + "\"company_name\" varchar(160)," + + "\"company_number\" varchar(8)," + + "\"generated_value\" varchar(8)" + ")"); importer.performImport("src/test/resources/test-data.csv"); @@ -300,28 +309,28 @@ public void testImportWithMap() throws Exception List expectedData = new ArrayList(); for (Object[] row : dataset) { - expectedData.add(new Object[]{ + expectedData.add(new Object[] { row[0].toString().toLowerCase(), row[1].toString(), - StringUtils.reverse(row[1].toString())}); + StringUtils.reverse(row[1].toString()) }); // map function will call emit(nameValues) twice - expectedData.add(new Object[]{ + expectedData.add(new Object[] { row[0].toString().toLowerCase(), row[1].toString(), - StringUtils.reverse(row[1].toString())}); + StringUtils.reverse(row[1].toString()) }); } - assertRecordCount(connection, expectedData, false); + assertRecordCount(connection, expectedData, false, "\""); connection.close(); } - private void dropTableIfExists(Connection connection) + private void dropTableIfExists(Connection connection, final String tableName) { try { connection.createStatement() - .executeUpdate("drop table companies_house_records"); + .executeUpdate("drop table " + tableName); } catch (SQLException e) { diff --git a/src/test/java/com/anjlab/csv2db/ValueDefinitionTest.java b/src/test/java/com/anjlab/csv2db/ValueDefinitionTest.java index 2efe0bf..fd1dab2 100644 --- a/src/test/java/com/anjlab/csv2db/ValueDefinitionTest.java +++ b/src/test/java/com/anjlab/csv2db/ValueDefinitionTest.java @@ -1,11 +1,11 @@ package com.anjlab.csv2db; -import java.util.HashMap; -import java.util.Map; - import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + public class ValueDefinitionTest { @Test @@ -19,19 +19,19 @@ public void toJson() insertValues.put("column", new StringLiteral("constant")); Assert.assertEquals( - "{\"insertValues\":{\"column\":\"constant\"},\"batchSize\":100,\"forceUpdate\":false,\"ignoreNullPK\":false,\"ignoreDuplicatePK\":false}", + "{\"sqlEscapeChar\":\"\\u0000\",\"insertValues\":{\"column\":\"constant\"},\"batchSize\":100,\"forceUpdate\":false,\"ignoreNullPK\":false,\"ignoreDuplicatePK\":false}", configuration.toJson()); insertValues.put("column", new SqlLiteral("clause")); Assert.assertEquals( - "{\"insertValues\":{\"column\":{\"sql\":\"clause\"}},\"batchSize\":100,\"forceUpdate\":false,\"ignoreNullPK\":false,\"ignoreDuplicatePK\":false}", + "{\"sqlEscapeChar\":\"\\u0000\",\"insertValues\":{\"column\":{\"sql\":\"clause\"}},\"batchSize\":100,\"forceUpdate\":false,\"ignoreNullPK\":false,\"ignoreDuplicatePK\":false}", configuration.toJson()); insertValues.put("column", new FunctionReference("name")); Assert.assertEquals( - "{\"insertValues\":{\"column\":{\"function\":\"name\"}},\"batchSize\":100,\"forceUpdate\":false,\"ignoreNullPK\":false,\"ignoreDuplicatePK\":false}", + "{\"sqlEscapeChar\":\"\\u0000\",\"insertValues\":{\"column\":{\"function\":\"name\"}},\"batchSize\":100,\"forceUpdate\":false,\"ignoreNullPK\":false,\"ignoreDuplicatePK\":false}", configuration.toJson()); } } diff --git a/src/test/resources/test-config-with-map.json b/src/test/resources/test-config-with-map.json index 35f375c..2d61210 100644 --- a/src/test/resources/test-config-with-map.json +++ b/src/test/resources/test-config-with-map.json @@ -1,5 +1,6 @@ { "extend": "test-config-with-scripting.json", + "sqlEscapeChar": "\"", "map": { "function": "testMap" }