Merge pull request #567 from NiklasMehner/master

Charset collation

h2/src/docsrc/help/help.csv

@@ -1180,12 +1180,16 @@ It is also used if the collation name starts with ICU4J_
 (in that case, the ICU4J must be in the classpath, otherwise an exception is thrown).
 The default collator is used if the collation name starts with DEFAULT_
 (even if ICU4J is in the classpath).
+The charset collator is used if the collation name starts with CHARSET_ (e.g. CHARSET_CP500). This collator sorts
+strings according to the binary representation in the given charset.
 
 Admin rights are required to execute this command.
 This command commits an open transaction in this connection.
 This setting is persistent.
+This setting can be appended to the database URL: ""jdbc:h2:test;COLLATION='ENGLISH'""
 ","
 SET COLLATION ENGLISH
+SET COLLATION CHARSET_CP500
 "
 
 "Commands (Other)","SET COMPRESS_LOB","

h2/src/main/org/h2/expression/Comparison.java

@@ -202,7 +202,8 @@ public class Comparison extends Condition {
                     // to constant type, but vise versa, then let's do this here
                     // once.
                     if (constType != resType) {
-                        right = ValueExpression.get(r.convertTo(resType, MathUtils.convertLongToInt(left.getPrecision()), session.getDatabase().getMode()));
+                        right = ValueExpression.get(r.convertTo(resType,
+                                MathUtils.convertLongToInt(left.getPrecision()), session.getDatabase().getMode()));
                     }
                 } else if (right instanceof Parameter) {
                     ((Parameter) right).setColumn(

h2/src/main/org/h2/table/TableSynonym.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
 package org.h2.table;
 
 import org.h2.command.ddl.CreateSynonymData;
@@ -7,8 +12,8 @@ import org.h2.message.Trace;
 import org.h2.schema.SchemaObjectBase;
 
 /**
- * Synonym for an existing table or view. All DML requests are forwarded to the backing table. Adding indices
- * to a synonym or altering the table is not supported.
+ * Synonym for an existing table or view. All DML requests are forwarded to the backing table.
+ * Adding indices to a synonym or altering the table is not supported.
  */
 public class TableSynonym extends SchemaObjectBase {
 

h2/src/main/org/h2/value/CharsetCollator.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.value;
+
+import java.nio.charset.Charset;
+import java.text.CollationKey;
+import java.text.Collator;
+import java.util.Comparator;
+
+/**
+ * The charset collator sorts strings according to the order in the given charset.
+ */
+public class CharsetCollator extends Collator {
+
+    private static final Comparator<byte[]> COMPARATOR = new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] b1, byte[] b2) {
+            int minLength = Math.min(b1.length, b2.length);
+            for (int index = 0; index < minLength; index++) {
+                int result = b1[index] - b2[index];
+                if (result != 0) {
+                    return result;
+                }
+            }
+            return b1.length - b2.length;
+        }
+    };
+    private final Charset charset;
+
+    public CharsetCollator(Charset charset) {
+        this.charset = charset;
+    }
+
+    public Charset getCharset() {
+        return charset;
+    }
+
+    @Override
+    public int compare(String source, String target) {
+        return COMPARATOR.compare(toBytes(source), toBytes(target));
+    }
+
+    private byte[] toBytes(String source) {
+        return source.getBytes(charset);
+    }
+
+    @Override
+    public CollationKey getCollationKey(final String source) {
+        return new CharsetCollationKey(source);
+    }
+
+    @Override
+    public int hashCode() {
+        return 255;
+    }
+
+    private class CharsetCollationKey extends CollationKey {
+
+        CharsetCollationKey(String source) {
+            super(source);
+        }
+
+        @Override
+        public int compareTo(CollationKey target) {
+            return COMPARATOR.compare(toByteArray(), toBytes(target.getSourceString()));
+        }
+
+        @Override
+        public byte[] toByteArray() {
+            return toBytes(getSourceString());
+        }
+
+    }
+}

h2/src/main/org/h2/value/CompareMode.java

@@ -5,6 +5,7 @@
  */
 package org.h2.value;
 
+import java.nio.charset.Charset;
 import java.text.Collator;
 import java.util.Locale;
 
@@ -35,6 +36,12 @@ public class CompareMode {
      */
     public static final String ICU4J = "ICU4J_";
 
+    /**
+     * This constant means the charset specified should be used.
+     * This will fail if the specified charset does not exist.
+     */
+    public static final String CHARSET = "CHARSET_";
+
     /**
      * This constant means that the BINARY columns are sorted as if the bytes
      * were signed.
@@ -210,6 +217,8 @@ public class CompareMode {
             name = name.substring(ICU4J.length());
         } else if (name.startsWith(DEFAULT)) {
             name = name.substring(DEFAULT.length());
+        } else if (name.startsWith(CHARSET)) {
+            return new CharsetCollator(Charset.forName(name.substring(CHARSET.length())));
         }
         if (name.length() == 2) {
             Locale locale = new Locale(StringUtils.toLowerEnglish(name), "");

h2/src/main/org/h2/value/ValueStringFixed.java

@@ -23,8 +23,8 @@ public class ValueStringFixed extends ValueString {
     public static final int PRECISION_DO_NOT_TRIM = Integer.MIN_VALUE;
 
     /**
-     * Special value for the precision in {@link #get(String, int, Mode)} to indicate that the default
-     * behaviour should of trimming the value should apply.
+     * Special value for the precision in {@link #get(String, int, Mode)} to indicate
+     * that the default behaviour should of trimming the value should apply.
      */
     public static final int PRECISION_TRIM = -1;
 
@@ -79,14 +79,16 @@ public class ValueStringFixed extends ValueString {
     /**
      * Get or create a fixed length string value for the given string.
      * <p>
-     * This method will use a {@link Mode}-specific conversion when <code>mode</code> is not <code>null</code>.
-     * Otherwise it will use the default H2 behaviour of trimming the given string if <code>precision</code>
-     * is not {@link #PRECISION_DO_NOT_TRIM}.
+     * This method will use a {@link Mode}-specific conversion when <code>mode</code> is not
+     * <code>null</code>.
+     * Otherwise it will use the default H2 behaviour of trimming the given string if
+     * <code>precision</code> is not {@link #PRECISION_DO_NOT_TRIM}.
      *
      * @param s the string
-     * @param precision if the {@link Mode#padFixedLengthStrings} indicates that strings should be padded, this
-     *        defines the overall length of the (potentially padded) string.
-     *        If the special constant {@link #PRECISION_DO_NOT_TRIM} is used the value will not be trimmed.
+     * @param precision if the {@link Mode#padFixedLengthStrings} indicates that strings should
+     *        be padded, this defines the overall length of the (potentially padded) string.
+     *        If the special constant {@link #PRECISION_DO_NOT_TRIM} is used the value will
+     *        not be trimmed.
      * @return the value
      */
     public static ValueStringFixed get(String s, int precision, Mode mode) {

h2/src/main/org/h2/value/ValueTimestamp.java

@@ -139,7 +139,8 @@ public class ValueTimestamp extends Value {
     }
 
     /**
-     * See: https://stackoverflow.com/questions/3976616/how-to-find-nth-occurrence-of-character-in-a-string#answer-3976656
+     * See:
+     * https://stackoverflow.com/questions/3976616/how-to-find-nth-occurrence-of-character-in-a-string#answer-3976656
      */
     private static int findNthIndexOf(String str, char chr, int n) {
         int pos = str.indexOf(chr);

h2/src/test/org/h2/test/TestAll.java

@@ -67,6 +67,7 @@ import org.h2.test.db.TestScriptSimple;
 import org.h2.test.db.TestSelectCountNonNullColumn;
 import org.h2.test.db.TestSequence;
 import org.h2.test.db.TestSessionsLocks;
+import org.h2.test.db.TestSetCollation;
 import org.h2.test.db.TestShow;
 import org.h2.test.db.TestSpaceReuse;
 import org.h2.test.db.TestSpatial;
@@ -172,6 +173,7 @@ import org.h2.test.unit.TestBitField;
 import org.h2.test.unit.TestBitStream;
 import org.h2.test.unit.TestBnf;
 import org.h2.test.unit.TestCache;
+import org.h2.test.unit.TestCharsetCollator;
 import org.h2.test.unit.TestClearReferences;
 import org.h2.test.unit.TestCollation;
 import org.h2.test.unit.TestCompress;
@@ -419,6 +421,10 @@ java org.h2.test.TestAll timer
      */
     String cacheType;
 
+    /** If not null the database should be opened with the collation parameter */
+    public String collation;
+
+
     /**
      * The AB-BA locking detector.
      */
@@ -431,6 +437,7 @@ java org.h2.test.TestAll timer
 
     private Server server;
 
+
     /**
      * Run all tests.
      *
@@ -791,6 +798,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         addTest(new TestUpdatableResultSet());
         addTest(new TestZloty());
         addTest(new TestCustomDataTypesHandler());
+        addTest(new TestSetCollation());
 
         // jdbcx
         addTest(new TestConnectionPool());
@@ -875,6 +883,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         addTest(new TestBitStream());
         addTest(new TestBnf());
         addTest(new TestCache());
+        addTest(new TestCharsetCollator());
         addTest(new TestClearReferences());
         addTest(new TestCollation());
         addTest(new TestCompress());
@@ -1102,6 +1111,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         appendIf(buff, stopOnError, "stopOnError");
         appendIf(buff, defrag, "defrag");
         appendIf(buff, splitFileSystem, "split");
+        appendIf(buff, collation != null, collation);
         return buff.toString();
     }
 

h2/src/test/org/h2/test/TestBase.java

@@ -335,6 +335,9 @@ public abstract class TestBase {
         if (config.defrag) {
             url = addOption(url, "DEFRAG_ALWAYS", "TRUE");
         }
+        if (config.collation != null) {
+            url = addOption(url, "COLLATION", config.collation);
+        }
         return "jdbc:h2:" + url;
     }
 

h2/src/test/org/h2/test/db/TestCompatibility.java

@@ -517,9 +517,12 @@ public class TestCompatibility extends TestBase {
         stat.execute("drop table test if exists");
         stat.execute("create table test(date TIMESTAMP)");
         stat.executeUpdate("insert into test (date) values ('2014-04-05-09.48.28.020005')");
-        assertResult("2014-04-05 09:48:28.020005", stat, "select date from test"); // <- result is always H2 format timestamp!
-        assertResult("2014-04-05 09:48:28.020005", stat, "select date from test where date = '2014-04-05-09.48.28.020005'");
-        assertResult("2014-04-05 09:48:28.020005", stat, "select date from test where date = '2014-04-05 09:48:28.020005'");
+        assertResult("2014-04-05 09:48:28.020005", stat,
+                "select date from test"); // <- result is always H2 format timestamp!
+        assertResult("2014-04-05 09:48:28.020005", stat,
+                "select date from test where date = '2014-04-05-09.48.28.020005'");
+        assertResult("2014-04-05 09:48:28.020005", stat,
+                "select date from test where date = '2014-04-05 09:48:28.020005'");
     }
 
     private void testDerby() throws SQLException {

h2/src/test/org/h2/test/db/TestQueryCache.java

@@ -72,7 +72,8 @@ public class TestQueryCache extends TestBase {
                 assertEquals(0, c);
                 time = System.nanoTime() - time;
                 if (i == 1000) {
-                    // take from cache and do not close, so that next iteration will have a cache miss
+                    // take from cache and do not close,
+                    // so that next iteration will have a cache miss
                     prep = conn.prepareStatement(query);
                 } else if (i == 1001) {
                     first = time;

h2/src/test/org/h2/test/db/TestSetCollation.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.db;
+
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.h2.jdbc.JdbcSQLException;
+import org.h2.test.TestBase;
+
+public class TestSetCollation extends TestBase {
+    private static final String[] TEST_STRINGS = new String[]{"A", "\u00c4", "AA", "B", "$", "1A", null};
+
+    private static final String DB_NAME = "collator";
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+    @Override
+    public void test() throws Exception {
+        testDefaultCollator();
+        testCp500Collator();
+        testDeCollator();
+        testUrlParameter();
+        testReopenDatabase();
+        testReopenDatabaseWithUrlParameter();
+        testReopenDatabaseWithDifferentCollationInUrl();
+        testReopenDatabaseWithSameCollationInUrl();
+    }
+
+
+    private void testDefaultCollator() throws Exception {
+        assertEquals(Arrays.asList(null, "$", "1A", "A", "AA", "B", "\u00c4"), orderedWithCollator(null));
+    }
+
+    private void testDeCollator() throws Exception {
+        assertEquals(Arrays.asList(null, "$", "1A", "A", "\u00c4", "AA", "B"), orderedWithCollator("DE"));
+        assertEquals(Arrays.asList(null, "$", "1A", "A", "\u00c4", "AA", "B"), orderedWithCollator("DEFAULT_DE"));
+    }
+
+    private void testCp500Collator() throws Exception {
+        // IBM z/OS codepage
+        assertEquals(Arrays.asList(null, "A", "AA", "B", "1A", "$", "\u00c4"),
+                orderedWithCollator("CHARSET_CP500"));
+    }
+
+    private void testUrlParameter() throws Exception {
+        // Specifying the collator in the JDBC Url should have the same effect
+        // as setting it with a set statement
+        config.collation = "CHARSET_CP500";
+        try {
+            assertEquals(Arrays.asList(null, "A", "AA", "B", "1A", "$", "\u00c4"), orderedWithCollator(null));
+        } finally {
+            config.collation = null;
+        }
+    }
+
+    private void testReopenDatabase() throws Exception {
+        if (config.memory) {
+            return;
+        }
+
+        orderedWithCollator("DE");
+
+        try (Connection con = getConnection(DB_NAME)) {
+            insertValues(con, new String[]{"A", "\u00c4"}, 100);
+
+            assertEquals(Arrays.asList(null, "$", "1A", "A", "A", "\u00c4", "\u00c4", "AA", "B"),
+                    loadTableValues(con));
+        }
+    }
+
+    private void testReopenDatabaseWithUrlParameter() throws Exception {
+        if (config.memory) {
+            return;
+        }
+
+        config.collation = "DE";
+        try {
+            orderedWithCollator(null);
+        } finally {
+            config.collation = null;
+        }
+
+        // reopen the database without specifying a collation in the url.
+        // This should keep the initial collation.
+        try (Connection con = getConnection(DB_NAME)) {
+            insertValues(con, new String[]{"A", "\u00c4"}, 100);
+
+            assertEquals(Arrays.asList(null, "$", "1A", "A", "A", "\u00c4", "\u00c4", "AA", "B"),
+                    loadTableValues(con));
+        }
+
+    }
+
+    private void testReopenDatabaseWithDifferentCollationInUrl() throws Exception {
+        if (config.memory) {
+            return;
+        }
+        config.collation = "DE";
+        try {
+            orderedWithCollator(null);
+        } finally {
+            config.collation = null;
+        }
+
+        config.collation = "CHARSET_CP500";
+        try {
+            getConnection(DB_NAME);
+            fail();
+        } catch (JdbcSQLException e) {
+            // expected
+        } finally {
+            config.collation = null;
+        }
+    }
+
+    private void testReopenDatabaseWithSameCollationInUrl() throws Exception {
+        if (config.memory) {
+            return;
+        }
+        config.collation = "DE";
+        try {
+            orderedWithCollator(null);
+        } finally {
+            config.collation = null;
+        }
+
+        config.collation = "DE";
+        try (Connection con = getConnection(DB_NAME)) {
+            insertValues(con, new String[]{"A", "\u00c4"}, 100);
+
+            assertEquals(Arrays.asList(null, "$", "1A", "A", "A", "\u00c4", "\u00c4", "AA", "B"),
+                    loadTableValues(con));
+        } finally {
+            config.collation = null;
+        }
+    }
+
+
+    private List<String> orderedWithCollator(String collator) throws SQLException {
+        deleteDb(DB_NAME);
+        try (Connection con = getConnection(DB_NAME); Statement statement = con.createStatement()) {
+            ;
+            if (collator != null) {
+                statement.execute("SET COLLATION " + collator);
+            }
+            statement.execute("CREATE TABLE charsettable(id INT PRIMARY KEY, testvalue VARCHAR(50))");
+
+            insertValues(con, TEST_STRINGS, 1);
+
+            return loadTableValues(con);
+        }
+    }
+
+    private void insertValues(Connection con, String[] values, int startId) throws SQLException {
+        PreparedStatement ps = con.prepareStatement("INSERT INTO charsettable VALUES (?, ?)");
+        int id = startId;
+        for (String value : values) {
+            ps.setInt(1, id++);
+            ps.setString(2, value);
+            ps.execute();
+        }
+        ps.close();
+    }
+
+    private List<String> loadTableValues(Connection con) throws SQLException {
+        List<String> results = new ArrayList<>();
+        Statement statement = con.createStatement();
+        ResultSet resultSet = statement.executeQuery("select testvalue from charsettable order by testvalue");
+        while (resultSet.next()) {
+            results.add(resultSet.getString(1));
+        }
+        statement.close();
+        return results;
+    }
+
+}

h2/src/test/org/h2/test/db/TestSynonymForTable.java

@@ -85,7 +85,8 @@ public class TestSynonymForTable extends TestBase {
         assertThrows(JdbcSQLException.class, stat).execute("SELECT id FROM testsynonym");
 
         // Synonym should be dropped as well
-        ResultSet synonyms = conn.createStatement().executeQuery("SELECT * FROM INFORMATION_SCHEMA.SYNONYMS WHERE SYNONYM_NAME='TESTSYNONYM'");
+        ResultSet synonyms = conn.createStatement().executeQuery(
+                "SELECT * FROM INFORMATION_SCHEMA.SYNONYMS WHERE SYNONYM_NAME='TESTSYNONYM'");
         assertFalse(synonyms.next());
         conn.close();
 

h2/src/test/org/h2/test/jdbc/TestStatement.java

@@ -403,7 +403,8 @@ public class TestStatement extends TestBase {
         stat.execute("create table test(id int primary key, name varchar(255))");
         stat.execute("insert into test values(1, 'Hello')");
         stat.execute("insert into test values(2, 'World')");
-        PreparedStatement ps = conn.prepareStatement("select name from test where id in (select id from test where name REGEXP ?)");
+        PreparedStatement ps = conn.prepareStatement(
+                "select name from test where id in (select id from test where name REGEXP ?)");
         ps.setString(1, "Hello");
         ResultSet rs = ps.executeQuery();
         assertTrue(rs.next());

h2/src/test/org/h2/test/unit/TestCharsetCollator.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.unit;
+
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.text.Collator;
+import org.h2.test.TestBase;
+import org.h2.value.CharsetCollator;
+import org.h2.value.CompareMode;
+
+/**
+ * Unittest for org.h2.value.CharsetCollator
+ */
+public class TestCharsetCollator extends TestBase {
+    private CharsetCollator cp500Collator = new CharsetCollator(Charset.forName("cp500"));
+    private CharsetCollator utf8Collator = new CharsetCollator(Charset.forName("UTF-8"));
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+
+    @Override
+    public void test() throws Exception {
+        testBasicComparison();
+        testNumberToCharacterComparison();
+        testLengthComparison();
+        testCreationFromCompareMode();
+        testCreationFromCompareModeWithInvalidCharset();
+    }
+
+    private void testCreationFromCompareModeWithInvalidCharset() {
+        try {
+            CompareMode.getCollator("CHARSET_INVALID");
+            fail();
+        } catch (UnsupportedCharsetException e) {
+            // expected
+        }
+    }
+
+    private void testCreationFromCompareMode() {
+        Collator utf8Col = CompareMode.getCollator("CHARSET_UTF-8");
+        assertTrue(utf8Col instanceof CharsetCollator);
+        assertEquals(((CharsetCollator) utf8Col).getCharset(), Charset.forName("UTF-8"));
+    }
+
+    private void testBasicComparison() {
+        assertTrue(cp500Collator.compare("A", "B") < 0);
+        assertTrue(cp500Collator.compare("AA", "AB") < 0);
+    }
+
+    private void testLengthComparison() {
+        assertTrue(utf8Collator.compare("AA", "A") > 0);
+    }
+
+    private void testNumberToCharacterComparison() {
+        assertTrue(cp500Collator.compare("A", "1") < 0);
+        assertTrue(utf8Collator.compare("A", "1") > 0);
+    }
+}

h2/src/tools/org/h2/build/doc/dictionary.txt

@@ -737,3 +737,7 @@ choosing optimise arte preparator katzyn bla jenkins tot artes pgserver npe
 suffers closeablem mni significance vise identiy vitalus aka ilike uppercasing reentrant
 aff ignite warm upstream producing sfu jit smtm affinity stashed tbl
 stumc numbered
+
+reopening cloudera hive clustername whomooz unittest anymore snowflakecomputing unpadded endpoint redshift backingtable
+trimming hadoop azure resolves snowflake testsynonym plays charsettable synonyms nonexisting impala codepage recognize
+dbm forwarded amazon stmnt excessive testvalue
\ No newline at end of file