Use MySQL-compatible conversion between CHAR and BINARY in MySQL mode

h2/src/main/org/h2/engine/Mode.java

@@ -186,6 +186,11 @@ public class Mode {
      */
     public boolean allowDB2TimestampFormat;
 
+    /**
+     * Convert (VAR)CHAR to VAR(BINARY) and vice versa with UTF-8 encoding instead of HEX.
+     */
+    public boolean charToBinaryInUtf8;
+
     /**
      * An optional Set of hidden/disallowed column types.
      * Certain DBMSs don't support all column types provided by H2, such as
@@ -269,6 +274,7 @@ public class Mode {
         mode.supportedClientInfoPropertiesRegEx =
                 Pattern.compile(".*");
         mode.prohibitEmptyInPredicate = true;
+        mode.charToBinaryInUtf8 = true;
         add(mode);
 
         mode = new Mode(ModeEnum.Oracle);

h2/src/main/org/h2/jdbc/JdbcResultSet.java

@@ -1065,7 +1065,7 @@ public class JdbcResultSet extends TraceObject implements ResultSet, JdbcResultS
     public byte[] getBytes(int columnIndex) throws SQLException {
         try {
             debugCodeCall("getBytes", columnIndex);
-            return get(columnIndex).getBytes();
+            return get(columnIndex).convertTo(Value.BYTES, -1, conn.getMode()).getBytes();
         } catch (Exception e) {
             throw logAndConvert(e);
         }

h2/src/main/org/h2/value/Value.java

@@ -957,6 +957,33 @@ public abstract class Value {
                 }
                 break;
             }
+            case STRING: {
+                String s;
+                if (getType() == BYTES && mode != null && mode.charToBinaryInUtf8) {
+                    s = new String(getBytesNoCopy());
+                } else {
+                    s = getString();
+                }
+                return ValueString.get(s);
+            }
+            case STRING_IGNORECASE: {
+                String s;
+                if (getType() == BYTES && mode != null && mode.charToBinaryInUtf8) {
+                    s = new String(getBytesNoCopy());
+                } else {
+                    s = getString();
+                }
+                return ValueStringIgnoreCase.get(s);
+            }
+            case STRING_FIXED: {
+                String s;
+                if (getType() == BYTES && mode != null && mode.charToBinaryInUtf8) {
+                    s = new String(getBytesNoCopy());
+                } else {
+                    s = getString();
+                }
+                return ValueStringFixed.get(s, precision, mode);
+            }
             case JAVA_OBJECT: {
                 switch (getType()) {
                 case BYTES:
@@ -1081,17 +1108,11 @@ public abstract class Value {
             case TIMESTAMP_TZ:
                 return ValueTimestampTimeZone.parse(s.trim());
             case BYTES:
-                return ValueBytes.getNoCopy(
-                        StringUtils.convertHexToBytes(s.trim()));
+                return ValueBytes.getNoCopy(mode != null && mode.charToBinaryInUtf8 ?
+                        s.getBytes(StandardCharsets.UTF_8): StringUtils.convertHexToBytes(s.trim()));
             case JAVA_OBJECT:
                 return ValueJavaObject.getNoCopy(null,
                         StringUtils.convertHexToBytes(s.trim()), getDataHandler());
-            case STRING:
-                return ValueString.get(s);
-            case STRING_IGNORECASE:
-                return ValueStringIgnoreCase.get(s);
-            case STRING_FIXED:
-                return ValueStringFixed.get(s, precision, mode);
             case DOUBLE:
                 return ValueDouble.get(Double.parseDouble(s.trim()));
             case FLOAT:

h2/src/test/org/h2/test/db/TestCompatibility.java

@@ -5,6 +5,7 @@
  */
 package org.h2.test.db;
 
+import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
@@ -304,6 +305,28 @@ public class TestCompatibility extends TestBase {
         // check the weird MySQL variant of DELETE
         stat.execute("DELETE TEST FROM TEST WHERE 1=2");
 
+        // Check conversion between VARCHAR and VARBINARY
+        String string = "ABCD\u1234";
+        byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
+        stat.execute("CREATE TABLE TEST2(C VARCHAR, B VARBINARY)");
+        stat.execute("INSERT INTO TEST2(C) VALUES ('" + string + "')");
+        assertEquals(1, stat.executeUpdate("UPDATE TEST2 SET B = C"));
+        ResultSet rs = stat.executeQuery("SELECT B FROM TEST2");
+        assertTrue(rs.next());
+        assertEquals(bytes, rs.getBytes(1));
+        assertEquals(1, stat.executeUpdate("UPDATE TEST2 SET C = B"));
+        testMySQLBytesCheck(stat, string, bytes);
+        PreparedStatement prep = conn.prepareStatement("UPDATE TEST2 SET C = ?");
+        prep.setBytes(1, bytes);
+        assertEquals(1, prep.executeUpdate());
+        testMySQLBytesCheck(stat, string, bytes);
+        stat.execute("DELETE FROM TEST2");
+        prep = conn.prepareStatement("INSERT INTO TEST2(C) VALUES (?)");
+        prep.setBytes(1, bytes);
+        assertEquals(1, prep.executeUpdate());
+        testMySQLBytesCheck(stat, string, bytes);
+        stat.execute("DROP TABLE TEST2");
+
         if (config.memory) {
             return;
         }
@@ -324,7 +347,7 @@ public class TestCompatibility extends TestBase {
         stat = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
                 ResultSet.CONCUR_UPDATABLE);
         assertResult("test", stat, "SHOW TABLES");
-        ResultSet rs = stat.executeQuery("SELECT * FROM TEST");
+        rs = stat.executeQuery("SELECT * FROM TEST");
         rs.next();
         rs.updateString(2, "Hallo");
         rs.updateRow();
@@ -391,6 +414,14 @@ public class TestCompatibility extends TestBase {
         conn = getConnection("compatibility");
     }
 
+    private void testMySQLBytesCheck(Statement stat, String string, byte[] bytes) throws SQLException {
+        ResultSet rs;
+        rs = stat.executeQuery("SELECT C FROM TEST2");
+        assertTrue(rs.next());
+        assertEquals(string, rs.getString(1));
+        assertEquals(bytes, rs.getBytes(1));
+    }
+
     private void testSybaseAndMSSQLServer() throws SQLException {
         Statement stat = conn.createStatement();
         stat.execute("SET MODE MSSQLServer");