Server: CLOB data with unicode characters between character code 0xd800 and…

Server: CLOB data with unicode characters between character code 0xd800 and 0xdfff were not transferred correctly. In many cases, the thread was stuck afterwards.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,9 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>Issue 264: the Hibernate dialect in src/tools was removed because it is outdated.
+<ul><li>Server: CLOB data with unicode characters between character code 0xd800 and 0xdfff
+    were not transferred correctly. In many cases, the thread was stuck afterwards.
+</li><li>Issue 264: the Hibernate dialect in src/tools was removed because it is outdated.
 </li></ul>
 
 <h2>Version 1.3.148 Beta (2010-12-12)</h2>

h2/src/docsrc/html/roadmap.html

@@ -397,6 +397,7 @@ See also <a href="build.html#providing_patches">Providing Patches</a>.
 </li><li>Compatibility: Java functions with SQLJ Part1 http://www.acm.org/sigmod/record/issues/9912/standards.pdf.gz
 </li><li>Compatibility: Java functions with SQL/PSM (Persistent Stored Modules) - need to find the documentation.
 </li><li>CACHE_SIZE: automatically use a fraction of Runtime.maxMemory - maybe automatically the second level cache.
+</li><li>Database file name suffix: a way to use no or a different suffix (for example using a slash).
 </li><li>Support date/time/timestamp as documented in http://en.wikipedia.org/wiki/ISO_8601
 </li><li>PostgreSQL compatibility: when in PG mode, treat BYTEA data like PG.
 </li><li>Support =ANY(array) as in PostgreSQL. See also http://www.postgresql.org/docs/8.0/interactive/arrays.html
@@ -441,7 +442,6 @@ See also <a href="build.html#providing_patches">Providing Patches</a>.
 </li><li>Scripting language support (Javascript).
 </li><li>The network client should better detect if the server is not an H2 server and fail early.
 </li><li>H2 Console: support CLOB/BLOB upload.
-</li><li>Database file name suffix: a way to use no or a different suffix (for example using a slash).
 </li><li>Database file lock: detect hibernate / standby / very slow threads (compare system time).
 </li><li>Automatic detection of redundant indexes.
 </li><li>Maybe reject join without "on" (except natural join).

h2/src/main/org/h2/store/Data.java

@@ -8,6 +8,10 @@
  * but stored in reverse order (least significant bits in the first byte).
  */
 package org.h2.store;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Reader;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.sql.Date;
@@ -251,6 +255,25 @@ public class Data {
         pos = p;
     }
 
+    private void writeStringWithoutLength(char[] chars, int len) {
+        int p = pos;
+        byte[] buff = data;
+        for (int i = 0; i < len; i++) {
+            int c = chars[i];
+            if (c < 0x80) {
+                buff[p++] = (byte) c;
+            } else if (c >= 0x800) {
+                buff[p++] = (byte) (0xe0 | (c >> 12));
+                buff[p++] = (byte) (((c >> 6) & 0x3f));
+                buff[p++] = (byte) (c & 0x3f);
+            } else {
+                buff[p++] = (byte) (0xc0 | (c >> 6));
+                buff[p++] = (byte) (c & 0x3f);
+            }
+        }
+        pos = p;
+    }
+
     /**
      * Create a new buffer for the given handler. The
      * handler will decide what type of buffer is created.
@@ -1096,4 +1119,18 @@ public class Data {
         }
     }
 
+    public static void copyString(Reader source, OutputStream target) throws IOException {
+        char[] buff = new char[Constants.IO_BUFFER_SIZE];
+        Data d = new Data(null, new byte[3 * Constants.IO_BUFFER_SIZE]);
+        while (true) {
+            int l = source.read(buff);
+            if (l < 0) {
+                break;
+            }
+            d.writeStringWithoutLength(buff, l);
+            target.write(d.data, 0, d.pos);
+            d.reset();
+        }
+    }
+
 }

h2/src/main/org/h2/store/DataReader.java

@@ -9,13 +9,14 @@ package org.h2.store;
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Reader;
 import org.h2.util.IOUtils;
 
 /**
  * This class is backed by an input stream and supports reading values and
  * variable size data.
  */
-public class DataReader {
+public class DataReader extends Reader {
 
     private static final EOFException EOF = new EOFException();
     private InputStream in;
@@ -34,7 +35,7 @@ public class DataReader {
      *
      * @return the byte
      */
-    public byte read() throws IOException {
+    public byte readByte() throws IOException {
         int x = in.read();
         if (x < 0) {
             throw EOF;
@@ -48,26 +49,26 @@ public class DataReader {
      * @return the value
      */
     public int readVarInt() throws IOException {
-        int b = read();
+        int b = readByte();
         if (b >= 0) {
             return b;
         }
         int x = b & 0x7f;
-        b = read();
+        b = readByte();
         if (b >= 0) {
             return x | (b << 7);
         }
         x |= (b & 0x7f) << 7;
-        b = read();
+        b = readByte();
         if (b >= 0) {
             return x | (b << 14);
         }
         x |= (b & 0x7f) << 14;
-        b = read();
+        b = readByte();
         if (b >= 0) {
             return x | b << 21;
         }
-        return x | ((b & 0x7f) << 21) | (read() << 28);
+        return x | ((b & 0x7f) << 21) | (readByte() << 28);
     }
 
     /**
@@ -76,13 +77,13 @@ public class DataReader {
      * @return the value
      */
     public long readVarLong() throws IOException {
-        long x = read();
+        long x = readByte();
         if (x >= 0) {
             return x;
         }
         x &= 0x7f;
         for (int s = 7;; s += 7) {
-            long b = read();
+            long b = readByte();
             x |= (b & 0x7f) << s;
             if (b >= 0) {
                 return x;
@@ -136,16 +137,42 @@ public class DataReader {
     private String readString(int len) throws IOException {
         char[] chars = new char[len];
         for (int i = 0; i < len; i++) {
-            int x = read() & 0xff;
-            if (x < 0x80) {
-                chars[i] = (char) x;
-            } else if (x >= 0xe0) {
-                chars[i] = (char) (((x & 0xf) << 12) + ((read() & 0x3f) << 6) + (read() & 0x3f));
-            } else {
-                chars[i] = (char) (((x & 0x1f) << 6) + (read() & 0x3f));
-            }
+            chars[i] = readChar();
         }
         return new String(chars);
     }
 
+    /**
+     * Read one character from the input stream.
+     *
+     * @param in the input stream
+     * @return the character
+     */
+    private char readChar() throws IOException {
+        int x = readByte() & 0xff;
+        if (x < 0x80) {
+            return (char) x;
+        } else if (x >= 0xe0) {
+            return (char) (((x & 0xf) << 12) + ((readByte() & 0x3f) << 6) + (readByte() & 0x3f));
+        } else {
+            return (char) (((x & 0x1f) << 6) + (readByte() & 0x3f));
+        }
+    }
+
+    public void close() throws IOException {
+        // ignore
+    }
+
+    public int read(char[] buff, int off, int len) throws IOException {
+        int i = 0;
+        try {
+            for (; i < len; i++) {
+                buff[i] = readChar();
+            }
+            return len;
+        } catch (EOFException e) {
+            return i;
+        }
+    }
+
 }

h2/src/main/org/h2/store/PageLog.java

@@ -267,7 +267,7 @@ public class PageLog {
         try {
             int pos = 0;
             while (true) {
-                int x = in.read();
+                int x = in.readByte();
                 if (x < 0) {
                     break;
                 }

h2/src/main/org/h2/tools/Recover.java

@@ -562,7 +562,7 @@ public class Recover extends Tool implements DataHandler {
         writer.println("---- Transaction log ----------");
         CompressLZF compress = new CompressLZF();
         while (true) {
-            int x = in.read();
+            int x = in.readByte();
             if (x < 0) {
                 break;
             }

h2/src/main/org/h2/util/ExactUTF8InputStreamReader.java

-/*
- * Copyright 2004-2010 H2 Group. Multiple-Licensed under the H2 License,
- * Version 1.0, and under the Eclipse Public License, Version 1.0
- * (http://h2database.com/html/license.html).
- * Initial Developer: H2 Group
- */
-package org.h2.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Reader;
-
-/**
- * The regular InputStreamReader may read some more bytes than required.
- * If this is a problem, use this class.
- */
-public class ExactUTF8InputStreamReader extends Reader {
-
-    private InputStream in;
-
-    public ExactUTF8InputStreamReader(InputStream in) {
-        this.in = in;
-    }
-
-    public void close() {
-        // nothing to do
-    }
-
-    public int read(char[] chars, int off, int len) throws IOException {
-        for (int i = 0; i < len; i++, off++) {
-            int x = in.read();
-            if (x < 0) {
-                return i == 0 ? -1 : i;
-            }
-            x = x & 0xff;
-            if (x < 0x80) {
-                chars[off] = (char) x;
-            } else if (x >= 0xe0) {
-                chars[off] = (char) (((x & 0xf) << 12) + ((in.read() & 0x3f) << 6) + (in.read() & 0x3f));
-            } else {
-                chars[off] = (char) (((x & 0x1f) << 6) + (in.read() & 0x3f));
-            }
-        }
-        return len;
-    }
-
-}

h2/src/main/org/h2/value/Transfer.java

@@ -8,15 +8,10 @@ package org.h2.value;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
-import java.io.BufferedWriter;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
-import java.io.FilterOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.io.Reader;
-import java.io.Writer;
 import java.math.BigDecimal;
 import java.net.InetAddress;
 import java.net.Socket;
@@ -31,9 +26,10 @@ import org.h2.engine.Constants;
 import org.h2.engine.SessionInterface;
 import org.h2.message.DbException;
 import org.h2.message.TraceSystem;
+import org.h2.store.Data;
+import org.h2.store.DataReader;
 import org.h2.tools.SimpleResultSet;
 import org.h2.util.DateTimeUtils;
-import org.h2.util.ExactUTF8InputStreamReader;
 import org.h2.util.IOUtils;
 import org.h2.util.NetUtils;
 import org.h2.util.StringUtils;
@@ -392,20 +388,7 @@ public class Transfer {
             }
             writeLong(length);
             Reader reader = v.getReader();
-            // below, writer.flush needs to be called to ensure the buffer is written
-            // but, this will also flush the output stream, and this slows things down
-            // so construct an output stream that will ignore this chained flush call
-            OutputStream out2 = new FilterOutputStream(out) {
-                public void flush() {
-                    // do nothing
-                }
-            };
-            Writer writer = new BufferedWriter(new OutputStreamWriter(out2, Constants.UTF8));
-            long written = IOUtils.copyAndCloseInput(reader, writer);
-            if (written != length) {
-                throw DbException.get(ErrorCode.CONNECTION_BROKEN_1, "length:" + length + " written:" + written);
-            }
-            writer.flush();
+            Data.copyString(reader, out);
             writeInt(LOB_MAGIC);
             break;
         }
@@ -519,7 +502,7 @@ public class Transfer {
         }
         case Value.CLOB: {
             long length = readLong();
-            Value v = session.getDataHandler().getLobStorage().createClob(new ExactUTF8InputStreamReader(in), length);
+            Value v = session.getDataHandler().getLobStorage().createClob(new DataReader(in), length);
             int magic = readInt();
             if (magic != LOB_MAGIC) {
                 throw DbException.get(ErrorCode.CONNECTION_BROKEN_1, "magic=" + magic);

h2/src/test/org/h2/test/db/TestCases.java

@@ -36,6 +36,7 @@ public class TestCases extends TestBase {
     }
 
     public void test() throws Exception {
+        testUnicode();
         testOuterJoin();
         testCommentOnColumnWithSchemaEqualDatabase();
         testColumnWithConstraintAndComment();
@@ -86,6 +87,27 @@ public class TestCases extends TestBase {
         deleteDb("cases");
     }
 
+    private void testUnicode() throws SQLException {
+        deleteDb("cases");
+        Connection conn = getConnection("cases");
+        Statement stat = conn.createStatement();
+        stat.execute("create table test(id identity, name text)");
+        String[] data = { "\uff1e", "\ud848\udf1e" };
+        PreparedStatement prep = conn.prepareStatement("insert into test(name) values(?)");
+        for (int i = 0; i < data.length; i++) {
+            prep.setString(1, data[i]);
+            prep.execute();
+        }
+        prep = conn.prepareStatement("select * from test order by id");
+        ResultSet rs = prep.executeQuery();
+        for (int i = 0; i < data.length; i++) {
+            assertTrue(rs.next());
+            assertEquals(data[i], rs.getString(2));
+        }
+        stat.execute("drop table test");
+        conn.close();
+    }
+
     private void testCheckConstraintWithFunction() throws SQLException {
         deleteDb("cases");
         Connection conn = getConnection("cases");