Build target for the MVStore; remove DataType.getMaxLength; reduce cross-dependencies

h2/src/docsrc/html/mvstore.html

@@ -412,6 +412,14 @@ The MVStore is included in the latest H2 jar file.
 </p><p>
 There are no special requirements to use it.
 The MVStore should run on any JVM as well as on Android.
+</p><p>
+To build just the MVStore (without the database engine), run:
+</p>
+<pre>
+./build.sh jarMVStore
+</pre>
+<p>
+This will create the file <code>bin/h2mvstore-${version}.jar</code> (about 130 KB).
 </p>
 
 <!-- [close] { --></div></td></tr></table><!-- } --><!-- analytics --></body></html>

h2/src/main/org/h2/compress/LZFInputStream.java

@@ -9,7 +9,7 @@ package org.h2.compress;
 import java.io.IOException;
 import java.io.InputStream;
 import org.h2.message.DbException;
-import org.h2.util.Utils;
+import org.h2.mvstore.DataUtils;
 
 /**
  * An input stream to read from an LZF stream.
@@ -32,7 +32,7 @@ public class LZFInputStream extends InputStream {
     }
 
     private static byte[] ensureSize(byte[] buff, int len) {
-        return buff == null || buff.length < len ? Utils.newBytes(len) : buff;
+        return buff == null || buff.length < len ? DataUtils.newBytes(len) : buff;
     }
 
     private void fillBuffer() throws IOException {

h2/src/main/org/h2/expression/ConditionInConstantSet.java

@@ -44,7 +44,7 @@ public class ConditionInConstantSet extends Condition {
         this.valueList = valueList;
         this.valueSet = new HashSet<Value>(valueList.size());
         for (Expression expression : valueList) {
-            this.valueSet.add(expression.getValue(session));
+            valueSet.add(expression.getValue(session));
         }
     }
 
@@ -53,6 +53,7 @@ public class ConditionInConstantSet extends Condition {
         if (leftVal == ValueNull.INSTANCE) {
             return leftVal;
         }
+        int todoFix;
         Value firstRightValue = null;
         for (Value v : valueSet) {
             if (v != ValueNull.INSTANCE) {

h2/src/main/org/h2/expression/Function.java

@@ -30,6 +30,7 @@ import org.h2.engine.Database;
 import org.h2.engine.Mode;
 import org.h2.engine.Session;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
 import org.h2.schema.Schema;
 import org.h2.schema.Sequence;
 import org.h2.security.BlockCipher;
@@ -1334,7 +1335,7 @@ public class Function extends Expression implements FunctionCall {
 
     private static byte[] getPaddedArrayCopy(byte[] data, int blockSize) {
         int size = MathUtils.roundUpInt(data.length, blockSize);
-        byte[] newData = Utils.newBytes(size);
+        byte[] newData = DataUtils.newBytes(size);
         System.arraycopy(data, 0, newData, 0, data.length);
         return newData;
     }

h2/src/main/org/h2/jdbc/JdbcResultSet.java

@@ -40,7 +40,6 @@ import org.h2.result.ResultInterface;
 import org.h2.result.UpdatableRow;
 import org.h2.util.DateTimeUtils;
 import org.h2.util.IOUtils;
-import org.h2.util.MathUtils;
 import org.h2.util.New;
 import org.h2.util.StringUtils;
 import org.h2.value.CompareMode;
@@ -721,7 +720,7 @@ public class JdbcResultSet extends TraceObject implements ResultSet {
                 throw DbException.getInvalidValueException("scale", scale);
             }
             BigDecimal bd = get(columnLabel).getBigDecimal();
-            return bd == null ? null : MathUtils.setScale(bd, scale);
+            return bd == null ? null : ValueDecimal.setScale(bd, scale);
         } catch (Exception e) {
             throw logAndConvert(e);
         }
@@ -747,7 +746,7 @@ public class JdbcResultSet extends TraceObject implements ResultSet {
                 throw DbException.getInvalidValueException("scale", scale);
             }
             BigDecimal bd = get(columnIndex).getBigDecimal();
-            return bd == null ? null : MathUtils.setScale(bd, scale);
+            return bd == null ? null : ValueDecimal.setScale(bd, scale);
         } catch (Exception e) {
             throw logAndConvert(e);
         }

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -13,12 +13,10 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.text.MessageFormat;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import org.h2.engine.Constants;
 import org.h2.util.New;
-import org.h2.util.StringUtils;
 
 /**
  * Utility methods
@@ -72,6 +70,11 @@ public class DataUtils {
      */
     public static final int PAGE_MEMORY_CHILD = 16;
 
+    /**
+     * An 0-size byte array.
+     */
+    private static final byte[] EMPTY_BYTES = {};
+
     /**
      * Get the length of the variable size int.
      *
@@ -199,8 +202,10 @@ public class DataUtils {
      * @param buff the target buffer
      * @param s the string
      * @param len the number of characters
+     * @return the byte buffer
      */
-    public static void writeStringData(ByteBuffer buff, String s, int len) {
+    public static ByteBuffer writeStringData(ByteBuffer buff, String s, int len) {
+        buff = DataUtils.ensureCapacity(buff, 3 * len);
         for (int i = 0; i < len; i++) {
             int c = s.charAt(i);
             if (c < 0x80) {
@@ -214,6 +219,7 @@ public class DataUtils {
                 buff.put((byte) (c & 0x3f));
             }
         }
+        return buff;
     }
 
     /**
@@ -584,7 +590,8 @@ public class DataUtils {
     public static IllegalArgumentException newIllegalArgumentException(
             String message, Object... arguments) {
         return initCause(new IllegalArgumentException(
-                        formatMessage(message, arguments)), arguments);
+                MessageFormat.format(message, arguments) + getVersion()),
+                arguments);
     }
 
     /**
@@ -608,7 +615,8 @@ public class DataUtils {
     public static IllegalStateException newIllegalStateException(
             String message, Object... arguments) {
         return initCause(new IllegalStateException(
-                formatMessage(message, arguments)), arguments);
+                MessageFormat.format(message, arguments) + getVersion()),
+                arguments);
     }
 
     private static <T extends Exception> T initCause(T e, Object... arguments) {
@@ -622,39 +630,111 @@ public class DataUtils {
         return e;
     }
 
-    private static String formatMessage(String pattern, Object... arguments) {
-        for (int i = 0, size = arguments.length; i < size; i++) {
-            Object o = arguments[i];
-            if (o instanceof String) {
-                arguments[i] = StringUtils.quoteIdentifier(o.toString());
-            }
-        }
-        return MessageFormat.format(pattern, arguments) + getVersion();
-    }
-
     private static String getVersion() {
         return " [" + Constants.VERSION_MAJOR + "." +
                 Constants.VERSION_MINOR + "." + Constants.BUILD_ID + "]";
     }
 
     /**
-     * Convert a char array to a byte array. The char array is cleared after
-     * use.
+     * Convert the text to UTF-8 format. For the Unicode characters
+     * 0xd800-0xdfff only one byte is returned.
+     *
+     * @param s the text
+     * @return the UTF-8 representation
+     */
+    public static byte[] utf8Encode(String s) {
+        try {
+            return s.getBytes(Constants.UTF8);
+        } catch (Exception e) {
+            // UnsupportedEncodingException
+            throw newIllegalArgumentException("UTF-8 not supported", e);
+        }
+    }
+
+    /**
+     * Convert a UTF-8 representation of a text to the text.
+     *
+     * @param utf8 the UTF-8 representation
+     * @return the text
+     */
+    public static String utf8Decode(byte[] utf8) {
+        try {
+            return new String(utf8, Constants.UTF8);
+        } catch (Exception e) {
+            // UnsupportedEncodingException
+            throw newIllegalArgumentException("UTF-8 not supported", e);
+        }
+    }
+
+    /**
+     * Convert a UTF-8 representation of a text to the text using the given
+     * offset and length.
+     *
+     * @param bytes the UTF-8 representation
+     * @param offset the offset in the bytes array
+     * @param length the number of bytes
+     * @return the text
+     */
+    public static String utf8Decode(byte[] bytes, int offset, int length) {
+        try {
+            return new String(bytes, offset, length, Constants.UTF8);
+        } catch (Exception e) {
+            // UnsupportedEncodingException
+            throw newIllegalArgumentException("UTF-8 not supported", e);
+        }
+    }
+
+    /**
+     * Create an array of bytes with the given size. If this is not possible
+     * because not enough memory is available, an OutOfMemoryError with the
+     * requested size in the message is thrown.
+     * <p>
+     * This method should be used if the size of the array is user defined, or
+     * stored in a file, so wrong size data can be distinguished from regular
+     * out-of-memory.
      *
-     * @param passwordChars the password characters
+     * @param len the number of bytes requested
      * @return the byte array
+     * @throws OutOfMemoryError
      */
-    static byte[] getPasswordBytes(char[] passwordChars) {
-        // using UTF-16
-        int len = passwordChars.length;
-        byte[] password = new byte[len * 2];
-        for (int i = 0; i < len; i++) {
-            char c = passwordChars[i];
-            password[i + i] = (byte) (c >>> 8);
-            password[i + i + 1] = (byte) c;
+    public static byte[] newBytes(int len) {
+        if (len == 0) {
+            return EMPTY_BYTES;
+        }
+        try {
+            return new byte[len];
+        } catch (OutOfMemoryError e) {
+            Error e2 = new OutOfMemoryError("Requested memory: " + len);
+            e2.initCause(e);
+            throw e2;
         }
-        Arrays.fill(passwordChars, (char) 0);
-        return password;
+    }
+
+    /**
+     * Ensure the byte buffer has the given capacity, plus 1 KB. If not, a new,
+     * larger byte buffer is created and the data is copied.
+     *
+     * @param buff the byte buffer
+     * @param len the minimum remaining capacity
+     * @return the byte buffer (possibly a new one)
+     */
+    public static ByteBuffer ensureCapacity(ByteBuffer buff, int len) {
+        len += 1024;
+        if (buff.remaining() > len) {
+            return buff;
+        }
+        return grow(buff, len);
+    }
+
+    private static ByteBuffer grow(ByteBuffer buff, int len) {
+        len = buff.remaining() + len;
+        int capacity = buff.capacity();
+        // grow at most 1 MB at a time
+        len = Math.max(len, Math.min(capacity + 1024 * 1024, capacity * 2));
+        ByteBuffer temp = ByteBuffer.allocate(len);
+        buff.flip();
+        temp.put(buff);
+        return temp;
     }
 
 }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -25,10 +25,9 @@ import org.h2.mvstore.cache.FilePathCache;
 import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FilePathCrypt;
-import org.h2.store.fs.FileUtils;
+import org.h2.store.fs.FilePathNio;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
-import org.h2.util.StringUtils;
 
 /*
 
@@ -43,7 +42,6 @@ H:3,...
 
 TODO:
 
-- separate jar file
 - automated 'kill process' and 'power failure' test
 - mvcc with multiple transactions
 - update checkstyle
@@ -53,7 +51,6 @@ TODO:
 - possibly split chunk data into immutable and mutable
 - compact: avoid processing pages using a counting bloom filter
 - defragment (re-creating maps, specially those with small pages)
-- remove DataType.getMaxLength (use ByteArrayOutputStream or getMemory)
 - chunk header: store changed chunk data as row; maybe after the root
 - chunk checksum (header, last page, 2 bytes per page?)
 - is there a better name for the file header,
@@ -84,7 +81,7 @@ TODO:
 - MVStoreTool.shrink to shrink a store (create, copy, rename, delete)
 -- and for MVStore on Windows, auto-detect renamed file
 - ensure data is overwritten eventually if the system doesn't have a timer
-- SSD-friendly write (always in blocks of 128 or 256 KB?)
+- SSD-friendly write (always in blocks of 4 MB / 1 second?)
 - close the file on out of memory or disk write error (out of disk space or so)
 - implement a sharded map (in one store, multiple stores)
 -- to support concurrent updates and writes, and very large maps
@@ -178,7 +175,13 @@ public class MVStore {
     private boolean closed;
 
     MVStore(HashMap<String, Object> config) {
-        this.fileName = (String) config.get("fileName");
+        String f = (String) config.get("fileName");
+        if (f != null && !f.startsWith("nio:")) {
+            // nio is used by default
+            FilePathNio.class.getName();
+            f = "nio:" + f;
+        }
+        this.fileName = f;
         this.readOnly = "r".equals(config.get("openMode"));
         this.compress = "1".equals(config.get("compress"));
         if (fileName != null) {
@@ -194,7 +197,8 @@ public class MVStore {
     }
 
     /**
-     * Open a store in exclusive mode.
+     * Open a store in exclusive mode. For a file-based store, the parent
+     * directory must already exist.
      *
      * @param fileName the file name (null for in-memory)
      * @return the store
@@ -370,7 +374,10 @@ public class MVStore {
         if (fileName == null) {
             return;
         }
-        FileUtils.createDirectories(FileUtils.getParent(fileName));
+        FilePath parent = FilePath.get(fileName).getParent();
+        if (!parent.exists()) {
+            throw DataUtils.newIllegalArgumentException("Directory does not exist: {0}", parent);
+        }
         try {
             if (readOnly) {
                 openFile();
@@ -402,7 +409,7 @@ public class MVStore {
             }
             file = f.open(readOnly ? "r" : "rw");
             if (filePassword != null) {
-                byte[] password = DataUtils.getPasswordBytes(filePassword);
+                byte[] password = FilePathCrypt.getPasswordBytes(filePassword);
                 file = new FilePathCrypt.FileCrypt(fileName, password, file);
             }
             file = FilePathCache.wrap(file);
@@ -496,7 +503,7 @@ public class MVStore {
         fileReadCount++;
         DataUtils.readFully(file, 0, buff);
         for (int i = 0; i < 3 * BLOCK_SIZE; i += BLOCK_SIZE) {
-            String s = StringUtils.utf8Decode(buff.array(), i, BLOCK_SIZE)
+            String s = DataUtils.utf8Decode(buff.array(), i, BLOCK_SIZE)
                     .trim();
             HashMap<String, String> m = DataUtils.parseMap(s);
             String f = m.remove("fletcher");
@@ -510,7 +517,7 @@ public class MVStore {
                 check = -1;
             }
             s = s.substring(0, s.lastIndexOf("fletcher") - 1) + " ";
-            byte[] bytes = StringUtils.utf8Encode(s);
+            byte[] bytes = DataUtils.utf8Encode(s);
             int checksum = DataUtils.getFletcher32(bytes, bytes.length / 2 * 2);
             if (check != checksum) {
                 continue;
@@ -535,10 +542,10 @@ public class MVStore {
         fileHeader.put("rootChunk", "" + rootChunkStart);
         fileHeader.put("version", "" + currentVersion);
         DataUtils.appendMap(buff, fileHeader);
-        byte[] bytes = StringUtils.utf8Encode(buff.toString() + " ");
+        byte[] bytes = DataUtils.utf8Encode(buff.toString() + " ");
         int checksum = DataUtils.getFletcher32(bytes, bytes.length / 2 * 2);
         DataUtils.appendMap(buff, "fletcher", Integer.toHexString(checksum));
-        bytes = StringUtils.utf8Encode(buff.toString());
+        bytes = DataUtils.utf8Encode(buff.toString());
         DataUtils.checkArgument(bytes.length <= BLOCK_SIZE,
                 "File header too large: {}", buff);
         return bytes;
@@ -655,7 +662,6 @@ public class MVStore {
         chunks.put(c.id, c);
         meta.put("chunk." + c.id, c.asString());
 
-        int maxLength = 1 + 4 + 4 + 8;
         for (MVMap<?, ?> m : mapsChanged.values()) {
             if (m == meta || !m.hasUnsavedChanges()) {
                 continue;
@@ -664,7 +670,6 @@ public class MVStore {
             if (p.getTotalCount() == 0) {
                 meta.put("root." + m.getId(), "0");
             } else {
-                maxLength += p.getMaxLengthTempRecursive();
                 meta.put("root." + m.getId(), String.valueOf(Long.MAX_VALUE));
             }
         }
@@ -681,17 +686,12 @@ public class MVStore {
                 applyFreedChunks();
             }
         } while (freedChunks.size() > 0);
-        maxLength += meta.getRoot().getMaxLengthTempRecursive();
         ByteBuffer buff;
-        if (maxLength > 16 * 1024 * 1024) {
-            buff = ByteBuffer.allocate(maxLength);
+        if (writeBuffer != null) {
+            buff = writeBuffer;
+            buff.clear();
         } else {
-            if (writeBuffer != null && writeBuffer.capacity() >= maxLength) {
-                buff = writeBuffer;
-                buff.clear();
-            } else {
-                writeBuffer = buff = ByteBuffer.allocate(maxLength + 128 * 1024);
-            }
+            buff = ByteBuffer.allocate(1024 * 1024);
         }
         // need to patch the header later
         c.writeHeader(buff);
@@ -703,7 +703,8 @@ public class MVStore {
             }
             Page p = m.openVersion(storeVersion).getRoot();
             if (p.getTotalCount() > 0) {
-                long root = p.writeUnsavedRecursive(c, buff);
+                buff = p.writeUnsavedRecursive(c, buff);
+                long root = p.getPos();
                 meta.put("root." + m.getId(), "" + root);
             }
         }
@@ -718,7 +719,7 @@ public class MVStore {
 
         // this will modify maxLengthLive, but
         // the correct value is written in the chunk header
-        meta.getRoot().writeUnsavedRecursive(c, buff);
+        buff = meta.getRoot().writeUnsavedRecursive(c, buff);
 
         int chunkLength = buff.position();
 
@@ -755,6 +756,9 @@ public class MVStore {
         fileWriteCount++;
         DataUtils.writeFully(file, filePos, buff);
         fileSize = Math.max(fileSize, filePos + buff.position());
+        if (buff.capacity() <= 4 * 1024 * 1024) {
+            writeBuffer = buff;
+        }
 
         // overwrite the header if required
         if (!atEnd) {
@@ -1440,7 +1444,7 @@ public class MVStore {
 
         /**
          * Use the following file name. If the file does not exist, it is
-         * automatically created.
+         * automatically created. The parent directory already must exist.
          *
          * @param fileName the file name
          * @return this

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -12,7 +12,6 @@ import java.io.Writer;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import org.h2.store.fs.FilePath;
-import org.h2.store.fs.FileUtils;
 
 /**
  * Utility methods used in combination with the MVStore.
@@ -47,7 +46,7 @@ public class MVStoreTool {
      */
     public static void dump(String fileName, Writer writer) {
         PrintWriter pw = new PrintWriter(writer, true);
-        if (!FileUtils.exists(fileName)) {
+        if (!FilePath.get(fileName).exists()) {
             pw.println("File not found: " + fileName);
             return;
         }

h2/src/main/org/h2/mvstore/Page.java

@@ -11,7 +11,6 @@ import java.nio.channels.FileChannel;
 import java.util.Arrays;
 import org.h2.compress.Compressor;
 import org.h2.mvstore.type.DataType;
-import org.h2.util.Utils;
 
 /**
  * A page (a node or a leaf).
@@ -720,7 +719,7 @@ public class Page {
             Compressor compressor = map.getStore().getCompressor();
             int lenAdd = DataUtils.readVarInt(buff);
             int compLen = pageLength + start - buff.position();
-            byte[] comp = Utils.newBytes(compLen);
+            byte[] comp = DataUtils.newBytes(compLen);
             buff.get(comp);
             int l = compLen + lenAdd;
             buff = ByteBuffer.allocate(l);
@@ -762,8 +761,9 @@ public class Page {
      *
      * @param chunk the chunk
      * @param buff the target buffer
+     * @return the target buffer
      */
-    private void write(Chunk chunk, ByteBuffer buff) {
+    private ByteBuffer write(Chunk chunk, ByteBuffer buff) {
         int start = buff.position();
         buff.putInt(0);
         buff.putShort((byte) 0);
@@ -776,7 +776,7 @@ public class Page {
         int compressStart = buff.position();
         DataType keyType = map.getKeyType();
         for (int i = 0; i < len; i++) {
-            keyType.write(buff, keys[i]);
+            buff = keyType.write(buff, keys[i]);
         }
         if (type == DataUtils.PAGE_TYPE_NODE) {
             for (int i = 0; i <= len; i++) {
@@ -788,7 +788,7 @@ public class Page {
         } else {
             DataType valueType = map.getValueType();
             for (int i = 0; i < len; i++) {
-                valueType.write(buff, values[i]);
+                buff = valueType.write(buff, values[i]);
             }
         }
         if (map.getStore().getCompress()) {
@@ -818,45 +818,7 @@ public class Page {
         chunk.maxLength += max;
         chunk.maxLengthLive += max;
         chunk.pageCount++;
-    }
-
-    /**
-     * Get the maximum length in bytes to store temporary records, recursively.
-     *
-     * @return the maximum length
-     */
-    int getMaxLengthTempRecursive() {
-        int maxLength = getMaxLength();
-        if (!isLeaf()) {
-            for (int i = 0; i <= keyCount; i++) {
-                Page p = childrenPages[i];
-                if (p != null) {
-                    maxLength += p.getMaxLengthTempRecursive();
-                }
-            }
-        }
-        return maxLength;
-    }
-
-    int getMaxLength() {
-        // length, check, map id, key length, type
-        int maxLength = 4 + 2 + DataUtils.MAX_VAR_INT_LEN
-                + DataUtils.MAX_VAR_INT_LEN + 1;
-        int len = keyCount;
-        DataType keyType = map.getKeyType();
-        for (int i = 0; i < len; i++) {
-            maxLength += keyType.getMaxLength(keys[i]);
-        }
-        if (isLeaf()) {
-            DataType valueType = map.getValueType();
-            for (int i = 0; i < len; i++) {
-                maxLength += valueType.getMaxLength(values[i]);
-            }
-        } else {
-            maxLength += 8 * len;
-            maxLength += DataUtils.MAX_VAR_LONG_LEN * len;
-        }
-        return maxLength;
+        return buff;
     }
 
     /**
@@ -865,21 +827,21 @@ public class Page {
      *
      * @param chunk the chunk
      * @param buff the target buffer
-     * @return the page id
+     * @return the target buffer
      */
-    long writeUnsavedRecursive(Chunk chunk, ByteBuffer buff) {
+    ByteBuffer writeUnsavedRecursive(Chunk chunk, ByteBuffer buff) {
         if (!isLeaf()) {
             int len = children.length;
             for (int i = 0; i < len; i++) {
                 Page p = childrenPages[i];
                 if (p != null) {
-                    children[i] = p.writeUnsavedRecursive(chunk, buff);
+                    buff = p.writeUnsavedRecursive(chunk, buff);
+                    children[i] = p.getPos();
                     childrenPages[i] = null;
                 }
             }
         }
-        write(chunk, buff);
-        return pos;
+        return write(chunk, buff);
     }
 
     long getVersion() {

h2/src/main/org/h2/mvstore/StreamStore.java

@@ -11,10 +11,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
-import org.h2.util.IOUtils;
-import org.h2.util.StringUtils;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * A facility to store streams in a map. Streams are split into blocks, which
@@ -41,6 +41,7 @@ public class StreamStore {
     private int minBlockSize = 256;
     private int maxBlockSize = 256 * 1024;
     private final AtomicLong nextKey = new AtomicLong();
+    private final AtomicReference<byte[]> nextBuffer = new AtomicReference<byte[]>();
 
     /**
      * Create a stream store instance.
@@ -128,25 +129,46 @@ public class StreamStore {
                 }
             }
         }
-        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-        int len = (int) IOUtils.copy(in, buffer, maxBlockSize);
+        byte[] readBuffer = nextBuffer.getAndSet(null);
+        if (readBuffer == null) {
+            readBuffer = new byte[maxBlockSize];
+        }
+        byte[] buff = read(in, readBuffer);
+        if (buff != readBuffer) {
+            // re-use the buffer if the result was shorter
+            nextBuffer.set(readBuffer);
+        }
+        int len = buff.length;
         if (len == 0) {
             return true;
         }
         boolean eof = len < maxBlockSize;
-        byte[] data = buffer.toByteArray();
         if (len < minBlockSize) {
             id.write(0);
             DataUtils.writeVarInt(id, len);
-            id.write(data);
+            id.write(buff);
         } else {
             id.write(1);
             DataUtils.writeVarInt(id, len);
-            DataUtils.writeVarLong(id, writeBlock(data));
+            DataUtils.writeVarLong(id, writeBlock(buff));
         }
         return eof;
     }
 
+    private static byte[] read(InputStream in, byte[] target) throws IOException {
+        int copied = 0;
+        int remaining = target.length;
+        while (remaining > 0) {
+            int len = in.read(target, copied, remaining);
+            if (len < 0) {
+                return Arrays.copyOf(target, copied);
+            }
+            copied += len;
+            remaining -= len;
+        }
+        return target;
+    }
+
     private ByteArrayOutputStream putIndirectId(ByteArrayOutputStream id) throws IOException {
         byte[] data = id.toByteArray();
         id = new ByteArrayOutputStream();
@@ -222,7 +244,7 @@ public class StreamStore {
                 break;
             default:
                 throw DataUtils.newIllegalArgumentException(
-                        "Unsupported id {0}", StringUtils.convertBytesToHex(id));
+                        "Unsupported id {0}", Arrays.toString(id));
             }
         }
     }
@@ -254,7 +276,7 @@ public class StreamStore {
                 break;
             default:
                 throw DataUtils.newIllegalArgumentException(
-                        "Unsupported id {0}", StringUtils.convertBytesToHex(id));
+                        "Unsupported id {0}", Arrays.toString(id));
             }
         }
         return length;
@@ -418,7 +440,7 @@ public class StreamStore {
                 }
                 default:
                     throw DataUtils.newIllegalArgumentException(
-                            "Unsupported id {0}", StringUtils.convertBytesToHex(idBuffer.array()));
+                            "Unsupported id {0}", Arrays.toString(idBuffer.array()));
                 }
             }
             return null;

h2/src/main/org/h2/mvstore/db/ValueArrayDataType.java

@@ -26,7 +26,6 @@ import org.h2.store.DataHandler;
 import org.h2.store.LobStorage;
 import org.h2.tools.SimpleResultSet;
 import org.h2.util.DateTimeUtils;
-import org.h2.util.Utils;
 import org.h2.value.CompareMode;
 import org.h2.value.Value;
 import org.h2.value.ValueArray;
@@ -136,20 +135,6 @@ public class ValueArrayDataType implements DataType {
         return a.compareTypeSave(b, compareMode);
     }
 
-    public int getMaxLength(Object obj) {
-        Value[] x = (Value[]) obj;
-        int len = x.length;
-        int result = DataUtils.MAX_VAR_INT_LEN;
-        for (int i = 0; i < len; i++) {
-            result += getMaxLength(x[i]);
-        }
-        return result;
-    }
-
-    private int getMaxLength(Value v) {
-        return Data.getValueLen(v, handler);
-    }
-
     public int getMemory(Object obj) {
         Value[] x = (Value[]) obj;
         int len = x.length;
@@ -173,20 +158,22 @@ public class ValueArrayDataType implements DataType {
         return x;
     }
 
-    public void write(ByteBuffer buff, Object obj) {
+    public ByteBuffer write(ByteBuffer buff, Object obj) {
         Value[] x = (Value[]) obj;
         int len = x.length;
         DataUtils.writeVarInt(buff, len);
         for (int i = 0; i < len; i++) {
-            writeValue(buff, x[i]);
+            buff = DataUtils.ensureCapacity(buff, 0);
+            buff = writeValue(buff, x[i]);
         }
+        return buff;
     }
 
-    private void writeValue(ByteBuffer buff, Value v) {
+    private ByteBuffer writeValue(ByteBuffer buff, Value v) {
         int start = buff.position();
         if (v == ValueNull.INSTANCE) {
             buff.put((byte) 0);
-            return;
+            return buff;
         }
         int type = v.getType();
         switch (type) {
@@ -251,6 +238,7 @@ public class ValueArrayDataType implements DataType {
                     writeVarInt(buff, scale);
                     byte[] bytes = b.toByteArray();
                     writeVarInt(buff, bytes.length);
+                    buff = DataUtils.ensureCapacity(buff, bytes.length);
                     buff.put(bytes, 0, bytes.length);
                 }
             }
@@ -305,6 +293,7 @@ public class ValueArrayDataType implements DataType {
             buff.put((byte) type);
             byte[] b = v.getBytesNoCopy();
             writeVarInt(buff, b.length);
+            buff = DataUtils.ensureCapacity(buff, b.length);
             buff.put(b, 0, b.length);
             break;
         }
@@ -317,6 +306,7 @@ public class ValueArrayDataType implements DataType {
             } else {
                 buff.put((byte) type);
                 writeVarInt(buff, b.length);
+                buff = DataUtils.ensureCapacity(buff, b.length);
                 buff.put(b, 0, b.length);
             }
             break;
@@ -333,17 +323,17 @@ public class ValueArrayDataType implements DataType {
             int len = s.length();
             if (len < 32) {
                 buff.put((byte) (STRING_0_31 + len));
-                writeStringWithoutLength(buff, s, len);
+                buff = writeStringWithoutLength(buff, s, len);
             } else {
                 buff.put((byte) type);
-                writeString(buff, s);
+                buff = writeString(buff, s);
             }
             break;
         }
         case Value.STRING_IGNORECASE:
         case Value.STRING_FIXED:
             buff.put((byte) type);
-            writeString(buff, v.getString());
+            buff = writeString(buff, v.getString());
             break;
         case Value.DOUBLE: {
             double x = v.getDouble();
@@ -393,10 +383,11 @@ public class ValueArrayDataType implements DataType {
                     writeVarLong(buff, lob.getPrecision());
                     buff.put((byte) (lob.useCompression() ? 1 : 0));
                     if (t == -2) {
-                        writeString(buff, lob.getFileName());
+                        buff = writeString(buff, lob.getFileName());
                     }
                 } else {
                     writeVarInt(buff, small.length);
+                    buff = DataUtils.ensureCapacity(buff, small.length);
                     buff.put(small, 0, small.length);
                 }
             } else {
@@ -409,6 +400,7 @@ public class ValueArrayDataType implements DataType {
                     writeVarLong(buff, lob.getPrecision());
                 } else {
                     writeVarInt(buff, small.length);
+                    buff = DataUtils.ensureCapacity(buff, small.length);
                     buff.put(small, 0, small.length);
                 }
             }
@@ -419,7 +411,8 @@ public class ValueArrayDataType implements DataType {
             Value[] list = ((ValueArray) v).getList();
             writeVarInt(buff, list.length);
             for (Value x : list) {
-                writeValue(buff, x);
+                buff = DataUtils.ensureCapacity(buff, 0);
+                buff = writeValue(buff, x);
             }
             break;
         }
@@ -432,7 +425,8 @@ public class ValueArrayDataType implements DataType {
                 int columnCount = meta.getColumnCount();
                 writeVarInt(buff, columnCount);
                 for (int i = 0; i < columnCount; i++) {
-                    writeString(buff, meta.getColumnName(i + 1));
+                    buff = DataUtils.ensureCapacity(buff, 0);
+                    buff = writeString(buff, meta.getColumnName(i + 1));
                     writeVarInt(buff, meta.getColumnType(i + 1));
                     writeVarInt(buff, meta.getPrecision(i + 1));
                     writeVarInt(buff, meta.getScale(i + 1));
@@ -442,7 +436,7 @@ public class ValueArrayDataType implements DataType {
                     for (int i = 0; i < columnCount; i++) {
                         int t = org.h2.value.DataType.convertSQLTypeToValueType(meta.getColumnType(i + 1));
                         Value val = org.h2.value.DataType.readValue(null, rs, i + 1, t);
-                        writeValue(buff, val);
+                        buff = writeValue(buff, val);
                     }
                 }
                 buff.put((byte) 0);
@@ -461,6 +455,7 @@ public class ValueArrayDataType implements DataType {
                         .throwInternalError("value size error: got " + (buff.position() - start) + " expected " + Data.getValueLen(v, handler));
             }
         }
+        return buff;
     }
 
     private static void writeVarInt(ByteBuffer buff, int x) {
@@ -479,13 +474,14 @@ public class ValueArrayDataType implements DataType {
         buff.put((byte) x);
     }
 
-    private static void writeString(ByteBuffer buff, String s) {
+    private static ByteBuffer writeString(ByteBuffer buff, String s) {
         int len = s.length();
         writeVarInt(buff, len);
-        writeStringWithoutLength(buff, s, len);
+        return writeStringWithoutLength(buff, s, len);
     }
 
-    private static void writeStringWithoutLength(ByteBuffer buff, String s, int len) {
+    private static ByteBuffer writeStringWithoutLength(ByteBuffer buff, String s, int len) {
+        buff = DataUtils.ensureCapacity(buff, 3 * len);
         for (int i = 0; i < len; i++) {
             int c = s.charAt(i);
             if (c < 0x80) {
@@ -499,6 +495,7 @@ public class ValueArrayDataType implements DataType {
                 buff.put((byte) (c & 0x3f));
             }
         }
+        return buff;
     }
 
     /**
@@ -540,7 +537,7 @@ public class ValueArrayDataType implements DataType {
         case Value.DECIMAL: {
             int scale = readVarInt(buff);
             int len = readVarInt(buff);
-            byte[] buff2 = Utils.newBytes(len);
+            byte[] buff2 = DataUtils.newBytes(len);
             buff.get(buff2, 0, len);
             BigInteger b = new BigInteger(buff2);
             return ValueDecimal.get(new BigDecimal(b, scale));
@@ -571,13 +568,13 @@ public class ValueArrayDataType implements DataType {
         }
         case Value.BYTES: {
             int len = readVarInt(buff);
-            byte[] b = Utils.newBytes(len);
+            byte[] b = DataUtils.newBytes(len);
             buff.get(b, 0, len);
             return ValueBytes.getNoCopy(b);
         }
         case Value.JAVA_OBJECT: {
             int len = readVarInt(buff);
-            byte[] b = Utils.newBytes(len);
+            byte[] b = DataUtils.newBytes(len);
             buff.get(b, 0, len);
             return ValueJavaObject.getNoCopy(null, b);
         }
@@ -605,7 +602,7 @@ public class ValueArrayDataType implements DataType {
         case Value.CLOB: {
             int smallLen = readVarInt(buff);
             if (smallLen >= 0) {
-                byte[] small = Utils.newBytes(smallLen);
+                byte[] small = DataUtils.newBytes(smallLen);
                 buff.get(small, 0, smallLen);
                 return LobStorage.createSmallLob(type, small);
             } else if (smallLen == -3) {
@@ -666,7 +663,7 @@ public class ValueArrayDataType implements DataType {
                 return ValueLong.get(type - LONG_0_7);
             } else if (type >= BYTES_0_31 && type < BYTES_0_31 + 32) {
                 int len = type - BYTES_0_31;
-                byte[] b = Utils.newBytes(len);
+                byte[] b = DataUtils.newBytes(len);
                 buff.get(b, 0, len);
                 return ValueBytes.getNoCopy(b);
             } else if (type >= STRING_0_31 && type < STRING_0_31 + 32) {

h2/src/main/org/h2/mvstore/rtree/SpatialDataType.java

@@ -47,18 +47,13 @@ public class SpatialDataType implements DataType {
         return la == lb;
     }
 
-    @Override
-    public int getMaxLength(Object obj) {
-        return 1 + dimensions * 8 + DataUtils.MAX_VAR_LONG_LEN;
-    }
-
     @Override
     public int getMemory(Object obj) {
         return 40 + dimensions * 4;
     }
 
     @Override
-    public void write(ByteBuffer buff, Object obj) {
+    public ByteBuffer write(ByteBuffer buff, Object obj) {
         SpatialKey k = (SpatialKey) obj;
         int flags = 0;
         for (int i = 0; i < dimensions; i++) {
@@ -74,6 +69,7 @@ public class SpatialDataType implements DataType {
             }
         }
         DataUtils.writeVarLong(buff, k.getId());
+        return buff;
     }
 
     @Override

h2/src/main/org/h2/mvstore/type/DataType.java

@@ -22,15 +22,6 @@ public interface DataType {
      */
     int compare(Object a, Object b);
 
-    /**
-     * Get the maximum length in bytes used to store an object. In many cases,
-     * this method can be faster than calculating the exact length.
-     *
-     * @param obj the object
-     * @return the maximum length
-     */
-    int getMaxLength(Object obj);
-
     /**
      * Estimate the used memory in bytes.
      *
@@ -44,8 +35,9 @@ public interface DataType {
      *
      * @param buff the target buffer
      * @param obj the value
+     * @return the byte buffer
      */
-    void write(ByteBuffer buff, Object obj);
+    ByteBuffer write(ByteBuffer buff, Object obj);
 
     /**
      * Read an object.

h2/src/main/org/h2/mvstore/type/StringDataType.java

@@ -20,10 +20,6 @@ public class StringDataType implements DataType {
         return a.toString().compareTo(b.toString());
     }
 
-    public int getMaxLength(Object obj) {
-        return DataUtils.MAX_VAR_INT_LEN + 3 * obj.toString().length();
-    }
-
     public int getMemory(Object obj) {
         return 24 + 2 * obj.toString().length();
     }
@@ -33,11 +29,11 @@ public class StringDataType implements DataType {
         return DataUtils.readString(buff, len);
     }
 
-    public void write(ByteBuffer buff, Object obj) {
+    public ByteBuffer write(ByteBuffer buff, Object obj) {
         String s = obj.toString();
         int len = s.length();
         DataUtils.writeVarInt(buff, len);
-        DataUtils.writeStringData(buff, s, len);
+        return DataUtils.writeStringData(buff, s, len);
     }
 
 }

h2/src/main/org/h2/server/pg/PgServerThread.java

@@ -35,6 +35,7 @@ import org.h2.jdbc.JdbcConnection;
 import org.h2.jdbc.JdbcPreparedStatement;
 import org.h2.jdbc.JdbcStatement;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
 import org.h2.util.IOUtils;
 import org.h2.util.JdbcUtils;
 import org.h2.util.ScriptReader;
@@ -132,7 +133,7 @@ public class PgServerThread implements Runnable {
         }
         int len = dataInRaw.readInt();
         len -= 4;
-        byte[] data = Utils.newBytes(len);
+        byte[] data = DataUtils.newBytes(len);
         dataInRaw.readFully(data, 0, len);
         dataIn = new DataInputStream(new ByteArrayInputStream(data, 0, len));
         switchBlock: switch (x) {
@@ -246,7 +247,7 @@ public class PgServerThread implements Runnable {
             int paramCount = readShort();
             for (int i = 0; i < paramCount; i++) {
                 int paramLen = readInt();
-                byte[] d2 = Utils.newBytes(paramLen);
+                byte[] d2 = DataUtils.newBytes(paramLen);
                 readFully(d2);
                 try {
                     setParameter(prep.prep, i, d2, formatCodes);

h2/src/main/org/h2/server/web/WebThread.java

@@ -23,10 +23,10 @@ import java.util.StringTokenizer;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.message.TraceSystem;
+import org.h2.mvstore.DataUtils;
 import org.h2.util.IOUtils;
 import org.h2.util.NetUtils;
 import org.h2.util.StringUtils;
-import org.h2.util.Utils;
 
 /**
  * For each connection to a session, an object of this class is created.
@@ -320,7 +320,7 @@ class WebThread extends WebApp implements Runnable {
         if (multipart) {
             uploadMultipart(input, len);
         } else if (session != null && len > 0) {
-            byte[] bytes = Utils.newBytes(len);
+            byte[] bytes = DataUtils.newBytes(len);
             for (int pos = 0; pos < len;) {
                 pos += input.read(bytes, pos, len - pos);
             }
@@ -360,7 +360,7 @@ class WebThread extends WebApp implements Runnable {
         RandomAccessFile f = new RandomAccessFile(file, "rw");
         int testSize = (int) Math.min(f.length(), Constants.IO_BUFFER_SIZE);
         f.seek(f.length() - testSize);
-        byte[] bytes = Utils.newBytes(Constants.IO_BUFFER_SIZE);
+        byte[] bytes = DataUtils.newBytes(Constants.IO_BUFFER_SIZE);
         f.readFully(bytes, 0, testSize);
         String s = new String(bytes, "ASCII");
         int x = s.lastIndexOf(boundary);

h2/src/main/org/h2/store/Data.java

@@ -24,10 +24,10 @@ import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
 import org.h2.tools.SimpleResultSet;
 import org.h2.util.DateTimeUtils;
 import org.h2.util.MathUtils;
-import org.h2.util.Utils;
 import org.h2.value.DataType;
 import org.h2.value.Value;
 import org.h2.value.ValueArray;
@@ -720,7 +720,7 @@ public class Data {
         case Value.DECIMAL: {
             int scale = readVarInt();
             int len = readVarInt();
-            byte[] buff = Utils.newBytes(len);
+            byte[] buff = DataUtils.newBytes(len);
             read(buff, 0, len);
             BigInteger b = new BigInteger(buff);
             return ValueDecimal.get(new BigDecimal(b, scale));
@@ -751,13 +751,13 @@ public class Data {
         }
         case Value.BYTES: {
             int len = readVarInt();
-            byte[] b = Utils.newBytes(len);
+            byte[] b = DataUtils.newBytes(len);
             read(b, 0, len);
             return ValueBytes.getNoCopy(b);
         }
         case Value.JAVA_OBJECT: {
             int len = readVarInt();
-            byte[] b = Utils.newBytes(len);
+            byte[] b = DataUtils.newBytes(len);
             read(b, 0, len);
             return ValueJavaObject.getNoCopy(null, b);
         }
@@ -785,7 +785,7 @@ public class Data {
         case Value.CLOB: {
             int smallLen = readVarInt();
             if (smallLen >= 0) {
-                byte[] small = Utils.newBytes(smallLen);
+                byte[] small = DataUtils.newBytes(smallLen);
                 read(small, 0, smallLen);
                 return LobStorage.createSmallLob(type, small);
             } else if (smallLen == -3) {
@@ -846,7 +846,7 @@ public class Data {
                 return ValueLong.get(type - LONG_0_7);
             } else if (type >= BYTES_0_31 && type < BYTES_0_31 + 32) {
                 int len = type - BYTES_0_31;
-                byte[] b = Utils.newBytes(len);
+                byte[] b = DataUtils.newBytes(len);
                 read(b, 0, len);
                 return ValueBytes.getNoCopy(b);
             } else if (type >= STRING_0_31 && type < STRING_0_31 + 32) {
@@ -1260,7 +1260,7 @@ public class Data {
     }
 
     private void expand(int plus) {
-        byte[] d = Utils.newBytes((data.length + plus) * 2);
+        byte[] d = DataUtils.newBytes((data.length + plus) * 2);
         // must copy everything, because pos could be 0 and data may be
         // still required
         System.arraycopy(data, 0, d, 0, data.length);

h2/src/main/org/h2/store/FileStoreInputStream.java

@@ -10,8 +10,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
 import org.h2.tools.CompressTool;
-import org.h2.util.Utils;
 
 /**
  * An input stream that is backed by a file store.
@@ -114,7 +114,7 @@ public class FileStoreInputStream extends InputStream {
         page.readInt();
         if (compress != null) {
             int uncompressed = page.readInt();
-            byte[] buff = Utils.newBytes(remainingInBuffer);
+            byte[] buff = DataUtils.newBytes(remainingInBuffer);
             page.read(buff, 0, remainingInBuffer);
             page.reset();
             page.checkCapacity(uncompressed);

h2/src/main/org/h2/store/RecoverTester.java

@@ -100,7 +100,7 @@ public class RecoverTester implements Recorder {
     private synchronized void testDatabase(String fileName, PrintWriter out) {
         out.println("+ write #" + writeCount + " verify #" + verifyCount);
         try {
-            FileUtils.copy(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
+            IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
             verifyCount++;
             // avoid using the Engine class to avoid deadlocks
             Properties p = new Properties();
@@ -145,7 +145,7 @@ public class RecoverTester implements Recorder {
         }
         testDatabase += "X";
         try {
-            FileUtils.copy(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
+            IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
             // avoid using the Engine class to avoid deadlocks
             Properties p = new Properties();
             ConnectionInfo ci = new ConnectionInfo("jdbc:h2:" + testDatabase + ";FILE_LOCK=NO", p);

h2/src/main/org/h2/store/fs/FilePath.java

@@ -15,7 +15,6 @@ import java.util.List;
 import java.util.Map;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
-import org.h2.util.StringUtils;
 
 /**
  * A path to a file. It similar to the Java 7 <code>java.nio.file.Path</code>,
@@ -25,7 +24,7 @@ import org.h2.util.StringUtils;
  */
 public abstract class FilePath {
 
-    private static final FilePath DEFAULT = new FilePathDisk();
+    private static FilePath defaultProvider;
 
     private static Map<String, FilePath> providers;
 
@@ -51,25 +50,26 @@ public abstract class FilePath {
     public static FilePath get(String path) {
         path = path.replace('\\', '/');
         int index = path.indexOf(':');
+        registerDefaultProviders();
         if (index < 2) {
             // use the default provider if no prefix or
             // only a single character (drive name)
-            return DEFAULT.getPath(path);
+            return defaultProvider.getPath(path);
         }
         String scheme = path.substring(0, index);
-        registerDefaultProviders();
         FilePath p = providers.get(scheme);
         if (p == null) {
             // provider not found - use the default
-            p = DEFAULT;
+            p = defaultProvider;
         }
         return p.getPath(path);
     }
 
     private static void registerDefaultProviders() {
-        if (providers == null) {
+        if (providers == null || defaultProvider == null) {
             Map<String, FilePath> map = Collections.synchronizedMap(New.<String, FilePath>hashMap());
             for (String c : new String[] {
+                    "org.h2.store.fs.FilePathDisk",
                     "org.h2.store.fs.FilePathMem",
                     "org.h2.store.fs.FilePathMemLZF",
                     "org.h2.store.fs.FilePathSplit",
@@ -80,6 +80,9 @@ public abstract class FilePath {
                 try {
                     FilePath p = (FilePath) Class.forName(c).newInstance();
                     map.put(p.getScheme(), p);
+                    if (defaultProvider == null) {
+                        defaultProvider = p;
+                    }
                 } catch (Exception e) {
                     // ignore - the files may be excluded in purpose
                 }
@@ -267,9 +270,7 @@ public abstract class FilePath {
      */
     protected static synchronized String getNextTempFileNamePart(boolean newRandom) {
         if (newRandom || tempRandom == null) {
-            byte[] prefix = new byte[8];
-            MathUtils.randomBytes(prefix);
-            tempRandom = StringUtils.convertBytesToHex(prefix) + ".";
+            tempRandom = MathUtils.randomInt(Integer.MAX_VALUE) + ".";
         }
         return tempRandom + tempSequence++;
     }

h2/src/main/org/h2/store/fs/FilePathCrypt.java

@@ -19,7 +19,6 @@ import org.h2.security.AES;
 import org.h2.security.BlockCipher;
 import org.h2.security.SHA256;
 import org.h2.util.MathUtils;
-import org.h2.util.StringUtils;
 
 /**
  * An encrypted file.
@@ -38,7 +37,7 @@ public class FilePathCrypt extends FilePathWrapper {
     public FileChannel open(String mode) throws IOException {
         String[] parsed = parse(name);
         FileChannel file = FileUtils.open(parsed[1], mode);
-        byte[] passwordBytes = StringUtils.convertHexToBytes(parsed[0]);
+        byte[] passwordBytes = DataUtils.utf8Encode(parsed[0]);
         return new FileCrypt(name, passwordBytes, file);
     }
 
@@ -93,6 +92,26 @@ public class FilePathCrypt extends FilePathWrapper {
         return new String[] { password, fileName };
     }
 
+    /**
+     * Convert a char array to a byte array. The char array is cleared after
+     * use.
+     *
+     * @param passwordChars the password characters
+     * @return the byte array
+     */
+    public static byte[] getPasswordBytes(char[] passwordChars) {
+        // using UTF-16
+        int len = passwordChars.length;
+        byte[] password = new byte[len * 2];
+        for (int i = 0; i < len; i++) {
+            char c = passwordChars[i];
+            password[i + i] = (byte) (c >>> 8);
+            password[i + i + 1] = (byte) c;
+        }
+        Arrays.fill(passwordChars, (char) 0);
+        return password;
+    }
+
     /**
      * An encrypted file with a read cache.
      */

h2/src/main/org/h2/store/fs/FilePathWrapper.java

@@ -11,7 +11,6 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.channels.FileChannel;
 import java.util.List;
-import org.h2.message.DbException;
 
 /**
  * The base class for wrapping / delegating file systems such as
@@ -46,7 +45,7 @@ public abstract class FilePathWrapper extends FilePath {
             p.base = base;
             return p;
         } catch (Exception e) {
-            throw DbException.convert(e);
+            throw new IllegalArgumentException("Path: " + path, e);
         }
     }
 

h2/src/main/org/h2/store/fs/FileUtils.java

@@ -13,9 +13,6 @@ import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.List;
-import org.h2.constant.ErrorCode;
-import org.h2.message.DbException;
-import org.h2.util.IOUtils;
 import org.h2.util.New;
 
 /**
@@ -290,9 +287,8 @@ public class FileUtils {
         if (dir != null) {
             if (exists(dir)) {
                 if (!isDirectory(dir)) {
-                    DbException.get(ErrorCode.FILE_CREATION_FAILED_1,
-                            "Could not create directory, " +
-                            "because a file with the same name already exists: " + dir);
+                    // this will fail
+                    createDirectory(dir);
                 }
             } else {
                 String parent = getParent(dir);
@@ -302,18 +298,6 @@ public class FileUtils {
         }
     }
 
-    /**
-     * Copy a file from one directory to another, or to another file.
-     *
-     * @param original the original file name
-     * @param copy the file name of the copy
-     */
-    public static void copy(String original, String copy) throws IOException {
-        InputStream in = newInputStream(original);
-        OutputStream out = newOutputStream(copy, false);
-        IOUtils.copyAndClose(in, out);
-    }
-
     /**
      * Try to delete a file (ignore errors).
      *

h2/src/main/org/h2/tools/CompressTool.java

@@ -26,7 +26,7 @@ import org.h2.compress.LZFOutputStream;
 import org.h2.constant.ErrorCode;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
-import org.h2.util.Utils;
+import org.h2.mvstore.DataUtils;
 import org.h2.util.StringUtils;
 
 /**
@@ -43,10 +43,10 @@ public class CompressTool {
 
     private byte[] getBuffer(int min) {
         if (min > MAX_BUFFER_SIZE) {
-            return Utils.newBytes(min);
+            return DataUtils.newBytes(min);
         }
         if (cachedBuffer == null || cachedBuffer.length < min) {
-            cachedBuffer = Utils.newBytes(min);
+            cachedBuffer = DataUtils.newBytes(min);
         }
         return cachedBuffer;
     }
@@ -78,7 +78,7 @@ public class CompressTool {
         Compressor compress = getCompressor(algorithm);
         byte[] buff = getBuffer((len < 100 ? len + 100 : len) * 2);
         int newLen = compress(in, in.length, compress, buff);
-        byte[] out = Utils.newBytes(newLen);
+        byte[] out = DataUtils.newBytes(newLen);
         System.arraycopy(buff, 0, out, 0, newLen);
         return out;
     }
@@ -108,7 +108,7 @@ public class CompressTool {
         try {
             int len = readVariableInt(in, 1);
             int start = 1 + getVariableIntLength(len);
-            byte[] buff = Utils.newBytes(len);
+            byte[] buff = DataUtils.newBytes(len);
             compress.expand(in, start, in.length - start, buff, 0, len);
             return buff;
         } catch (Exception e) {

h2/src/main/org/h2/util/IOUtils.java

@@ -24,6 +24,7 @@ import java.io.Writer;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
+import org.h2.store.fs.FileUtils;
 
 /**
  * This utility class contains input/output functions.
@@ -477,4 +478,16 @@ public class IOUtils {
         return new ByteArrayInputStream(StringUtils.utf8Encode(s));
     }
 
+    /**
+     * Copy a file from one directory to another, or to another file.
+     *
+     * @param original the original file name
+     * @param copy the file name of the copy
+     */
+    public static void copyFiles(String original, String copy) throws IOException {
+        InputStream in = FileUtils.newInputStream(original);
+        OutputStream out = FileUtils.newOutputStream(copy, false);
+        copyAndClose(in, out);
+    }
+
 }

h2/src/main/org/h2/util/MathUtils.java

@@ -10,10 +10,8 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.lang.reflect.Method;
-import java.math.BigDecimal;
 import java.security.SecureRandom;
 import java.util.Random;
-import org.h2.message.DbException;
 
 /**
  * This is a utility class with mathematical helper functions.
@@ -32,14 +30,37 @@ public class MathUtils {
 
     private static final Random RANDOM  = new Random();
 
+    private MathUtils() {
+        // utility class
+    }
+
+
     /**
-     * The maximum scale of a BigDecimal value.
+     * Round the value up to the next block size. The block size must be a power
+     * of two. As an example, using the block size of 8, the following rounding
+     * operations are done: 0 stays 0; values 1..8 results in 8, 9..16 results
+     * in 16, and so on.
+     *
+     * @param x the value to be rounded
+     * @param blockSizePowerOf2 the block size
+     * @return the rounded value
      */
-    private static final int BIG_DECIMAL_SCALE_MAX = 100000;
-
+    public static int roundUpInt(int x, int blockSizePowerOf2) {
+        return (x + blockSizePowerOf2 - 1) & (-blockSizePowerOf2);
+    }
 
-    private MathUtils() {
-        // utility class
+    /**
+     * Round the value up to the next block size. The block size must be a power
+     * of two. As an example, using the block size of 8, the following rounding
+     * operations are done: 0 stays 0; values 1..8 results in 8, 9..16 results
+     * in 16, and so on.
+     *
+     * @param x the value to be rounded
+     * @param blockSizePowerOf2 the block size
+     * @return the rounded value
+     */
+    public static long roundUpLong(long x, long blockSizePowerOf2) {
+        return (x + blockSizePowerOf2 - 1) & (-blockSizePowerOf2);
     }
 
     private static synchronized SecureRandom getSecureRandom() {
@@ -198,34 +219,6 @@ public class MathUtils {
         }
     }
 
-    /**
-     * Round the value up to the next block size. The block size must be a power
-     * of two. As an example, using the block size of 8, the following rounding
-     * operations are done: 0 stays 0; values 1..8 results in 8, 9..16 results
-     * in 16, and so on.
-     *
-     * @param x the value to be rounded
-     * @param blockSizePowerOf2 the block size
-     * @return the rounded value
-     */
-    public static int roundUpInt(int x, int blockSizePowerOf2) {
-        return (x + blockSizePowerOf2 - 1) & (-blockSizePowerOf2);
-    }
-
-    /**
-     * Round the value up to the next block size. The block size must be a power
-     * of two. As an example, using the block size of 8, the following rounding
-     * operations are done: 0 stays 0; values 1..8 results in 8, 9..16 results
-     * in 16, and so on.
-     *
-     * @param x the value to be rounded
-     * @param blockSizePowerOf2 the block size
-     * @return the rounded value
-     */
-    public static long roundUpLong(long x, long blockSizePowerOf2) {
-        return (x + blockSizePowerOf2 - 1) & (-blockSizePowerOf2);
-    }
-
     /**
      * Get the value that is equal or higher than this value, and that is a
      * power of two.
@@ -241,20 +234,6 @@ public class MathUtils {
         return (int) i;
     }
 
-    /**
-     * Set the scale of a BigDecimal value.
-     *
-     * @param bd the BigDecimal value
-     * @param scale the new scale
-     * @return the scaled value
-     */
-    public static BigDecimal setScale(BigDecimal bd, int scale) {
-        if (scale > BIG_DECIMAL_SCALE_MAX || scale < -BIG_DECIMAL_SCALE_MAX) {
-            throw DbException.getInvalidValueException("scale", scale);
-        }
-        return bd.setScale(scale, BigDecimal.ROUND_HALF_UP);
-    }
-
     /**
      * Convert a long value to an int value. Values larger than the biggest int
      * value is converted to the biggest int value, and values smaller than the

h2/src/main/org/h2/util/Utils.java

@@ -407,32 +407,6 @@ public class Utils {
         }
     }
 
-    /**
-     * Create an array of bytes with the given size. If this is not possible
-     * because not enough memory is available, an OutOfMemoryError with the
-     * requested size in the message is thrown.
-     * <p>
-     * This method should be used if the size of the array is user defined, or
-     * stored in a file, so wrong size data can be distinguished from regular
-     * out-of-memory.
-     *
-     * @param len the number of bytes requested
-     * @return the byte array
-     * @throws OutOfMemoryError
-     */
-    public static byte[] newBytes(int len) {
-        if (len == 0) {
-            return EMPTY_BYTES;
-        }
-        try {
-            return new byte[len];
-        } catch (OutOfMemoryError e) {
-            Error e2 = new OutOfMemoryError("Requested memory: " + len);
-            e2.initCause(e);
-            throw e2;
-        }
-    }
-
     /**
      * Create an int array with the given size.
      *

h2/src/main/org/h2/value/Transfer.java

@@ -26,6 +26,7 @@ import org.h2.engine.Constants;
 import org.h2.engine.SessionInterface;
 import org.h2.message.DbException;
 import org.h2.message.TraceSystem;
+import org.h2.mvstore.DataUtils;
 import org.h2.security.SHA256;
 import org.h2.store.Data;
 import org.h2.store.DataReader;
@@ -288,7 +289,7 @@ public class Transfer {
         if (len == -1) {
             return null;
         }
-        byte[] b = Utils.newBytes(len);
+        byte[] b = DataUtils.newBytes(len);
         in.readFully(b);
         return b;
     }

h2/src/main/org/h2/value/ValueDecimal.java

@@ -45,6 +45,11 @@ public class ValueDecimal extends Value {
 
     private static final int DIVIDE_SCALE_ADD = 25;
 
+    /**
+     * The maximum scale of a BigDecimal value.
+     */
+    private static final int BIG_DECIMAL_SCALE_MAX = 100000;
+
     private final BigDecimal value;
     private String valueString;
     private int precision;
@@ -183,7 +188,7 @@ public class ValueDecimal extends Value {
                 return this;
             }
         }
-        BigDecimal bd = MathUtils.setScale(value, targetScale);
+        BigDecimal bd = ValueDecimal.setScale(value, targetScale);
         return ValueDecimal.get(bd);
     }
 
@@ -229,4 +234,18 @@ public class ValueDecimal extends Value {
         return value.precision() + 120;
     }
 
+    /**
+     * Set the scale of a BigDecimal value.
+     *
+     * @param bd the BigDecimal value
+     * @param scale the new scale
+     * @return the scaled value
+     */
+    public static BigDecimal setScale(BigDecimal bd, int scale) {
+        if (scale > BIG_DECIMAL_SCALE_MAX || scale < -BIG_DECIMAL_SCALE_MAX) {
+            throw DbException.getInvalidValueException("scale", scale);
+        }
+        return bd.setScale(scale, BigDecimal.ROUND_HALF_UP);
+    }
+
 }

h2/src/main/org/h2/value/ValueLob.java

@@ -17,6 +17,7 @@ import java.sql.SQLException;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
 import org.h2.store.DataHandler;
 import org.h2.store.FileStore;
 import org.h2.store.FileStoreInputStream;
@@ -360,11 +361,11 @@ public class ValueLob extends Value {
                 buff = IOUtils.readBytesAndClose(in, -1);
                 len = buff.length;
             } else {
-                buff = Utils.newBytes(len);
+                buff = DataUtils.newBytes(len);
                 len = IOUtils.readFully(in, buff, 0, len);
             }
             if (len <= handler.getMaxLengthInplaceLob()) {
-                byte[] small = Utils.newBytes(len);
+                byte[] small = DataUtils.newBytes(len);
                 System.arraycopy(buff, 0, small, 0, len);
                 return ValueLob.createSmallLob(Value.BLOB, small);
             }
@@ -696,7 +697,7 @@ public class ValueLob extends Value {
             int len = getBufferSize(h, compress, Long.MAX_VALUE);
             int tabId = tableId;
             if (type == Value.BLOB) {
-                createFromStream(Utils.newBytes(len), 0, getInputStream(), Long.MAX_VALUE, h);
+                createFromStream(DataUtils.newBytes(len), 0, getInputStream(), Long.MAX_VALUE, h);
             } else {
                 createFromReader(new char[len], 0, getReader(), Long.MAX_VALUE, h);
             }
@@ -757,7 +758,7 @@ public class ValueLob extends Value {
     private static void copyFileTo(DataHandler h, String sourceFileName, String targetFileName) {
         synchronized (h.getLobSyncObject()) {
             try {
-                FileUtils.copy(sourceFileName, targetFileName);
+                IOUtils.copyFiles(sourceFileName, targetFileName);
             } catch (IOException e) {
                 throw DbException.convertIOException(e, null);
             }

h2/src/main/org/h2/value/ValueLobDb.java

@@ -16,6 +16,7 @@ import java.sql.SQLException;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
 import org.h2.store.DataHandler;
 import org.h2.store.FileStore;
 import org.h2.store.FileStoreInputStream;
@@ -438,11 +439,11 @@ public class ValueLobDb extends Value implements Value.ValueClob, Value.ValueBlo
                 buff = IOUtils.readBytesAndClose(in, -1);
                 len = buff.length;
             } else {
-                buff = Utils.newBytes(len);
+                buff = DataUtils.newBytes(len);
                 len = IOUtils.readFully(in, buff, 0, len);
             }
             if (len <= handler.getMaxLengthInplaceLob()) {
-                byte[] small = Utils.newBytes(len);
+                byte[] small = DataUtils.newBytes(len);
                 System.arraycopy(buff, 0, small, 0, len);
                 return ValueLobDb.createSmallLob(Value.BLOB, small, small.length);
             }

h2/src/main/org/h2/value/ValueTimestamp.java

@@ -211,7 +211,7 @@ public class ValueTimestamp extends Value {
         long n = nanos;
         BigDecimal bd = BigDecimal.valueOf(n);
         bd = bd.movePointLeft(9);
-        bd = MathUtils.setScale(bd, targetScale);
+        bd = ValueDecimal.setScale(bd, targetScale);
         bd = bd.movePointRight(9);
         long n2 = bd.longValue();
         if (n2 == n) {

h2/src/test/org/h2/test/store/RowDataType.java

@@ -46,16 +46,6 @@ public class RowDataType implements DataType {
         return 0;
     }
 
-    public int getMaxLength(Object obj) {
-        Object[] x = (Object[]) obj;
-        int len = x.length;
-        int result = DataUtils.MAX_VAR_INT_LEN;
-        for (int i = 0; i < len; i++) {
-            result += types[i].getMaxLength(x[i]);
-        }
-        return result;
-    }
-
     public int getMemory(Object obj) {
         Object[] x = (Object[]) obj;
         int len = x.length;
@@ -75,13 +65,15 @@ public class RowDataType implements DataType {
         return x;
     }
 
-    public void write(ByteBuffer buff, Object obj) {
+    public ByteBuffer write(ByteBuffer buff, Object obj) {
         Object[] x = (Object[]) obj;
         int len = x.length;
         DataUtils.writeVarInt(buff, len);
         for (int i = 0; i < len; i++) {
-            types[i].write(buff, x[i]);
+            buff = DataUtils.ensureCapacity(buff, 0);
+            buff = types[i].write(buff, x[i]);
         }
+        return buff;
     }
 
 }

h2/src/test/org/h2/test/store/TestConcurrent.java

@@ -37,6 +37,8 @@ public class TestConcurrent extends TestMVStore {
 
     public void test() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
+        FileUtils.createDirectories(getBaseDir());
+
         testConcurrentOnlineBackup();
         testConcurrentMap();
         testConcurrentIterate();

h2/src/test/org/h2/test/store/TestMVRTree.java

@@ -43,6 +43,9 @@ public class TestMVRTree extends TestMVStore {
     }
 
     public void test() {
+        FileUtils.deleteRecursive(getBaseDir(), true);
+        FileUtils.createDirectories(getBaseDir());
+
         testExample();
         testMany();
         testSimple();

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -40,6 +40,8 @@ public class TestMVStore extends TestBase {
 
     public void test() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
+        FileUtils.createDirectories(getBaseDir());
+
         testEncryptedFile();
         testFileFormatChange();
         testRecreateMap();

h2/src/test/org/h2/test/store/TestObjectDataType.java

@@ -127,14 +127,14 @@ public class TestObjectDataType extends TestBase {
         assertTrue(ot.getMemory(x) >= 0);
 
         ot.getMemory(last);
-        assertTrue(ot.getMaxLength(x) >= 1);
+        assertTrue(ot.getMemory(x) >= 0);
 
         ot.getMemory(last);
         assertEquals(0, ot.compare(x, x));
-        ByteBuffer buff = ByteBuffer.allocate(ot.getMaxLength(x) + 1);
+        ByteBuffer buff = ByteBuffer.allocate(1024);
 
         ot.getMemory(last);
-        ot.write(buff, x);
+        buff = ot.write(buff, x);
         buff.put((byte) 123);
         buff.flip();
 

h2/src/test/org/h2/test/store/TestStreamStore.java

@@ -40,6 +40,9 @@ public class TestStreamStore extends TestBase {
 
     @Override
     public void test() throws IOException {
+        FileUtils.deleteRecursive(getBaseDir(), true);
+        FileUtils.createDirectories(getBaseDir());
+
         testVeryLarge();
         testDetectIllegalId();
         testTreeStructure();

h2/src/test/org/h2/test/unit/TestClearReferences.java

@@ -29,6 +29,7 @@ public class TestClearReferences extends TestBase {
         "org.h2.engine.SessionRemote.sessionFactory",
         "org.h2.jdbcx.JdbcDataSourceFactory.cachedTraceSystem",
         "org.h2.store.RecoverTester.instance",
+        "org.h2.store.fs.FilePath.defaultProvider",
         "org.h2.store.fs.FilePath.providers",
         "org.h2.store.fs.FilePath.tempRandom",
         "org.h2.tools.CompressTool.cachedBuffer",

h2/src/test/org/h2/test/unit/TestFileSystem.java

@@ -31,6 +31,7 @@ import org.h2.test.utils.AssertThrows;
 import org.h2.test.utils.FilePathDebug;
 import org.h2.tools.Backup;
 import org.h2.tools.DeleteDbFiles;
+import org.h2.util.IOUtils;
 
 /**
  * Tests various file system.
@@ -425,7 +426,7 @@ public class TestFileSystem extends TestBase {
         List<String> list = FileUtils.newDirectoryStream(fsBase);
         assertEquals(1, list.size());
         assertTrue(list.get(0).endsWith("test"));
-        FileUtils.copy(fsBase + "/test", fsBase + "/test3");
+        IOUtils.copyFiles(fsBase + "/test", fsBase + "/test3");
         FileUtils.moveTo(fsBase + "/test3", fsBase + "/test2");
         FileUtils.moveTo(fsBase + "/test2", fsBase + "/test2");
         assertTrue(!FileUtils.exists(fsBase + "/test3"));

h2/src/test/org/h2/test/unit/TestReopen.java

@@ -20,6 +20,7 @@ import org.h2.store.fs.FileUtils;
 import org.h2.store.fs.Recorder;
 import org.h2.test.TestBase;
 import org.h2.tools.Recover;
+import org.h2.util.IOUtils;
 import org.h2.util.New;
 import org.h2.util.Profiler;
 import org.h2.util.Utils;
@@ -95,7 +96,7 @@ public class TestReopen extends TestBase implements Recorder {
         System.out.println("+ write #" + writeCount + " verify #" + verifyCount);
 
         try {
-            FileUtils.copy(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
+            IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
             verifyCount++;
             // avoid using the Engine class to avoid deadlocks
             Properties p = new Properties();
@@ -142,7 +143,7 @@ public class TestReopen extends TestBase implements Recorder {
         }
         testDatabase += "X";
         try {
-            FileUtils.copy(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
+            IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
             // avoid using the Engine class to avoid deadlocks
             Properties p = new Properties();
             ConnectionInfo ci = new ConnectionInfo("jdbc:h2:" + testDatabase + ";FILE_LOCK=NO", p);

h2/src/tools/org/h2/dev/fs/FileShell.java

@@ -200,7 +200,7 @@ public class FileShell extends Tool {
             String source = getFile(list[i++]);
             String target = getFile(list[i++]);
             end(list, i);
-            FileUtils.copy(source, target);
+            IOUtils.copyFiles(source, target);
         } else if ("head".equals(c)) {
             String file = getFile(list[i++]);
             end(list, i);