MVStore: improved serialization (support arrays, java.util.Date)

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -596,9 +596,9 @@ public class DataUtils {
     private static String formatMessage(String pattern, Object... arguments) {
         for (int i = 0, size = arguments.length; i < size; i++) {
             Object o = arguments[i];
-            String s = o == null ? "null" : o instanceof String ? StringUtils
-                    .quoteIdentifier(o.toString()) : o.toString();
-            arguments[i] = s;
+            if (o instanceof String) {
+                arguments[i] = StringUtils.quoteIdentifier(o.toString());
+            }
         }
         return MessageFormat.format(pattern, arguments) + getVersion();
     }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -41,49 +41,43 @@ H:3,...
 
 TODO:
 
+- file system encryption: check standard
+
+- mvcc with multiple transactions
 - update checkstyle
 - automated 'kill process' and 'power failure' test
 - maybe split database into multiple files, to speed up compact
 - auto-compact from time to time and on close
 - test and possibly improve compact operation (for large dbs)
-- limited support for writing to old versions (branches)
-- on insert, if the child page is already full, don't load and modify it
--- split directly (for leaves with 1 entry)
 - performance test with encrypting file system
 - possibly split chunk data into immutable and mutable
 - compact: avoid processing pages using a counting bloom filter
 - defragment (re-creating maps, specially those with small pages)
-- write using ByteArrayOutputStream; remove DataType.getMaxLength
-- file header: check formatRead and format (is formatRead
--- needed if equal to format?)
+- remove DataType.getMaxLength (use ByteArrayOutputStream or getMemory)
 - chunk header: store changed chunk data as row; maybe after the root
 - chunk checksum (header, last page, 2 bytes per page?)
-- allow renaming maps
 - file locking: solve problem that locks are shared for a VM
-- online backup
-- data types: maybe support InputStream, Reader
-- data types: maybe support ResultSet, Date, Time, Timestamp
-- data types: maybe support boolean[], short[],...
 - store file "header" at the end of each chunk; at the end of the file
 - is there a better name for the file header,
 -- if it's no longer always at the beginning of a file?
+- on insert, if the child page is already full, don't load and modify it
+-- split directly (for leaves with 1 entry)
 - maybe let a chunk point to possible next chunks
 -- (so no fixed location header is needed)
 - support stores that span multiple files (chunks stored in other files)
 - triggers (can be implemented with a custom map)
-- store write operations per page (maybe defragment
+- store number of write operations per page (maybe defragment
 -- if much different than count)
 - r-tree: nearest neighbor search
 - use FileChannel by default (nio file system), but:
+-- an interrupt closes the FileChannel
 - auto-save temporary data if it uses too much memory,
 -- but revert it on startup if needed.
--- but revert it on startup if needed.
+- chunk metadata: do not store default values
 - support maps without values (just existence of the key)
 - support maps without keys (counted b-tree features)
 - use a small object cache (StringCache)
 - dump values
-- dump values
 - tool to import / manipulate CSV files (maybe concurrently)
 - map split / merge (fast if no overlap)
 - auto-save if there are too many changes (required for StreamStore)
@@ -97,10 +91,7 @@ TODO:
 - implement a shareded map (in one store, multiple stores)
 -- to support concurrent updates and writes, and very large maps
 - implement an off-heap file system
-- optimize API for Java 7 (diamond operator)
-- use new MVStore.Builder().open();
-- see Google Guice: Generic Type
-- JAXB (java xml binding) new TypeReference<String, String>(){}
+- remove change cursor, or add support for writing to branches
 
 */
 
@@ -120,6 +111,9 @@ public class MVStore {
      */
     static final int BLOCK_SIZE = 4 * 1024;
 
+    private static final int FORMAT_WRITE = 1;
+    private static final int FORMAT_READ = 1;
+
     private final String fileName;
 
     private int pageSize = 6 * 1024;
@@ -374,6 +368,23 @@ public class MVStore {
             return;
         }
         FileUtils.createDirectories(FileUtils.getParent(fileName));
+        if (readOnly) {
+            openFile();
+        } else if (!openFile()) {
+            readOnly = true;
+            openFile();
+        }
+    }
+
+    /**
+     * Try to open the file in read or write mode.
+     *
+     * @return if opening the file was successful, and false if the file could
+     *         not be opened in write mode because the write file format it too
+     *         high (in which case the file can be opened in read-only mode)
+     * @throw IllegalStateException if the file could not be opened at all
+     */
+    private boolean openFile() {
         try {
             log("file open");
             FilePath f = FilePath.get(fileName);
@@ -398,11 +409,24 @@ public class MVStore {
                 creationTime = getTime();
                 fileHeader.put("H", "3");
                 fileHeader.put("blockSize", "" + BLOCK_SIZE);
-                fileHeader.put("format", "1");
+                fileHeader.put("format", "" + FORMAT_WRITE);
                 fileHeader.put("creationTime", "" + creationTime);
                 writeFileHeader();
             } else {
                 readFileHeader();
+                int formatWrite = Integer.parseInt(fileHeader.get("format"));
+                String x = fileHeader.get("formatRead");
+                int formatRead = x == null ? formatWrite : Integer.parseInt(x);
+                if (formatRead > FORMAT_READ) {
+                    throw DataUtils.newIllegalStateException(
+                        "The file format {0} is larger than the supported format {1}",
+                        formatRead, FORMAT_READ);
+                }
+                if (formatWrite > FORMAT_WRITE) {
+                    readOnly = true;
+                    file.close();
+                    return false;
+                }
                 if (rootChunkStart > 0) {
                     readMeta();
                 }
@@ -416,6 +440,7 @@ public class MVStore {
             throw DataUtils.newIllegalStateException(
                     "Could not open file {0}", fileName, e);
         }
+        return true;
     }
 
     private void readMeta() {
@@ -1458,4 +1483,8 @@ public class MVStore {
 
     }
 
+    public boolean isReadOnly() {
+        return readOnly;
+    }
+
 }

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -59,6 +59,9 @@ public class MVTableEngine implements TableEngine {
                 store = STORES.get(storeName);
                 if (store == null) {
                     builder.fileName(storeName + Constants.SUFFIX_MV_FILE);
+                    if (db.isReadOnly()) {
+                        builder.readOnly();
+                    }
                     store = new Store(db, builder.open());
                     STORES.put(storeName, store);
                 } else if (store.db != db) {

h2/src/main/org/h2/security/CipherFactory.java

@@ -21,6 +21,7 @@ import java.security.PrivateKey;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateFactory;
 import java.security.spec.PKCS8EncodedKeySpec;
+import java.util.Arrays;
 import java.util.Properties;
 import javax.net.ServerSocketFactory;
 import javax.net.ssl.SSLServerSocket;
@@ -33,7 +34,6 @@ import org.h2.message.DbException;
 import org.h2.store.fs.FileUtils;
 import org.h2.util.IOUtils;
 import org.h2.util.StringUtils;
-import org.h2.util.Utils;
 
 /**
  * A factory to create new block cipher objects.
@@ -231,7 +231,7 @@ public class CipherFactory {
                 // don't need to overwrite the file if it did not change
                 InputStream fin = FileUtils.newInputStream(fileName);
                 byte[] now = IOUtils.readBytesAndClose(fin, 0);
-                if (now != null && Utils.compareNotNull(data, now) == 0) {
+                if (now != null && Arrays.equals(data, now)) {
                     needWrite = false;
                 }
             }

h2/src/main/org/h2/store/FileStore.java

@@ -10,6 +10,7 @@ import java.io.IOException;
 import java.lang.ref.Reference;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.util.Arrays;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
@@ -17,7 +18,6 @@ import org.h2.message.DbException;
 import org.h2.security.SecureFileStore;
 import org.h2.store.fs.FileUtils;
 import org.h2.util.TempFileDeleter;
-import org.h2.util.Utils;
 
 /**
  * This class is an abstraction of a random access file.
@@ -193,7 +193,7 @@ public class FileStore {
             seek(0);
             byte[] buff = new byte[len];
             readFullyDirect(buff, 0, len);
-            if (Utils.compareNotNull(buff, magic) != 0) {
+            if (!Arrays.equals(buff, magic)) {
                 throw DbException.get(ErrorCode.FILE_VERSION_ERROR_1, name);
             }
             salt = new byte[len];
@@ -201,7 +201,7 @@ public class FileStore {
             initKey(salt);
             // read (maybe) encrypted
             readFully(buff, 0, Constants.FILE_BLOCK_SIZE);
-            if (Utils.compareNotNull(buff, magic) != 0) {
+            if (!Arrays.equals(buff, magic)) {
                 throw DbException.get(ErrorCode.FILE_ENCRYPTION_ERROR_1, name);
             }
         }

h2/src/main/org/h2/util/Utils.java

@@ -204,12 +204,14 @@ public class Utils {
      * first array is smaller than the second array, -1 is returned. If the
      * content or length of the second array is smaller than the first array, 1
      * is returned. If the contents and lengths are the same, 0 is returned.
+     * <p>
+     * This method interprets bytes as signed.
      *
      * @param data1 the first byte array (must not be null)
      * @param data2 the second byte array (must not be null)
      * @return the result of the comparison (-1, 1 or 0)
      */
-    public static int compareNotNull(byte[] data1, byte[] data2) {
+    public static int compareNotNullSigned(byte[] data1, byte[] data2) {
         if (data1 == data2) {
             return 0;
         }
@@ -224,6 +226,33 @@ public class Utils {
         return Integer.signum(data1.length - data2.length);
     }
 
+    /**
+     * Compare the contents of two byte arrays. If the content or length of the
+     * first array is smaller than the second array, -1 is returned. If the
+     * content or length of the second array is smaller than the first array, 1
+     * is returned. If the contents and lengths are the same, 0 is returned.
+     * <p>
+     * This method interprets bytes as unsigned.
+     *
+     * @param data1 the first byte array (must not be null)
+     * @param data2 the second byte array (must not be null)
+     * @return the result of the comparison (-1, 1 or 0)
+     */
+    public static int compareNotNull(byte[] data1, byte[] data2) {
+        if (data1 == data2) {
+            return 0;
+        }
+        int len = Math.min(data1.length, data2.length);
+        for (int i = 0; i < len; i++) {
+            int b = data1[i] & 255;
+            int b2 = data2[i] & 255;
+            if (b != b2) {
+                return b > b2 ? 1 : -1;
+            }
+        }
+        return Integer.signum(data1.length - data2.length);
+    }
+
     /**
      * Copy the contents of the source array to the target array. If the size if
      * the target array is too small, a larger array is created.

h2/src/main/org/h2/value/ValueBytes.java

@@ -8,6 +8,7 @@ package org.h2.value;
 
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.Arrays;
 import org.h2.constant.SysProperties;
 import org.h2.util.MathUtils;
 import org.h2.util.StringUtils;
@@ -86,7 +87,7 @@ public class ValueBytes extends Value {
 
     protected int compareSecure(Value v, CompareMode mode) {
         byte[] v2 = ((ValueBytes) v).value;
-        return Utils.compareNotNull(value, v2);
+        return Utils.compareNotNullSigned(value, v2);
     }
 
     public String getString() {
@@ -121,7 +122,7 @@ public class ValueBytes extends Value {
     }
 
     public boolean equals(Object other) {
-        return other instanceof ValueBytes && Utils.compareNotNull(value, ((ValueBytes) other).value) == 0;
+        return other instanceof ValueBytes && Arrays.equals(value, ((ValueBytes) other).value);
     }
 
     public Value convertPrecision(long precision, boolean force) {

h2/src/main/org/h2/value/ValueJavaObject.java

@@ -122,7 +122,7 @@ public class ValueJavaObject extends ValueBytes {
                     return 0;
                 }
 
-                return Utils.compareNotNull(getBytesNoCopy(), v.getBytesNoCopy());
+                return Utils.compareNotNullSigned(getBytesNoCopy(), v.getBytesNoCopy());
             }
 
             return h1 > h2 ? 1 : -1;

h2/src/main/org/h2/value/ValueLob.java

@@ -606,7 +606,7 @@ public class ValueLob extends Value {
             return Integer.signum(getString().compareTo(v.getString()));
         }
         byte[] v2 = v.getBytesNoCopy();
-        return Utils.compareNotNull(getBytes(), v2);
+        return Utils.compareNotNullSigned(getBytes(), v2);
     }
 
     public Object getObject() {

h2/src/main/org/h2/value/ValueLobDb.java

@@ -265,7 +265,7 @@ public class ValueLobDb extends Value implements Value.ValueClob, Value.ValueBlo
             return Integer.signum(getString().compareTo(v.getString()));
         }
         byte[] v2 = v.getBytesNoCopy();
-        return Utils.compareNotNull(getBytes(), v2);
+        return Utils.compareNotNullSigned(getBytes(), v2);
     }
 
     public Object getObject() {

h2/src/test/org/h2/test/jdbcx/SimpleXid.java

@@ -6,10 +6,10 @@
  */
 package org.h2.test.jdbcx;
 
+import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicInteger;
 import javax.transaction.xa.Xid;
 import org.h2.util.MathUtils;
-import org.h2.util.Utils;
 
 /**
  * A simple Xid implementation.
@@ -62,8 +62,8 @@ public class SimpleXid implements Xid {
         if (other instanceof Xid) {
             Xid xid = (Xid) other;
             if (xid.getFormatId() == formatId) {
-                if (Utils.compareNotNull(branchQualifier, xid.getBranchQualifier()) == 0) {
-                    if (Utils.compareNotNull(globalTransactionId, xid.getGlobalTransactionId()) == 0) {
+                if (Arrays.equals(branchQualifier, xid.getBranchQualifier())) {
+                    if (Arrays.equals(globalTransactionId, xid.getGlobalTransactionId())) {
                         return true;
                     }
                 }

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -40,6 +40,7 @@ public class TestMVStore extends TestBase {
 
     public void test() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
+        testFileFormatChange();
         testRecreateMap();
         testRenameMapRollback();
         testCustomMapType();
@@ -73,6 +74,29 @@ public class TestMVStore extends TestBase {
         testSimple();
     }
 
+    private void testFileFormatChange() {
+        String fileName = getBaseDir() + "/testFileFormatChange.h3";
+        FileUtils.delete(fileName);
+        MVStore s;
+        MVMap<Integer, Integer> m;
+        s = openStore(fileName);
+        m = s.openMap("test");
+        m.put(1, 1);
+        Map<String, String> header = s.getFileHeader();
+        int format = Integer.parseInt(header.get("format"));
+        assertEquals(1, format);
+        header.put("format", Integer.toString(format + 1));
+        s.store();
+        s.close();
+        try {
+            openStore(fileName).close();
+            fail();
+        } catch (IllegalStateException e) {
+            assertTrue(e.getCause() != null);
+        }
+        FileUtils.delete(fileName);
+    }
+
     private void testRecreateMap() {
         String fileName = getBaseDir() + "/testRecreateMap.h3";
         FileUtils.delete(fileName);
@@ -168,6 +192,10 @@ public class TestMVStore extends TestBase {
         } catch (IllegalStateException e) {
             // expected
         }
+        assertFalse(s.isReadOnly());
+        s.close();
+        s = new MVStore.Builder().fileName(fileName).readOnly().open();
+        assertTrue(s.isReadOnly());
         s.close();
     }
 

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -33,12 +33,32 @@ public class TestMVTableEngine extends TestBase {
     }
 
     public void test() throws Exception {
+        testReadOnly();
         testReuseDiskSpace();
         testDataTypes();
         testLocking();
         testSimple();
     }
 
+    private void testReadOnly() throws Exception {
+        FileUtils.deleteRecursive(getBaseDir(), true);
+        String dbName = "mvstore" +
+                ";DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
+        Connection conn;
+        Statement stat;
+        conn = getConnection(dbName);
+        stat = conn.createStatement();
+        stat.execute("create table test(id int)");
+        conn.close();
+        FileUtils.setReadOnly(getBaseDir() + "/mvstore.h2.db");
+        conn = getConnection(dbName);
+        for (MVTableEngine.Store s : MVTableEngine.getStores()) {
+            assertTrue(s.getStore().isReadOnly());
+        }
+        conn.close();
+        FileUtils.deleteRecursive(getBaseDir(), true);
+    }
+
     private void testReuseDiskSpace() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
         String dbName = "mvstore" +

h2/src/test/org/h2/test/store/TestObjectDataType.java

@@ -69,19 +69,34 @@ public class TestObjectDataType extends TestBase {
                 new UUID(Long.MIN_VALUE, Long.MIN_VALUE),
                 new UUID(Long.MIN_VALUE, 0), new UUID(0, 0),
                 new UUID(Long.MAX_VALUE, Long.MAX_VALUE),
+                new java.util.Date(0), new java.util.Date(1000),
+                new java.util.Date(4000), new java.util.Date(5000),
+                new boolean[0], new boolean[] { false, false },
+                new boolean[] { true },
                 new byte[0], new byte[1], new byte[15], new byte[16],
                 new byte[10000], new byte[] { (byte) 1 },
+                new byte[] { (byte) 0xff },
+                new short[0], new short[] { -1 }, new short[] { 1 },
+                new char[0], new char[1], new char[10000],
+                new char[] { (char) 1 },
                 new int[0], new int[1], new int[15], new int[16],
                 new int[10000], new int[] { (byte) 1 },
                 new long[0], new long[1], new long[15], new long[16],
                 new long[10000], new long[] { (byte) 1 },
-                new char[0], new char[1], new char[10000], new char[] { (char) 1 },
-                new java.util.Date(0), new java.util.Date(1000),
-                new Timestamp(2000), new Timestamp(3000),
-                new java.util.Date(4000), new java.util.Date(5000),
-                new Object[0], new Object[] { 1 },
+                new float[0], new float[]{Float.NEGATIVE_INFINITY},
+                new float[1], new float[]{Float.POSITIVE_INFINITY},
+                new double[0], new double[]{Double.NEGATIVE_INFINITY},
+                new double[1], new double[]{Double.POSITIVE_INFINITY},
+                new Object[0],
+                new Object[100],
+                new Object[] { 1 },
                 new Object[] { 0.0, "Hello", null, Double.NaN },
-                new Object[100]
+                new String[] { "Hello", null },
+                new String[] { "World" },
+                new java.sql.Date[] { },
+                new Timestamp[] { },
+                new Timestamp[] { null },
+                new Timestamp(2000), new Timestamp(3000),
             };
         Object otherType = false;
         Object last = null;
@@ -91,7 +106,8 @@ public class TestObjectDataType extends TestBase {
                 int comp = ot.compare(x, last);
                 if (comp <= 0) {
                     ot.compare(x, last);
-                    fail(x.getClass().getName() + ": " + x.toString() + " " + comp);
+                    fail(x.getClass().getSimpleName() + ": " +
+                            x.toString() + " " + comp);
                 }
                 assertTrue(x.toString(), ot.compare(last, x) < 0);
             }
@@ -133,6 +149,14 @@ public class TestObjectDataType extends TestBase {
         if (x.getClass().isArray()) {
             if (x instanceof byte[]) {
                 assertTrue(Arrays.equals((byte[]) x, (byte[]) y));
+            } else if (x instanceof boolean[]) {
+                assertTrue(Arrays.equals((boolean[]) x, (boolean[]) y));
+            } else if (x instanceof short[]) {
+                assertTrue(Arrays.equals((short[]) x, (short[]) y));
+            } else if (x instanceof float[]) {
+                assertTrue(Arrays.equals((float[]) x, (float[]) y));
+            } else if (x instanceof double[]) {
+                assertTrue(Arrays.equals((double[]) x, (double[]) y));
             } else if (x instanceof char[]) {
                 assertTrue(Arrays.equals((char[]) x, (char[]) y));
             } else if (x instanceof int[]) {

h2/src/test/org/h2/test/unit/TestSecurity.java

@@ -9,12 +9,12 @@ package org.h2.test.unit;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.Arrays;
 import org.h2.security.BlockCipher;
 import org.h2.security.CipherFactory;
 import org.h2.security.SHA256;
 import org.h2.test.TestBase;
 import org.h2.util.StringUtils;
-import org.h2.util.Utils;
 
 /**
  * Tests various security primitives.
@@ -142,7 +142,7 @@ public class TestSecurity extends TestBase {
         byte[] enc = new byte[128];
         test.encrypt(enc, 0, 128);
         test.decrypt(enc, 0, 128);
-        if (Utils.compareNotNull(in, enc) != 0) {
+        if (!Arrays.equals(in, enc)) {
             throw new AssertionError();
         }
 

h2/src/tools/org/h2/build/code/CheckTextFiles.java

@@ -9,8 +9,7 @@ package org.h2.build.code;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.RandomAccessFile;
-
-import org.h2.util.Utils;
+import java.util.Arrays;
 
 /**
  * This tool checks that source code files only contain the allowed set of
@@ -239,7 +238,7 @@ public class CheckTextFiles {
         }
         if (fix) {
             byte[] changed = out.toByteArray();
-            if (Utils.compareNotNull(data, changed) != 0) {
+            if (!Arrays.equals(data, changed)) {
                 RandomAccessFile f = new RandomAccessFile(file, "rw");
                 f.write(changed);
                 f.setLength(changed.length);

h2/src/tools/org/h2/dev/fs/FilePathCrypt.java

@@ -124,7 +124,6 @@ class FileCrypt extends FileBase {
     static final int BLOCK_SIZE = Constants.FILE_BLOCK_SIZE;
 
     // TODO improve the header
-    // TODO store the number of empty blocks in the last block
     private static final byte[] HEADER = "-- H2 crypt --\n\0".getBytes();
     private static final int SALT_POS = HEADER.length;
     private static final int SALT_LENGTH = 16;