A persistent multi-version map - generic data type serialization

h2/src/test/org/h2/test/TestAll.java

@@ -108,6 +108,7 @@ import org.h2.test.store.TestConcurrent;
 import org.h2.test.store.TestDataUtils;
 import org.h2.test.store.TestMVStore;
 import org.h2.test.store.TestMVRTree;
+import org.h2.test.store.TestObjectType;
 import org.h2.test.synth.TestBtreeIndex;
 import org.h2.test.synth.TestCrashAPI;
 import org.h2.test.synth.TestDiskFull;
@@ -671,6 +672,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestDataUtils().runTest(this);
         new TestMVRTree().runTest(this);
         new TestMVStore().runTest(this);
+        new TestObjectType().runTest(this);
 
         // unit
         new TestAutoReconnect().runTest(this);

h2/src/test/org/h2/test/store/TestDataUtils.java

@@ -26,6 +26,7 @@ public class TestDataUtils extends TestBase {
 
     public void test() throws Exception {
         testMap();
+        testMaxShortVarIntVarLong();
         testVarIntVarLong();
         testCheckValue();
         testPagePos();
@@ -50,6 +51,22 @@ public class TestDataUtils extends TestBase {
         assertEquals("\"test\"", m.get("d"));
     }
 
+    private void testMaxShortVarIntVarLong() {
+        ByteBuffer buff = ByteBuffer.allocate(100);
+        DataUtils.writeVarInt(buff, DataUtils.COMPRESSED_VAR_INT_MAX);
+        assertEquals(3, buff.position());
+        buff.rewind();
+        DataUtils.writeVarInt(buff, DataUtils.COMPRESSED_VAR_INT_MAX + 1);
+        assertEquals(4, buff.position());
+        buff.rewind();
+        DataUtils.writeVarLong(buff, DataUtils.COMPRESSED_VAR_LONG_MAX);
+        assertEquals(7, buff.position());
+        buff.rewind();
+        DataUtils.writeVarLong(buff, DataUtils.COMPRESSED_VAR_LONG_MAX + 1);
+        assertEquals(8, buff.position());
+        buff.rewind();
+    }
+
     private void testVarIntVarLong() {
         ByteBuffer buff = ByteBuffer.allocate(100);
         for (long x = 0; x < 1000; x++) {

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -12,7 +12,6 @@ import java.util.Random;
 import java.util.TreeMap;
 import org.h2.dev.store.btree.MVMap;
 import org.h2.dev.store.btree.MVStore;
-import org.h2.jaqu.bytecode.Null;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
 import org.h2.util.New;
@@ -32,6 +31,7 @@ public class TestMVStore extends TestBase {
     }
 
     public void test() throws InterruptedException {
+        testObjects();
         testExample();
         testIterateOverChanges();
         testOpenStoreCloseLoop();
@@ -52,8 +52,31 @@ public class TestMVStore extends TestBase {
         testSimple();
     }
 
+    private void testObjects() {
+        String fileName = getBaseDir() + "/testObjects.h3";
+        FileUtils.delete(fileName);
+        MVStore s;
+        Map<Object, Object> map;
+        s = MVStore.open(fileName, new TestMapFactory());
+        map = s.openMap("test");
+        map.put(1,  "Hello");
+        map.put("2", 200);
+        map.put(new Object[1], new Object[]{1, "2"});
+        s.store();
+        s.close();
+        s = MVStore.open(fileName, new TestMapFactory());
+        map = s.openMap("test");
+        assertEquals("Hello", map.get(1).toString());
+        assertEquals(200, ((Integer) map.get("2")).intValue());
+        Object[] x = (Object[]) map.get(new Object[1]);
+        assertEquals(2, x.length);
+        assertEquals(1, ((Integer) x[0]).intValue());
+        assertEquals("2", (String) x[1]);
+        s.close();
+    }
+
     private void testExample() {
-        String fileName = getBaseDir() + "/testOpenClose.h3";
+        String fileName = getBaseDir() + "/testExample.h3";
         FileUtils.delete(fileName);
 
         // open the store (in-memory if fileName is null)
@@ -649,29 +672,34 @@ public class TestMVStore extends TestBase {
     }
 
     private void testKeyValueClasses() {
-        String fileName = getBaseDir() + "/testKeyValueClasses.h3";
-        FileUtils.delete(fileName);
-        MVStore s = openStore(fileName);
-        MVMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
-        is.put(1, "Hello");
-        MVMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
-        ii.put(1, 10);
-        MVMap<String, Integer> si = s.openMap("stringInt", String.class, Integer.class);
-        si.put("Test", 10);
-        MVMap<String, String> ss = s.openMap("stringString", String.class, String.class);
-        ss.put("Hello", "World");
+        MVStore s;
+        s = MVStore.open(null);
+        s.openMap("test", String.class, String.class);
         try {
-            s.openMap("invalid", Null.class, Integer.class);
+            s.openMap("unsupportedKey", ArrayList.class, String.class);
             fail();
         } catch (RuntimeException e) {
             // expected
         }
         try {
-            s.openMap("invalid", Integer.class, Null.class);
+            s.openMap("unsupportedValue", String.class, ArrayList.class);
             fail();
         } catch (RuntimeException e) {
             // expected
         }
+        s.close();
+
+        String fileName = getBaseDir() + "/testKeyValueClasses.h3";
+        FileUtils.delete(fileName);
+        s = openStore(fileName);
+        MVMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
+        is.put(1, "Hello");
+        MVMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
+        ii.put(1, 10);
+        MVMap<String, Integer> si = s.openMap("stringInt", String.class, Integer.class);
+        si.put("Test", 10);
+        MVMap<String, String> ss = s.openMap("stringString", String.class, String.class);
+        ss.put("Hello", "World");
         s.store();
         s.close();
         s = openStore(fileName);

h2/src/test/org/h2/test/store/TestMapFactory.java

@@ -41,6 +41,8 @@ public class TestMapFactory implements MapFactory {
             return RowType.fromString(s, this);
         case 's':
             return SpatialType.fromString(s);
+        case 'o':
+            return new ObjectType();
         }
         throw new RuntimeException("Unknown data type " + s);
     }
@@ -49,6 +51,8 @@ public class TestMapFactory implements MapFactory {
     public String getDataType(Class<?> objectClass) {
         if (objectClass == Integer.class) {
             return "i";
+        } else if (Object.class == Object.class) {
+            return "o";
         }
         throw new RuntimeException("Unsupported object class " + objectClass.toString());
     }

h2/src/test/org/h2/test/store/TestObjectType.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.store;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.UUID;
+import org.h2.test.TestBase;
+
+/**
+ * Test the ObjectType class.
+ */
+public class TestObjectType extends TestBase {
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+    @Override
+    public void test() {
+        testCommonValues();
+    }
+
+    private void testCommonValues() {
+        BigInteger largeBigInt = BigInteger.probablePrime(200,  new Random(1));
+        ObjectType ot = new ObjectType();
+        assertEquals("o", ot.asString());
+        Object[] array = {
+                false, true,
+                Byte.MIN_VALUE, (byte) -1, (byte) 0, (byte) 1, Byte.MAX_VALUE,
+                Short.MIN_VALUE, (short) -1, (short) 0, (short) 1, Short.MAX_VALUE,
+                Integer.MIN_VALUE, Integer.MIN_VALUE + 1,
+                -1000, -100, -1, 0, 1, 2, 14,
+                15, 16, 17, 100, Integer.MAX_VALUE - 1, Integer.MAX_VALUE,
+                Long.MIN_VALUE, Long.MIN_VALUE + 1, -1000L, -1L, 0L, 1L, 2L, 14L,
+                15L, 16L, 17L, 100L, Long.MAX_VALUE - 1, Long.MAX_VALUE,
+                largeBigInt.negate(), BigInteger.valueOf(-1), BigInteger.ZERO,
+                BigInteger.ONE, BigInteger.TEN, largeBigInt,
+                Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, -1f, -0f, 0f,
+                Float.MIN_VALUE, 1f, Float.MAX_VALUE,
+                Float.POSITIVE_INFINITY, Float.NaN,
+                Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, -1d, -0d, 0d,
+                Double.MIN_VALUE, 1d, Double.MAX_VALUE,
+                Double.POSITIVE_INFINITY, Double.NaN,
+                BigDecimal.valueOf(Double.MAX_VALUE).negate(),
+                new BigDecimal(largeBigInt).negate(),
+                BigDecimal.valueOf(-100.0), BigDecimal.ZERO, BigDecimal.ONE,
+                BigDecimal.TEN, BigDecimal.valueOf(Long.MAX_VALUE),
+                new BigDecimal(largeBigInt),
+                BigDecimal.valueOf(Double.MAX_VALUE),
+                Character.MIN_VALUE, '0', 'a', Character.MAX_VALUE,
+                "", " ", "  ", "123456789012345", "1234567890123456",
+                new String(new char[100]).replace((char) 0, 'x'),
+                new String(new char[100000]).replace((char) 0, 'x'), "y",
+                "\u1234", "\u2345", "\u6789", "\uffff",
+                new UUID(Long.MIN_VALUE, Long.MIN_VALUE),
+                new UUID(Long.MIN_VALUE, 0), new UUID(0, 0),
+                new UUID(Long.MAX_VALUE, Long.MAX_VALUE),
+                new byte[0], new byte[1], new byte[15], new byte[16],
+                new byte[10000], new byte[] { (byte) 1 },
+                new int[0], new int[1], new int[15], new int[16],
+                new int[10000], new int[] { (byte) 1 },
+                new long[0], new long[1], new long[15], new long[16],
+                new long[10000], new long[] { (byte) 1 },
+                new char[0], new char[1], new char[10000], new char[] { (char) 1 },
+                new java.util.Date(0), new java.util.Date(1000),
+                new Timestamp(2000), new Timestamp(3000),
+                new java.util.Date(4000), new java.util.Date(5000),
+                new Object[0], new Object[] { 1 },
+                new Object[] { 0.0, "Hello", null, Double.NaN },
+                new Object[100]
+            };
+        Object otherType = false;
+        Object last = null;
+        for (Object x : array) {
+            test(otherType, x);
+            if (last != null) {
+                int comp = ot.compare(x, last);
+                if (comp <= 0) {
+                    ot.compare(x, last);
+                    fail(x.getClass().getName() + ": " + x.toString() + " " + comp);
+                }
+                assertTrue(x.toString(), ot.compare(last, x) < 0);
+            }
+            if (last != null && last.getClass() != x.getClass()) {
+                otherType = last;
+            }
+            last = x;
+        }
+    }
+
+    private void test(Object last, Object x) {
+        ObjectType ot = new ObjectType();
+
+        // switch to the last type before every operation,
+        // to test switching types
+        ot.getMemory(last);
+        assertTrue(ot.getMemory(x) >= 0);
+
+        ot.getMemory(last);
+        assertTrue(ot.getMaxLength(x) >= 1);
+
+        ot.getMemory(last);
+        assertEquals(0, ot.compare(x, x));
+        ByteBuffer buff = ByteBuffer.allocate(ot.getMaxLength(x) + 1);
+
+        ot.getMemory(last);
+        ot.write(buff, x);
+        buff.put((byte) 123);
+        buff.flip();
+
+        ot.getMemory(last);
+        Object y = ot.read(buff);
+        assertEquals(123, buff.get());
+        assertEquals(0, buff.remaining());
+        assertEquals(x.getClass().getName(), y.getClass().getName());
+
+        ot.getMemory(last);
+        assertEquals(0, ot.compare(x,  y));
+        if (x.getClass().isArray()) {
+            if (x instanceof byte[]) {
+                assertTrue(Arrays.equals((byte[]) x, (byte[]) y));
+            } else if (x instanceof char[]) {
+                assertTrue(Arrays.equals((char[]) x, (char[]) y));
+            } else if (x instanceof int[]) {
+                assertTrue(Arrays.equals((int[]) x, (int[]) y));
+            } else if (x instanceof long[]) {
+                assertTrue(Arrays.equals((long[]) x, (long[]) y));
+            } else {
+                assertTrue(Arrays.equals((Object[]) x, (Object[]) y));
+            }
+        } else {
+            assertEquals(x.hashCode(), y.hashCode());
+            assertTrue(x.equals(y));
+        }
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/DataUtils.java

@@ -42,6 +42,18 @@ public class DataUtils {
      */
     public static final int MAX_VAR_LONG_LEN = 10;
 
+    /**
+     * The maximum integer that needs less space when using variable size
+     * encoding (only 3 bytes instead of 4).
+     */
+    public static final int COMPRESSED_VAR_INT_MAX = 0x1fffff;
+
+    /**
+     * The maximum long that needs less space when using variable size
+     * encoding (only 7 bytes instead of 8).
+     */
+    public static final long COMPRESSED_VAR_LONG_MAX = 0x1ffffffffffffL;
+
     /**
      * Get the length of the variable size int.
      *
@@ -149,6 +161,51 @@ public class DataUtils {
         buff.put((byte) x);
     }
 
+    /**
+     * Write characters from a string (without the length).
+     *
+     * @param buff the target buffer
+     * @param s the string
+     * @param len the number of characters
+     */
+    public static void writeStringData(ByteBuffer buff, String s, int len) {
+        for (int i = 0; i < len; i++) {
+            int c = s.charAt(i);
+            if (c < 0x80) {
+                buff.put((byte) c);
+            } else if (c >= 0x800) {
+                buff.put((byte) (0xe0 | (c >> 12)));
+                buff.put((byte) (((c >> 6) & 0x3f)));
+                buff.put((byte) (c & 0x3f));
+            } else {
+                buff.put((byte) (0xc0 | (c >> 6)));
+                buff.put((byte) (c & 0x3f));
+            }
+        }
+    }
+
+    /**
+     * Read a string.
+     *
+     * @param buff the source buffer
+     * @param len the number of characters
+     * @return the value
+     */
+    public static String readString(ByteBuffer buff, int len) {
+        char[] chars = new char[len];
+        for (int i = 0; i < len; i++) {
+            int x = buff.get() & 0xff;
+            if (x < 0x80) {
+                chars[i] = (char) x;
+            } else if (x >= 0xe0) {
+                chars[i] = (char) (((x & 0xf) << 12) + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f));
+            } else {
+                chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f));
+            }
+        }
+        return new String(chars);
+    }
+
     /**
      * Write a variable size int.
      *

h2/src/tools/org/h2/dev/store/btree/MVStore.java

@@ -35,7 +35,6 @@ header:
 H:3,blockSize=4096,...
 
 TODO:
-- support all objects (using serialization)
 - concurrent iterator (when to increment version; read on first hasNext())
 - how to iterate (just) over deleted pages / entries
 - compact: use total max length instead of page count (liveCount)
@@ -60,6 +59,9 @@ TODO:
 - recovery: ensure data is not overwritten for 1 minute
 - pluggable caching (specially for in-memory file systems)
 - file locking
+- allocate memory Utils.newBytes
+- unified exception handling
+- check if locale specific string comparison can make data disappear
 
 */
 
@@ -73,7 +75,10 @@ public class MVStore {
      */
     public static final boolean ASSERT = false;
 
-    private static final StringType STRING_TYPE = new StringType();
+    /**
+     * A string data type.
+     */
+    public static final StringType STRING_TYPE = new StringType();
 
     private final String fileName;
     private final MapFactory mapFactory;
@@ -169,6 +174,41 @@ public class MVStore {
         return (T) m;
     }
 
+    /**
+     * Open a map with the previous key and value type (if the map already
+     * exists), or Object if not.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     * @param name the name of the map
+     * @return the map
+     */
+    public <K, V> MVMap<K, V> openMap(String name) {
+        String keyType = getDataType(Object.class);
+        String valueType = getDataType(Object.class);
+        @SuppressWarnings("unchecked")
+        MVMap<K, V> m = (MVMap<K, V>) openMap(name, "", keyType, valueType);
+        return m;
+    }
+
+    /**
+     * Open a map.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     * @param name the name of the map
+     * @param keyClass the key class
+     * @param valueClass the value class
+     * @return the map
+     */
+    public <K, V> MVMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
+        String keyType = getDataType(keyClass);
+        String valueType = getDataType(valueClass);
+        @SuppressWarnings("unchecked")
+        MVMap<K, V> m = (MVMap<K, V>) openMap(name, "", keyType, valueType);
+        return m;
+    }
+
     /**
      * Open a map.
      *
@@ -187,6 +227,7 @@ public class MVStore {
             int id;
             long root;
             long createVersion;
+            // TODO use the json formatting for map metadata
             if (identifier == null) {
                 id = ++lastMapId;
                 createVersion = currentVersion;
@@ -259,24 +300,6 @@ public class MVStore {
         }
     }
 
-    /**
-     * Open a map.
-     *
-     * @param <K> the key type
-     * @param <V> the value type
-     * @param name the name of the map
-     * @param keyClass the key class
-     * @param valueClass the value class
-     * @return the map
-     */
-    public <K, V> MVMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
-        String keyType = getDataType(keyClass);
-        String valueType = getDataType(valueClass);
-        @SuppressWarnings("unchecked")
-        MVMap<K, V> m = (MVMap<K, V>) openMap(name, "", keyType, valueType);
-        return m;
-    }
-
     /**
      * Remove a map.
      *

h2/src/tools/org/h2/dev/store/btree/MapFactory.java

@@ -7,7 +7,7 @@
 package org.h2.dev.store.btree;
 
 /**
- * A factory for data types.
+ * A factory for maps and data types.
  */
 public interface MapFactory {
 

h2/src/tools/org/h2/dev/store/btree/StringType.java

@@ -18,46 +18,23 @@ public class StringType implements DataType {
     }
 
     public int getMaxLength(Object obj) {
-        return DataUtils.MAX_VAR_INT_LEN + obj.toString().length() * 3;
+        return DataUtils.MAX_VAR_INT_LEN + 3 * obj.toString().length();
     }
 
     public int getMemory(Object obj) {
-        return obj.toString().length() * 2 + 48;
+        return 24 + 2 * obj.toString().length();
     }
 
     public String read(ByteBuffer buff) {
         int len = DataUtils.readVarInt(buff);
-        char[] chars = new char[len];
-        for (int i = 0; i < len; i++) {
-            int x = buff.get() & 0xff;
-            if (x < 0x80) {
-                chars[i] = (char) x;
-            } else if (x >= 0xe0) {
-                chars[i] = (char) (((x & 0xf) << 12) + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f));
-            } else {
-                chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f));
-            }
-        }
-        return new String(chars);
+        return DataUtils.readString(buff, len);
     }
 
     public void write(ByteBuffer buff, Object obj) {
         String s = obj.toString();
         int len = s.length();
         DataUtils.writeVarInt(buff, len);
-        for (int i = 0; i < len; i++) {
-            int c = s.charAt(i);
-            if (c < 0x80) {
-                buff.put((byte) c);
-            } else if (c >= 0x800) {
-                buff.put((byte) (0xe0 | (c >> 12)));
-                buff.put((byte) (((c >> 6) & 0x3f)));
-                buff.put((byte) (c & 0x3f));
-            } else {
-                buff.put((byte) (0xc0 | (c >> 6)));
-                buff.put((byte) (c & 0x3f));
-            }
-        }
+        DataUtils.writeStringData(buff, s, len);
     }
 
     public String asString() {