A persistent multi-version map: a utility to store and read streams

h2/src/test/org/h2/test/TestAll.java

@@ -111,6 +111,7 @@ import org.h2.test.store.TestDataUtils;
 import org.h2.test.store.TestMVStore;
 import org.h2.test.store.TestMVRTree;
 import org.h2.test.store.TestObjectType;
+import org.h2.test.store.TestStreamStore;
 import org.h2.test.synth.TestBtreeIndex;
 import org.h2.test.synth.TestCrashAPI;
 import org.h2.test.synth.TestDiskFull;
@@ -677,6 +678,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestMVRTree().runTest(this);
         new TestMVStore().runTest(this);
         new TestObjectType().runTest(this);
+        new TestStreamStore().runTest(this);
 
         // unit
         new TestAutoReconnect().runTest(this);

h2/src/test/org/h2/test/store/TestDataUtils.java

@@ -5,6 +5,8 @@
  */
 package org.h2.test.store;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.HashMap;
 import org.h2.dev.store.btree.DataUtils;
@@ -96,13 +98,31 @@ public class TestDataUtils extends TestBase {
         buff.rewind();
         assertEquals(-1, DataUtils.readVarLong(buff));
         assertEquals(10, buff.position());
+
+        buff.clear();
+        testVarIntVarLong(buff, DataUtils.COMPRESSED_VAR_INT_MAX);
+        testVarIntVarLong(buff, DataUtils.COMPRESSED_VAR_INT_MAX + 1);
+        testVarIntVarLong(buff, DataUtils.COMPRESSED_VAR_LONG_MAX);
+        testVarIntVarLong(buff, DataUtils.COMPRESSED_VAR_LONG_MAX + 1);
     }
 
     private void testVarIntVarLong(ByteBuffer buff, long x) {
         int len;
-
+        byte[] data;
+        try {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            DataUtils.writeVarLong(out, x);
+            data = out.toByteArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
         DataUtils.writeVarLong(buff, x);
         len = buff.position();
+        assertEquals(data.length, len);
+        byte[] data2 = new byte[len];
+        buff.position(0);
+        buff.get(data2);
+        assertEquals(data2, data);
         buff.flip();
         long y = DataUtils.readVarLong(buff);
         assertEquals(y, x);
@@ -111,8 +131,20 @@ public class TestDataUtils extends TestBase {
         buff.clear();
 
         int intX = (int) x;
+        try {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            DataUtils.writeVarInt(out, intX);
+            data = out.toByteArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
         DataUtils.writeVarInt(buff, intX);
         len = buff.position();
+        assertEquals(data.length, len);
+        data2 = new byte[len];
+        buff.position(0);
+        buff.get(data2);
+        assertEquals(data2, data);
         buff.flip();
         int intY = DataUtils.readVarInt(buff);
         assertEquals(intY, intX);

h2/src/test/org/h2/test/store/TestStreamStore.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.store;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.Random;
+import org.h2.dev.store.btree.StreamStore;
+import org.h2.test.TestBase;
+import org.h2.util.IOUtils;
+import org.h2.util.New;
+
+/**
+ * Test the stream store.
+ */
+public class TestStreamStore extends TestBase {
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+    @Override
+    public void test() throws IOException {
+        testWithExistingData();
+        testSmall();
+    }
+
+    private void testWithExistingData() throws IOException {
+        Map<Long, byte[]> map = New.hashMap();
+        StreamStore store = new StreamStore(map);
+        store.setMinBlockSize(10);
+        store.setMaxBlockSize(20);
+        store.setNextKey(0);
+        for (int i = 0; i < 10; i++) {
+            store.put(new ByteArrayInputStream(new byte[20]));
+        }
+        assertEquals(10, map.size());
+        for (int i = 0; i < 10; i++) {
+            map.containsKey(i);
+        }
+        store = new StreamStore(map);
+        store.setMinBlockSize(10);
+        store.setMaxBlockSize(20);
+        store.setNextKey(0);
+        for (int i = 0; i < 10; i++) {
+            store.put(new ByteArrayInputStream(new byte[20]));
+        }
+        assertEquals(20, map.size());
+        for (int i = 0; i < 20; i++) {
+            map.containsKey(i);
+        }
+    }
+
+    private void testSmall() throws IOException {
+        Map<Long, byte[]> map = New.hashMap();
+        StreamStore store = new StreamStore(map);
+        assertEquals(256 * 1024, store.getMaxBlockSize());
+        assertEquals(256, store.getMinBlockSize());
+        store.setNextKey(0);
+        for (int i = 0; i < 20; i++) {
+            test(store, 0, 128, i);
+            test(store, 10, 128, i);
+        }
+        for (int i = 20; i < 200; i += 10) {
+            test(store, 0, 128, i);
+            test(store, 10, 128, i);
+        }
+        test(store, 10, 20, 1000);
+    }
+
+    private void test(StreamStore store, int minBlockSize, int maxBlockSize, int length) throws IOException {
+        store.setMinBlockSize(minBlockSize);
+        assertEquals(minBlockSize, store.getMinBlockSize());
+        store.setMaxBlockSize(maxBlockSize);
+        assertEquals(maxBlockSize, store.getMaxBlockSize());
+        long next = store.getNextKey();
+        Random r = new Random(length);
+        byte[] data = new byte[length];
+        r.nextBytes(data);
+        byte[] id = store.put(new ByteArrayInputStream(data));
+        if (length > 0 && length >= minBlockSize) {
+            assertFalse(store.isInPlace(id));
+        } else {
+            assertTrue(store.isInPlace(id));
+        }
+        long next2 = store.getNextKey();
+        if (length > 0 && length >= minBlockSize) {
+            assertTrue(next2 > next);
+        } else {
+            assertEquals(next, next2);
+        }
+        if (length == 0) {
+            assertEquals(0, id.length);
+        }
+
+        assertEquals(length, store.length(id));
+
+        InputStream in = store.get(id);
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.copy(in, out);
+        assertEquals(data, out.toByteArray());
+
+        in = store.get(id);
+        assertEquals(0, in.skip(0));
+        if (length > 0) {
+            assertEquals(1, in.skip(1));
+            if (length > 1) {
+                assertEquals(data[1] & 255, in.read());
+                if (length > 2) {
+                    assertEquals(1, in.skip(1));
+                    if (length > 3) {
+                        assertEquals(data[3] & 255, in.read());
+                    }
+                } else {
+                    assertEquals(0, in.skip(1));
+                }
+            } else {
+                assertEquals(-1, in.read());
+            }
+        } else {
+            assertEquals(0, in.skip(1));
+        }
+
+        if (length > 12) {
+            in = store.get(id);
+            assertEquals(12, in.skip(12));
+            assertEquals(data[12] & 255, in.read());
+        }
+
+        store.remove(id);
+        assertEquals(0, store.getMap().size());
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/DataUtils.java

@@ -7,6 +7,7 @@
 package org.h2.dev.store.btree;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.HashMap;
@@ -150,7 +151,21 @@ public class DataUtils {
     /**
      * Write a variable size int.
      *
-     * @param buff the target buffer
+     * @param out the output stream
+     * @param x the value
+     */
+    public static void writeVarInt(OutputStream out, int x) throws IOException {
+        while ((x & ~0x7f) != 0) {
+            out.write((byte) (0x80 | (x & 0x7f)));
+            x >>>= 7;
+        }
+        out.write((byte) x);
+    }
+
+    /**
+     * Write a variable size int.
+     *
+     * @param buff the source buffer
      * @param x the value
      */
     public static void writeVarInt(ByteBuffer buff, int x) {
@@ -207,7 +222,7 @@ public class DataUtils {
     }
 
     /**
-     * Write a variable size int.
+     * Write a variable size long.
      *
      * @param buff the target buffer
      * @param x the value
@@ -220,6 +235,20 @@ public class DataUtils {
         buff.put((byte) x);
     }
 
+    /**
+     * Write a variable size long.
+     *
+     * @param out the output stream
+     * @param x the value
+     */
+    public static void writeVarLong(OutputStream out, long x) throws IOException {
+        while ((x & ~0x7f) != 0) {
+            out.write((byte) (0x80 | (x & 0x7f)));
+            x >>>= 7;
+        }
+        out.write((byte) x);
+    }
+
     /**
      * Copy the elements of an array, with a gap.
      *

h2/src/tools/org/h2/dev/store/btree/MVStore.java

@@ -35,10 +35,8 @@ header:
 H:3,blockSize=4096,...
 
 TODO:
-- how to iterate (just) over deleted pages / entries
 - compact: use total max length instead of page count (liveCount)
 - support background writes (store old version)
-- support large binaries
 - support database version / app version
 - limited support for writing to old versions (branches)
 - atomic test-and-set (when supporting concurrent writes)
@@ -62,6 +60,9 @@ TODO:
 - unified exception handling
 - check if locale specific string comparison can make data disappear
 - concurrent map; avoid locking during IO (pre-load pages)
+- maybe split database into multiple files, to speed up compact operation
+- automated 'kill process' and 'power failure' test
+- implement table engine for H2
 
 */