A persistent multi-version map (work in progress) - remove should return the value

h2/src/test/org/h2/test/TestAll.java

@@ -106,7 +106,7 @@ import org.h2.test.server.TestInit;
 import org.h2.test.store.TestCacheLIRS;
 import org.h2.test.store.TestDataUtils;
 import org.h2.test.store.TestMVStore;
-import org.h2.test.store.TestRtree;
+import org.h2.test.store.TestMVRTree;
 import org.h2.test.synth.TestBtreeIndex;
 import org.h2.test.synth.TestCrashAPI;
 import org.h2.test.synth.TestDiskFull;
@@ -668,7 +668,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestMVStore().runTest(this);
         new TestCacheLIRS().runTest(this);
         new TestDataUtils().runTest(this);
-        new TestRtree().runTest(this);
+        new TestMVRTree().runTest(this);
 
         // unit
         new TestAutoReconnect().runTest(this);

h2/src/test/org/h2/test/store/RtreeMap.java → h2/src/test/org/h2/test/store/MVRTreeMap.java

@@ -21,12 +21,12 @@ import org.h2.util.New;
  * @param <K> the key class
  * @param <V> the value class
  */
-public class RtreeMap<K, V> extends MVMap<K, V> {
+public class MVRTreeMap<K, V> extends MVMap<K, V> {
 
     private final SpatialType keyType;
     private boolean quadraticSplit;
 
-    RtreeMap(MVStore store, int id, String name, DataType keyType,
+    MVRTreeMap(MVStore store, int id, String name, DataType keyType,
             DataType valueType, long createVersion) {
         super(store, id, name, keyType, valueType, createVersion);
         this.keyType = (SpatialType) keyType;
@@ -99,46 +99,47 @@ public class RtreeMap<K, V> extends MVMap<K, V> {
         return null;
     }
 
-    protected Page remove(Page p, long writeVersion, Object key) {
-        if (!p.isLeaf()) {
+    protected Object remove(Page p, long writeVersion, Object key) {
+        Object result = null;
+        if (p.isLeaf()) {
             for (int i = 0; i < p.getKeyCount(); i++) {
-                if (contains(p, i, key)) {
-                    Page c = p.getChildPage(i);
-                    long oldSize = c.getTotalCount();
-                    Page c2 = remove(c, writeVersion, key);
-                    if (c2 == null) {
-                        // this child was deleted
-                        if (p.getKeyCount() == 1) {
-                            removePage(p);
-                            return null;
-                        }
-                        p = p.copyOnWrite(writeVersion);
-                        p.remove(i);
-                    } else if (oldSize != c2.getTotalCount()) {
-                        p = p.copyOnWrite(writeVersion);
-                        Object oldBounds = p.getKey(i);
-                        if (!keyType.isInside(key, oldBounds)) {
-                            p.setKey(i, getBounds(c));
-                        }
-                        p.setChild(i, c2);
-                        break;
+                if (keyType.equals(p.getKey(i), key)) {
+                    result = p.getValue(i);
+                    p.remove(i);
+                    if (p.getKeyCount() == 0) {
+                        removePage(p);
                     }
+                    break;
                 }
             }
-        } else {
-            for (int i = 0; i < p.getKeyCount(); i++) {
-                if (keyType.equals(p.getKey(i), key)) {
-                    if (p.getKeyCount() == 1) {
+            return result;
+        }
+        for (int i = 0; i < p.getKeyCount(); i++) {
+            if (contains(p, i, key)) {
+                Page cOld = p.getChildPage(i);
+                Page c = cOld.copyOnWrite(writeVersion);
+                long oldSize = c.getTotalCount();
+                result = remove(c, writeVersion, key);
+                if (oldSize == c.getTotalCount()) {
+                    continue;
+                }
+                if (c.getTotalCount() == 0) {
+                    // this child was deleted
+                    p.remove(i);
+                    if (p.getKeyCount() == 0) {
                         removePage(p);
-                        return null;
                     }
-                    p = p.copyOnWrite(writeVersion);
-                    p.remove(i);
                     break;
                 }
+                Object oldBounds = p.getKey(i);
+                if (!keyType.isInside(key, oldBounds)) {
+                    p.setKey(i, getBounds(c));
+                }
+                p.setChild(i, c);
+                break;
             }
         }
-        return p;
+        return result;
     }
 
     private Object getBounds(Page x) {
@@ -149,15 +150,44 @@ public class RtreeMap<K, V> extends MVMap<K, V> {
         return bounds;
     }
 
-    public void put(K key, V data) {
+    public void put(K key, V value) {
+        putOrAdd(key, value, false);
+    }
+
+    public void add(K key, V value) {
+        putOrAdd(key, value, true);
+    }
+
+    public void putOrAdd(K key, V value, boolean alwaysAdd) {
         checkWrite();
-        Page oldRoot = root;
-        if (get(key) != null) {
-            root = set(root, store.getCurrentVersion(), key, data);
+        long writeVersion = store.getCurrentVersion();
+        Page p = root;
+        if (p == null) {
+            Object[] keys = { key };
+            Object[] values = { value };
+            p = Page.create(this, writeVersion, 1,
+                    keys, values, null, null, null, 1, 0);
+        } else if (alwaysAdd || get(key) == null) {
+            if (p.getKeyCount() > store.getMaxPageSize()) {
+                // only possible if this is the root, else we would have split earlier
+                // (this requires maxPageSize is fixed)
+                p = p.copyOnWrite(writeVersion);
+                long totalCount = p.getTotalCount();
+                Page split = split(p, writeVersion);
+                Object[] keys = { getBounds(p), getBounds(split) };
+                long[] children = { p.getPos(), split.getPos(), 0 };
+                Page[] childrenPages = { p, split, null };
+                long[] counts = { p.getTotalCount(), split.getTotalCount(), 0 };
+                p = Page.create(this, writeVersion, 2,
+                        keys, null, children, childrenPages, counts,
+                        totalCount, 0);
+                // now p is a node; continues
+            }
+            p = add(p, writeVersion, key, value);
         } else {
-            root = add(root, store.getCurrentVersion(), key, data, store.getMaxPageSize());
+            p = set(p, writeVersion, key, value);
         }
-        markChanged(oldRoot);
+        setRoot(p);
     }
 
     protected Page set(Page p, long writeVersion, Object key, Object value) {
@@ -185,36 +215,8 @@ public class RtreeMap<K, V> extends MVMap<K, V> {
         return p;
     }
 
-    public void add(K key, V data) {
-        checkWrite();
-        Page oldRoot = root;
-        root = add(root, store.getCurrentVersion(), key, data, store.getMaxPageSize());
-        markChanged(oldRoot);
-    }
-
-    protected Page add(Page p, long writeVersion, Object key, Object value, int maxPageSize) {
-        if (p == null) {
-            Object[] keys = { key };
-            Object[] values = { value };
-            p = Page.create(this, writeVersion, 1,
-                    keys, values, null, null, null, 1, 0);
-            return p;
-        }
-        if (p.getKeyCount() > maxPageSize) {
-            // only possible if this is the root, else we would have split earlier
-            // (this requires maxPageSize is fixed)
-            p = p.copyOnWrite(writeVersion);
-            long totalCount = p.getTotalCount();
-            Page split = split(p, writeVersion);
-            Object[] keys = { getBounds(p), getBounds(split) };
-            long[] children = { p.getPos(), split.getPos(), 0 };
-            Page[] childrenPages = { p, split, null };
-            long[] counts = { p.getTotalCount(), split.getTotalCount(), 0 };
-            p = Page.create(this, writeVersion, 2,
-                    keys, null, children, childrenPages, counts,
-                    totalCount, 0);
-            // now p is a node; continues
-        } else if (p.isLeaf()) {
+    protected Page add(Page p, long writeVersion, Object key, Object value) {
+        if (p.isLeaf()) {
             p = p.copyOnWrite(writeVersion);
             p.insertLeaf(p.getKeyCount(), key, value);
             return p;
@@ -240,7 +242,7 @@ public class RtreeMap<K, V> extends MVMap<K, V> {
             }
         }
         Page c = p.getChildPage(index);
-        if (c.getKeyCount() >= maxPageSize) {
+        if (c.getKeyCount() >= store.getMaxPageSize()) {
             // split on the way down
             c = c.copyOnWrite(writeVersion);
             Page split = split(c, writeVersion);
@@ -249,9 +251,9 @@ public class RtreeMap<K, V> extends MVMap<K, V> {
             p.setChild(index, c);
             p.insertNode(index, getBounds(split), split);
             // now we are not sure where to add
-            return add(p, writeVersion, key, value, maxPageSize);
+            return add(p, writeVersion, key, value);
         }
-        Page c2 = add(c, writeVersion, key, value, maxPageSize);
+        Page c2 = add(c, writeVersion, key, value);
         p = p.copyOnWrite(writeVersion);
         Object bounds = p.getKey(index);
         keyType.increaseBounds(bounds, key);

h2/src/test/org/h2/test/store/TestRtree.java → h2/src/test/org/h2/test/store/TestMVRTree.java

@@ -22,7 +22,7 @@ import org.h2.util.New;
 /**
  * Tests the r-tree.
  */
-public class TestRtree extends TestMVStore {
+public class TestMVRTree extends TestMVStore {
 
     /**
      * Run just this test.
@@ -46,7 +46,7 @@ public class TestRtree extends TestMVStore {
         MVStore s;
         s = openStore(fileName);
         // s.setMaxPageSize(50);
-        RtreeMap<SpatialKey, String> r = s.openMap("data", "r", "s2", "");
+        MVRTreeMap<SpatialKey, String> r = s.openMap("data", "r", "s2", "");
         // r.setQuadraticSplit(true);
         Random rand = new Random(1);
         int len = 1000;
@@ -108,7 +108,7 @@ public class TestRtree extends TestMVStore {
         FileUtils.delete(fileName);
         MVStore s;
         s = openStore(fileName);
-        RtreeMap<SpatialKey, String> r = s.openMap("data", "r", "s2", "");
+        MVRTreeMap<SpatialKey, String> r = s.openMap("data", "r", "s2", "");
         add(r, "Bern", 46.57, 7.27, 124381);
         add(r, "Basel", 47.34, 7.36, 170903);
         add(r, "Zurich", 47.22, 8.33, 376008);
@@ -134,7 +134,7 @@ public class TestRtree extends TestMVStore {
         s.close();
     }
 
-    private static void add(RtreeMap<SpatialKey, String> r, String name, double y, double x, int population) {
+    private static void add(MVRTreeMap<SpatialKey, String> r, String name, double y, double x, int population) {
         int id = r.size();
         float a = (float) ((int) x + (x - (int) x) * 5 / 3);
         float b = 50 - (float) ((int) y + (y - (int) y) * 5 / 3);
@@ -143,7 +143,7 @@ public class TestRtree extends TestMVStore {
         r.put(k, name);
     }
 
-    private static void render(RtreeMap<SpatialKey, String> r, String fileName) {
+    private static void render(MVRTreeMap<SpatialKey, String> r, String fileName) {
         int width = 1000, height = 500;
         BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
         Graphics2D g2d = (Graphics2D) img.getGraphics();
@@ -199,7 +199,7 @@ public class TestRtree extends TestMVStore {
         String fileName = getBaseDir() + "/testRtreeRandom.h3";
         FileUtils.delete(fileName);
         MVStore s = openStore(fileName);
-        RtreeMap<SpatialKey, String> m = s.openMap("data", "r", "s2", "");
+        MVRTreeMap<SpatialKey, String> m = s.openMap("data", "r", "s2", "");
         HashMap<SpatialKey, String> map = new HashMap<SpatialKey, String>();
         Random rand = new Random(1);
         int operationCount = 1000;

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -113,7 +113,7 @@ public class TestMVStore extends TestBase {
         assertEquals("World", mOld.get("2"));
 
         m.put("1",  "Hi");
-        m.remove("2");
+        assertEquals("Welt", m.remove("2"));
         s.store();
         s.close();
 
@@ -351,10 +351,11 @@ public class TestMVStore extends TestBase {
             // System.out.println("get: " + (System.currentTimeMillis() - t));
             // t = System.currentTimeMillis();
             for (int i = 0; i < len; i++) {
-                m.remove(i);
+                assertEquals("Hello World", m.remove(i));
             }
             // System.out.println("remove: " + (System.currentTimeMillis() - t));
             // System.out.println();
+            assertEquals(null, m.get(0));
             assertEquals(0, m.size());
             s.close();
         }
@@ -414,7 +415,7 @@ public class TestMVStore extends TestBase {
         s.store();
         // System.out.println("store: " + (System.currentTimeMillis() - t));
         // System.out.println(p.getTop(5));
-        m.remove(0);
+        assertEquals("hello 0", m.remove(0));
         assertNull(m.get(0));
         for (int i = 1; i < count; i++) {
             assertEquals("hello " + i, m.get(i));
@@ -492,7 +493,7 @@ public class TestMVStore extends TestBase {
             s.store();
             for (int i = 0; i < 10; i++) {
                 assertEquals("Hello", m.get(i));
-                m.remove(i);
+                assertEquals("Hello", m.remove(i));
             }
             s.store();
             s.close();
@@ -527,8 +528,13 @@ public class TestMVStore extends TestBase {
                 break;
             case 1:
                 log(i + ": remove " + k);
-                m.remove(k);
-                map.remove(k);
+                Integer expected = map.remove(k);
+                Integer got = m.remove(k);
+                if (expected == null) {
+                    assertNull(got);
+                } else {
+                    assertEquals(expected, got);
+                }
                 compareAll = true;
                 break;
             default:
@@ -637,7 +643,7 @@ public class TestMVStore extends TestBase {
             m.put(i, "hello " + i);
         }
         s.store();
-        m.remove(0);
+        assertEquals("hello 0", m.remove(0));
 
         assertNull(m.get(0));
         for (int i = 1; i < 3; i++) {

h2/src/test/org/h2/test/store/TestMapFactory.java

@@ -23,7 +23,7 @@ public class TestMapFactory implements MapFactory {
         if (mapType.equals("s")) {
             return new SequenceMap<K, V>(store, id, name, keyType, valueType, createVersion);
         } else if (mapType.equals("r")) {
-            return new RtreeMap<K, V>(store, id, name, keyType, valueType, createVersion);
+            return new MVRTreeMap<K, V>(store, id, name, keyType, valueType, createVersion);
         } else {
             throw new RuntimeException("Unsupported map type " + mapType);
         }

h2/src/tools/org/h2/dev/store/btree/MVMap.java

@@ -50,13 +50,35 @@ public class MVMap<K, V> {
      * Store a key-value pair.
      *
      * @param key the key
-     * @param data the value
+     * @param value the value
      */
-    public void put(K key, V data) {
+    public void put(K key, V value) {
         checkWrite();
-        Page oldRoot = root;
-        root = put(oldRoot, store.getCurrentVersion(), key, data, store.getMaxPageSize());
-        markChanged(oldRoot);
+        long writeVersion = store.getCurrentVersion();
+        Page p = root;
+        if (p == null) {
+            Object[] keys = { key };
+            Object[] values = { value };
+            p = Page.create(this, writeVersion, 1,
+                    keys, values, null, null, null, 1, 0);
+        } else {
+            p = p.copyOnWrite(writeVersion);
+            if (p.getKeyCount() > store.getMaxPageSize()) {
+                int at = p.getKeyCount() / 2;
+                long totalCount = p.getTotalCount();
+                Object k = p.getKey(at);
+                Page split = p.split(at);
+                Object[] keys = { k };
+                long[] children = { p.getPos(), split.getPos() };
+                Page[] childrenPages = { p, split };
+                long[] counts = { p.getTotalCount(), split.getTotalCount() };
+                p = Page.create(this, writeVersion, 1,
+                        keys, null, children, childrenPages, counts, totalCount, 0);
+                // now p is a node; insert continues
+            }
+            put(p, writeVersion, key, value);
+        }
+        setRoot(p);
     }
 
     /**
@@ -67,42 +89,17 @@ public class MVMap<K, V> {
      * @param writeVersion the write version
      * @param key the key
      * @param value the value (may not be null)
-     * @param maxPageSize the maximum page size
-     * @return the root page
      */
-    protected Page put(Page p, long writeVersion, Object key, Object value, int maxPageSize) {
-        if (p == null) {
-            Object[] keys = { key };
-            Object[] values = { value };
-            p = Page.create(this, writeVersion, 1,
-                    keys, values, null, null, null, 1, 0);
-            return p;
-        }
-        if (p.getKeyCount() > maxPageSize) {
-            // only possible if this is the root, else we would have split earlier
-            // (this requires maxPageSize is fixed)
-            p = p.copyOnWrite(writeVersion);
-            int at = p.getKeyCount() / 2;
-            long totalCount = p.getTotalCount();
-            Object k = p.getKey(at);
-            Page split = p.split(at);
-            Object[] keys = { k };
-            long[] children = { p.getPos(), split.getPos() };
-            Page[] childrenPages = { p, split };
-            long[] counts = { p.getTotalCount(), split.getTotalCount() };
-            p = Page.create(this, writeVersion, 1,
-                    keys, null, children, childrenPages, counts, totalCount, 0);
-            // now p is a node; insert continues
-        } else if (p.isLeaf()) {
+    protected void put(Page p, long writeVersion, Object key, Object value) {
+        if (p.isLeaf()) {
             int index = p.binarySearch(key);
-            p = p.copyOnWrite(writeVersion);
             if (index < 0) {
                 index = -index - 1;
                 p.insertLeaf(index, key, value);
             } else {
                 p.setValue(index, value);
             }
-            return p;
+            return;
         }
         // p is a node
         int index = p.binarySearch(key);
@@ -111,26 +108,24 @@ public class MVMap<K, V> {
         } else {
             index++;
         }
-        Page c = p.getChildPage(index);
-        if (c.getKeyCount() >= maxPageSize) {
+        Page cOld = p.getChildPage(index);
+        Page c = cOld.copyOnWrite(writeVersion);
+        if (c.getKeyCount() >= store.getMaxPageSize()) {
             // split on the way down
-            c = c.copyOnWrite(writeVersion);
             int at = c.getKeyCount() / 2;
             Object k = c.getKey(at);
             Page split = c.split(at);
-            p = p.copyOnWrite(writeVersion);
             p.setChild(index, split);
             p.insertNode(index, k, c);
             // now we are not sure where to add
-            return put(p, writeVersion, key, value, maxPageSize);
+            put(p, writeVersion, key, value);
+            return;
         }
         long oldSize = c.getTotalCount();
-        Page c2 = put(c, writeVersion, key, value, maxPageSize);
-        if (c != c2 || oldSize != c2.getTotalCount()) {
-            p = p.copyOnWrite(writeVersion);
-            p.setChild(index, c2);
+        put(c, writeVersion, key, value);
+        if (cOld != c || oldSize != c.getTotalCount()) {
+            p.setChild(index, c);
         }
-        return p;
     }
 
     /**
@@ -285,17 +280,15 @@ public class MVMap<K, V> {
     public void clear() {
         checkWrite();
         if (root != null) {
-            Page oldRoot = root;
             root.removeAllRecursive();
-            root = null;
-            markChanged(oldRoot);
+            setRoot(null);
         }
     }
 
     /**
      * Remove all entries, and close the map.
      */
-    public void remove() {
+    public void removeMap() {
         checkWrite();
         if (root != null) {
             root.removeAllRecursive();
@@ -319,14 +312,23 @@ public class MVMap<K, V> {
      * Remove a key-value pair, if the key exists.
      *
      * @param key the key
+     * @return the old value if the key existed
      */
-    public void remove(K key) {
+    public V remove(K key) {
         checkWrite();
-        Page oldRoot = root;
-        if (oldRoot != null) {
-            root = remove(oldRoot, store.getCurrentVersion(), key);
-            markChanged(oldRoot);
+        Page p = root;
+        if (p == null) {
+            return null;
         }
+        long writeVersion = store.getCurrentVersion();
+        p = p.copyOnWrite(writeVersion);
+        @SuppressWarnings("unchecked")
+        V result = (V) remove(p, writeVersion, key);
+        if (p.getTotalCount() == 0) {
+            p = null;
+        }
+        setRoot(p);
+        return result;
     }
 
     /**
@@ -335,20 +337,19 @@ public class MVMap<K, V> {
      * @param p the page (may not be null)
      * @param writeVersion the write version
      * @param key the key
-     * @return the new root page (null if empty)
      */
-    protected Page remove(Page p, long writeVersion, Object key) {
+    protected Object remove(Page p, long writeVersion, Object key) {
         int index = p.binarySearch(key);
+        Object result = null;
         if (p.isLeaf()) {
             if (index >= 0) {
-                if (p.getKeyCount() == 1) {
+                result = p.getValue(index);
+                p.remove(index);
+                if (p.getKeyCount() == 0) {
                     removePage(p);
-                    return null;
                 }
-                p = p.copyOnWrite(writeVersion);
-                p.remove(index);
             }
-            return p;
+            return result;
         }
         // node
         if (index < 0) {
@@ -356,30 +357,35 @@ public class MVMap<K, V> {
         } else {
             index++;
         }
-        Page c = p.getChildPage(index);
+        Page cOld = p.getChildPage(index);
+        Page c = cOld.copyOnWrite(writeVersion);
         long oldCount = c.getTotalCount();
-        Page c2 = remove(c, writeVersion, key);
-        if (c2 == null) {
+        result = remove(c, writeVersion, key);
+        if (oldCount == c.getTotalCount()) {
+            return null;
+        }
+        // TODO merge if the c key count is below the threshold
+        if (c.getTotalCount() == 0) {
             // this child was deleted
-            p = p.copyOnWrite(writeVersion);
-            p.remove(index);
             if (p.getKeyCount() == 0) {
+                p.setChild(index, c);
                 removePage(p);
-                p = p.getChildPage(0);
+            } else {
+                p.remove(index);
             }
-        } else if (oldCount != c2.getTotalCount()) {
-            p = p.copyOnWrite(writeVersion);
-            p.setChild(index, c2);
+        } else {
+            p.setChild(index, c);
         }
-        return p;
+        return result;
     }
 
-    protected void markChanged(Page oldRoot) {
-        if (oldRoot != root) {
+    protected void setRoot(Page newRoot) {
+        if (root != newRoot) {
             long v = store.getCurrentVersion();
             if (!oldRoots.containsKey(v)) {
-                oldRoots.put(v, oldRoot);
+                oldRoots.put(v, root);
             }
+            root = newRoot;
             store.markChanged(this);
         }
     }
@@ -516,7 +522,7 @@ public class MVMap<K, V> {
     void rollbackTo(long version) {
         checkWrite();
         if (version < createVersion) {
-            remove();
+            removeMap();
         } else {
             // iterating in ascending order, and pick the last version -
             // this is not terribly efficient if there are many versions

h2/src/tools/org/h2/dev/store/btree/MVStore.java

@@ -38,6 +38,7 @@ header:
 blockSize=4096
 
 TODO:
+- rename commit to incrementVersion
 - implement complete java.util.Map interface
 - limited support for writing to old versions (branches)
 - atomic test-and-set (when supporting concurrent writes)
@@ -47,6 +48,7 @@ TODO:
 - test with very small chunks, possibly speed up very small transactions
 - compact: use total max length instead of page count (liveCount)
 - check what happens on concurrent reads and 1 write; multiple writes
+- concurrent iterator (when to call commit)
 - support large binaries
 - support stores that span multiple files (chunks stored in other files)
 - triggers
@@ -56,6 +58,7 @@ TODO:
 - r-tree: add missing features (NN search for example)
 - compression: maybe hash table reset speeds up compression
 - avoid using java.util.Properties (it allocates quite a lot of memory)
+- support all objects (using serialization)
 
 */