MVStore: larger stores (multiple GB) are now much faster.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,8 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>When using local temporary tables and not dropping them manually before closing the session,
+<ul><li>MVStore: larger stores (multiple GB) are now much faster.
+</li><li>When using local temporary tables and not dropping them manually before closing the session,
     and then killing the process could result in a database that couldn't be opened (except when using
     the recover tool).
 </li><li>Support TRUNC(timestamp) for improved Oracle compatiblity.

h2/src/main/org/h2/mvstore/ChangeCursor.java

-/*
- * Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
- * Version 1.0, and under the Eclipse Public License, Version 1.0
- * (http://h2database.com/html/license.html).
- * Initial Developer: H2 Group
- */
-package org.h2.mvstore;
-
-import java.util.Iterator;
-
-/**
- * A cursor to iterate over all keys in new pages.
- *
- * @param <K> the key type
- * @param <V> the value type
- */
-public class ChangeCursor<K, V> implements Iterator<K> {
-
-    private final MVMap<K, V> map;
-    private final Page root1, root2;
-
-    /**
-     * The state of this cursor.
-     * 0: not initialized
-     * 1: reading from root1
-     * 2: reading from root2
-     * 3: closed
-     */
-    private int state;
-    private CursorPos pos1, pos2;
-    private K current;
-
-    ChangeCursor(MVMap<K, V> map, Page root1, Page root2) {
-        this.map = map;
-        this.root1 = root1;
-        this.root2 = root2;
-    }
-
-    public K next() {
-        K c = current;
-        fetchNext();
-        return c;
-    }
-
-    public boolean hasNext() {
-        if (state == 0) {
-            pos1 = new CursorPos(root1, 0, null);
-            pos1 = min(pos1);
-            state = 1;
-            fetchNext();
-        }
-        return current != null;
-    }
-
-    public void remove() {
-        throw DataUtils.newUnsupportedOperationException(
-                "Removing is not supported");
-    }
-
-    private void fetchNext() {
-        while (fetchNextKey()) {
-            if (pos1 == null || pos2 == null) {
-                break;
-            }
-            @SuppressWarnings("unchecked")
-            V v1 = (V) map.binarySearch(root1, current);
-            @SuppressWarnings("unchecked")
-            V v2 = (V) map.binarySearch(root2, current);
-            if (!v1.equals(v2)) {
-                break;
-            }
-        }
-    }
-
-    private boolean fetchNextKey() {
-        while (true) {
-            if (state == 3) {
-                return false;
-            }
-            if (state == 1) {
-                // read from root1
-                pos1 = fetchNext(pos1);
-                if (pos1 == null) {
-                    // reached the end of pos1
-                    state = 2;
-                    pos2 = null;
-                    continue;
-                }
-                pos2 = find(root2, current);
-                if (pos2 == null) {
-                    // not found in root2
-                    return true;
-                }
-                if (!pos1.page.equals(pos2.page)) {
-                    // the page is different,
-                    // so the entry has possibly changed
-                    return true;
-                }
-                while (true) {
-                    pos1 = pos1.parent;
-                    if (pos1 == null) {
-                        // reached end of pos1
-                        state = 2;
-                        pos2 = null;
-                        break;
-                    }
-                    pos2 = pos2.parent;
-                    if (pos2 == null || !pos1.page.equals(pos2.page)) {
-                        if (pos1.index + 1 < map.getChildPageCount(pos1.page)) {
-                            pos1 = new CursorPos(pos1.page.getChildPage(++pos1.index), 0, pos1);
-                            pos1 = min(pos1);
-                            break;
-                        }
-                    }
-                }
-            }
-            if (state == 2) {
-                if (pos2 == null) {
-                    // init reading from root2
-                    pos2 = new CursorPos(root2, 0, null);
-                    pos2 = min(pos2);
-                }
-                // read from root2
-                pos2 = fetchNext(pos2);
-                if (pos2 == null) {
-                    // reached the end of pos2
-                    state = 3;
-                    current = null;
-                    continue;
-                }
-                pos1 = find(root1, current);
-                if (pos1 != null) {
-                    // found a corresponding record
-                    // so it was not deleted
-                    // but now we may need to skip pages
-                    if (!pos1.page.equals(pos2.page)) {
-                        // the page is different
-                        pos1 = null;
-                        continue;
-                    }
-                    while (true) {
-                        pos2 = pos2.parent;
-                        if (pos2 == null) {
-                            // reached end of pos1
-                            state = 3;
-                            current = null;
-                            pos1 = null;
-                            break;
-                        }
-                        pos1 = pos1.parent;
-                        if (pos1 == null || !pos2.page.equals(pos1.page)) {
-                            if (pos2.index + 1 < map.getChildPageCount(pos2.page)) {
-                                pos2 = new CursorPos(pos2.page.getChildPage(++pos2.index), 0, pos2);
-                                pos2 = min(pos2);
-                                break;
-                            }
-                        }
-                    }
-                    pos1 = null;
-                    continue;
-                }
-                // found no corresponding record
-                // so it was deleted
-                return true;
-            }
-        }
-    }
-
-    private CursorPos find(Page p, K key) {
-        // TODO combine with RangeCursor.min
-        // possibly move to MVMap
-        CursorPos pos = null;
-        while (true) {
-            if (p.isLeaf()) {
-                int x = key == null ? 0 : p.binarySearch(key);
-                if (x < 0) {
-                    return null;
-                }
-                return new CursorPos(p, x, pos);
-            }
-            int x = key == null ? -1 : p.binarySearch(key);
-            if (x < 0) {
-                x = -x - 1;
-            } else {
-                x++;
-            }
-            pos = new CursorPos(p, x, pos);
-            p = p.getChildPage(x);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private CursorPos fetchNext(CursorPos p) {
-        while (p != null) {
-            if (p.index < p.page.getKeyCount()) {
-                current = (K) p.page.getKey(p.index++);
-                return p;
-            }
-            p = p.parent;
-            if (p == null) {
-                break;
-            }
-            if (p.index + 1 < map.getChildPageCount(p.page)) {
-                p = new CursorPos(p.page.getChildPage(++p.index), 0, p);
-                p = min(p);
-            }
-        }
-        current = null;
-        return p;
-    }
-
-    private static CursorPos min(CursorPos p) {
-        while (true) {
-            if (p.page.isLeaf()) {
-                return p;
-            }
-            Page c = p.page.getChildPage(0);
-            p = new CursorPos(c, 0, p);
-        }
-    }
-
-}

h2/src/main/org/h2/mvstore/Chunk.java

@@ -42,9 +42,14 @@ public class Chunk {
     int length;
 
     /**
-     * The number of pages.
+     * The total number of pages in this chunk.
      */
     int pageCount;
+    
+    /**
+     * The number of pages still alive.
+     */
+    int pageCountLive;
 
     /**
      * The sum of the max length of all pages.
@@ -101,6 +106,7 @@ public class Chunk {
         Chunk c = new Chunk(chunkId);
         c.length = length;
         c.pageCount = pageCount;
+        c.pageCountLive = pageCount;
         c.start = start;
         c.metaRootPos = metaRootPos;
         c.maxLength = maxLength;
@@ -136,6 +142,7 @@ public class Chunk {
         c.start = Long.parseLong(map.get("start"));
         c.length = Integer.parseInt(map.get("length"));
         c.pageCount = Integer.parseInt(map.get("pageCount"));
+        c.pageCountLive = Integer.parseInt(map.get("pageCountLive"));
         c.maxLength = Long.parseLong(map.get("maxLength"));
         c.maxLengthLive = Long.parseLong(map.get("maxLengthLive"));
         c.metaRootPos = Long.parseLong(map.get("metaRoot"));
@@ -169,6 +176,7 @@ public class Chunk {
                 "maxLengthLive:" + maxLengthLive + "," +
                 "metaRoot:" + metaRootPos + "," +
                 "pageCount:" + pageCount + "," +
+                "pageCountLive:" + pageCountLive + "," +
                 "start:" + start + "," +
                 "time:" + time + "," +
                 "version:" + version;

h2/src/main/org/h2/mvstore/FreeSpaceList.java

+/*
+ * Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
 package org.h2.mvstore;
 
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * A list that maintains ranges of free space (in pages) in a file.
+ */
 public class FreeSpaceList {
 
-    /** the first 2 pages are occupied by the file header. */
+    /** 
+     * The first 2 pages are occupied by the file header. 
+     */
     private static final int FIRST_FREE_PAGE = 2;
-    /** I use max_value/2 to avoid overflow errors during arithmetic. */
-    private static final int MAX_NO_PAGES = Integer.MAX_VALUE / 2;
+    
+    /**
+     * The maximum number of pages. Smaller than than MAX_VALUE to avoid
+     * overflow errors during arithmetic operations.
+     */
+    private static final int MAX_PAGE_COUNT = Integer.MAX_VALUE / 2;
 
     private List<PageRange> freeSpaceList = new ArrayList<PageRange>();
 
     public FreeSpaceList() {
-        freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_NO_PAGES));
+        clear();
     }
-
-    private static final class PageRange {
-        /** the start point, in pages */
-        public int start;
-        /** the length, in pages */
-        public int length;
-
-        public PageRange(int start, int length) {
-            this.start = start;
-            this.length = length;
-        }
-
-        @Override
-        public String toString() {
-            return "start:" + start + " length:" + length;
-        }
+    
+    /**
+     * Reset the list.
+     */
+    public synchronized void clear() {
+        freeSpaceList.clear();
+        freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_PAGE_COUNT));
     }
 
     /**
-     * @return position in pages
+     * Allocate a number of pages.
+     * 
+     * @param length the number of bytes to allocate
+     * @return the position in pages
      */
     public synchronized int allocatePages(long length) {
         int required = (int) (length / MVStore.BLOCK_SIZE) + 1;
@@ -43,9 +52,15 @@ public class FreeSpaceList {
                 return pr.start;
             }
         }
-        throw DataUtils.newIllegalStateException("could not find a free page to allocate");
+        throw DataUtils.newIllegalStateException(
+                "Could not find a free page to allocate");
     }
 
+    /**
+     * Mark a chunk as used.
+     * 
+     * @param c the chunk
+     */
     public synchronized void markUsed(Chunk c) {
         int chunkStart = (int) (c.start / MVStore.BLOCK_SIZE);
         int required = (int) ((c.start + c.length) / MVStore.BLOCK_SIZE) + 2 - chunkStart;
@@ -59,11 +74,12 @@ public class FreeSpaceList {
             i++;
         }
         if (found == null) {
-            throw DataUtils.newIllegalStateException("cannot find spot to mark chunk as used in free list, "
-                    + c.toString());
+            throw DataUtils.newIllegalStateException(
+                    "Cannot find spot to mark chunk as used in free list: {0}", c);
         }
         if (chunkStart + required > found.start + found.length) {
-            throw DataUtils.newIllegalStateException("chunk runs over edge of free space, " + c.toString());
+            throw DataUtils.newIllegalStateException(
+                    "Chunk runs over edge of free space: {0}", c);
         }
         if (found.start == chunkStart) {
             // if the used-chunk is at the beginning of a free-space-range
@@ -92,6 +108,11 @@ public class FreeSpaceList {
         }
     }
 
+    /**
+     * Mark the chunk as free.
+     * 
+     * @param c the chunk
+     */
     public synchronized void markFree(Chunk c) {
         int chunkStart = (int) (c.start / MVStore.BLOCK_SIZE);
         int required = (c.length / MVStore.BLOCK_SIZE) + 1;
@@ -105,15 +126,15 @@ public class FreeSpaceList {
             i++;
         }
         if (found == null) {
-            throw DataUtils.newIllegalStateException("cannot find spot to mark chunk as unused in free list, "
-                    + c.toString());
+            throw DataUtils.newIllegalStateException(
+                    "Cannot find spot to mark chunk as unused in free list: {0}", c);
         }
         if (chunkStart + required + 1 == found.start) {
             // if the used-chunk is adjacent to the beginning of a
             // free-space-range
             found.start = chunkStart;
             found.length += required;
-            // compactify-free-list: merge the previous entry into this one if
+            // compact: merge the previous entry into this one if
             // they are now adjacent
             if (i > 0) {
                 PageRange previous = freeSpaceList.get(i - 1);
@@ -129,7 +150,7 @@ public class FreeSpaceList {
             PageRange previous = freeSpaceList.get(i - 1);
             if (previous.start + previous.length + 1 == chunkStart) {
                 previous.length += required;
-                // compactify-free-list: merge the next entry into this one if
+                // compact: merge the next entry into this one if
                 // they are now adjacent
                 if (previous.start + previous.length + 1 == found.start) {
                     previous.length += found.length;
@@ -144,23 +165,44 @@ public class FreeSpaceList {
         freeSpaceList.add(i, newRange);
     }
 
-    public synchronized void clear() {
-        freeSpaceList.clear();
-        freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_NO_PAGES));
-    }
-
-    /** debug method */
-    public void dumpFreeList() {
-        StringBuilder buf = new StringBuilder("free list : ");
+    public String toString() {
+        StringBuilder buff = new StringBuilder();
         boolean first = true;
-        for (PageRange pr : freeSpaceList) {
+        for (PageRange r : freeSpaceList) {
             if (first) {
                 first = false;
             } else {
-                buf.append(", ");
+                buff.append(", ");
             }
-            buf.append(pr.start + "-" + (pr.start + pr.length - 1));
+            buff.append(r.start + "-" + (r.start + r.length - 1));
         }
-        System.out.println(buf.toString());
+        return buff.toString();
     }
+    
+    /**
+     * A range of free pages.
+     */
+    private static final class PageRange {
+        
+        /** 
+         * The starting point, in pages.
+         */
+        public int start;
+        
+        /** 
+         * The length, in pages.
+         */
+        public int length;
+
+        public PageRange(int start, int length) {
+            this.start = start;
+            this.length = length;
+        }
+
+        @Override
+        public String toString() {
+            return "start:" + start + " length:" + length;
+        }
+    }
+
 }

h2/src/main/org/h2/mvstore/MVMap.java

@@ -773,18 +773,6 @@ public class MVMap<K, V> extends AbstractMap<K, V>
         return new Cursor<K>(this, root, from);
     }
 
-    /**
-     * Iterate over all keys in changed pages.
-     *
-     * @param version the old version
-     * @return the iterator
-     */
-    public Iterator<K> changeIterator(long version) {
-        checkOpen();
-        MVMap<K, V> old = openVersion(version);
-        return new ChangeCursor<K, V>(this, root, old.root);
-    }
-
     public Set<Map.Entry<K, V>> entrySet() {
         HashMap<K, V> map = new HashMap<K, V>();
         for (K k : keySet()) {

h2/src/main/org/h2/mvstore/MVStore.java

@@ -44,8 +44,6 @@ H:3,...
 TODO:
 
 - rolling docs review: at convert "Features" to top-level (linked) entries
-- mvcc with multiple transactions
-- test LZ4
 - additional test async write / read algorithm for speed and errors
 - move setters to the builder, except for setRetainVersion, setReuseSpace,
     and settings that are persistent (setStoreVersion)
@@ -96,7 +94,9 @@ TODO:
 -- use a transaction log where only the deltas are stored
 - serialization for lists, sets, sets, sorted sets, maps, sorted maps
 - maybe rename 'rollback' to 'revert'
-- support other compression algorithms (deflate,...)
+- support other compression algorithms (deflate, LZ4,...)
+- only retain the last version, unless explicitly set (setRetainVersion)
+- unit test for the FreeSpaceList; maybe find a simpler implementation
 
 */
 
@@ -155,12 +155,11 @@ public class MVStore {
 
 
     /**
-     * The map of temporarily freed entries in the chunks. The key is the
-     * unsaved version, the value is the map of chunks. The maps of chunks
-     * contains the number of freed entries per chunk.
-     * Access is synchronized.
+     * The map of temporarily removed pages. The key is the unsaved version, the
+     * value is the map of chunks. The maps of chunks contains the number of
+     * freed entries per chunk. Access is synchronized.
      */
-    private final HashMap<Long, HashMap<Integer, Chunk>> freedChunks = New.hashMap();
+    private final HashMap<Long, HashMap<Integer, Chunk>> freedPages = New.hashMap();
 
     private MVMapConcurrent<String, String> meta;
 
@@ -227,8 +226,8 @@ public class MVStore {
 
     MVStore(HashMap<String, Object> config) {
         String f = (String) config.get("fileName");
-        if (f != null && !f.startsWith("nio:")) {
-            // NIO is used by default
+        if (f != null && f.indexOf(':') < 0) {
+            // NIO is used, unless a different file system is specified
             // the following line is to ensure the NIO file system is compiled
             FilePathNio.class.getName();
             f = "nio:" + f;
@@ -551,23 +550,32 @@ public class MVStore {
         lastChunkId = header.id;
         chunks.put(header.id, header);
         meta.setRootPos(header.metaRootPos, -1);
-        Iterator<String> it = meta.keyIterator("chunk.");
-        while (it.hasNext()) {
-            String s = it.next();
-            if (!s.startsWith("chunk.")) {
-                break;
+        // load chunks in reverse order, because data about previous chunks
+        // might only be available in later chunks
+        // if this is a performance problem when there are many
+        // chunks, the id of the previous / next chunk might need to
+        // be maintained
+        for (int id = lastChunkId; id >= 0; id--) {
+            String s = meta.get("chunk." + id);
+            if (s == null) {
+                continue;
             }
-            Chunk c = Chunk.fromString(meta.get(s));
+            Chunk c = Chunk.fromString(s);
             if (c.id == header.id) {
                 c.start = header.start;
                 c.length = header.length;
                 c.metaRootPos = header.metaRootPos;
-                c.maxLengthLive = header.maxLengthLive;
                 c.pageCount = header.pageCount;
+                c.pageCountLive = header.pageCountLive;
                 c.maxLength = header.maxLength;
+                c.maxLengthLive = header.maxLengthLive;
             }
             lastChunkId = Math.max(c.id, lastChunkId);
             chunks.put(c.id, c);
+            if (c.pageCountLive == 0) {
+                // remove this chunk in the next save operation
+                registerFreePage(currentVersion, c.id, 0, 0);
+            }
         }
         // rebuild the free space list
         freeSpaceList.clear();
@@ -835,7 +843,6 @@ public class MVStore {
         c.maxLengthLive = Long.MAX_VALUE;
         c.start = Long.MAX_VALUE;
         c.length = Integer.MAX_VALUE;
-        time = Math.max(0, time - creationTime);
         c.time = time;
         c.version = version;
         chunks.put(c.id, c);
@@ -848,7 +855,9 @@ public class MVStore {
                 if (v >= 0 && m.getVersion() >= lastStoredVersion) {
                     MVMap<?, ?> r = m.openVersion(storeVersion);
                     r.waitUntilWritten(r.getRoot());
-                    changed.add(r);
+                    if (r.getRoot().getPos() == 0) {
+                        changed.add(r);
+                    }
                 }
             }
         }
@@ -859,23 +868,8 @@ public class MVStore {
             } else {
                 meta.put("root." + m.getId(), String.valueOf(Integer.MAX_VALUE));
             }
-        }
-
-        applyFreedChunks(storeVersion);
-        Set<Integer> removedChunks = New.hashSet();
-        // do it twice, because changing the meta table
-        // could cause a chunk to become empty
-        for (int i = 0; i < 2; i++) {
-            for (Chunk x : chunks.values()) {
-                if (x.maxLengthLive == 0 && canOverwriteChunk(x, time)) {
-                    meta.remove("chunk." + x.id);
-                    removedChunks.add(x.id);
-                } else {
-                    meta.put("chunk." + x.id, x.asString());
-                }
-                applyFreedChunks(storeVersion);
-            }
-        }
+        }   
+        Set<Chunk> removedChunks = applyFreedPages(storeVersion, time);
         ByteBuffer buff;
         if (writeBuffer != null) {
             buff = writeBuffer;
@@ -899,7 +893,7 @@ public class MVStore {
         meta.put("chunk." + c.id, c.asString());
 
         if (ASSERT) {
-            if (freedChunks.size() > 0) {
+            if (freedPages.size() > 0) {
                 throw DataUtils.newIllegalStateException("Temporary freed chunks");
             }
         }
@@ -920,13 +914,9 @@ public class MVStore {
         long filePos = reuseSpace ? allocateChunk(length) : fileLength;
         boolean storeAtEndOfFile = filePos + length >= fileLength;
 
-        // need to keep old chunks
-        // until they are are no longer referenced
-        // by an old version
-        // so empty space is not reused too early
-        for (int x : removedChunks) {
-            freeSpaceList.markFree(chunks.get(x));
-            chunks.remove(x);
+        // free up the space of unused chunks now
+        for (Chunk x : removedChunks) {
+            freeSpaceList.markFree(x);
         }
 
         c.start = filePos;
@@ -971,7 +961,6 @@ public class MVStore {
         currentStoreVersion = -1;
         metaChanged = false;
         lastStoredVersion = storeVersion;
-
         return version;
     }
 
@@ -990,25 +979,64 @@ public class MVStore {
         return System.currentTimeMillis() - creationTime;
     }
 
-    private void applyFreedChunks(long storeVersion) {
-        synchronized (freedChunks) {
-            for (Iterator<Long> it = freedChunks.keySet().iterator(); it.hasNext();) {
-                long v = it.next();
-                if (v > storeVersion) {
-                    continue;
+    /**
+     * Apply the freed pages to the chunk metadata. The metadata is updated, but
+     * freed chunks are not removed yet.
+     *
+     * @param storeVersion apply up to the given version
+     * @return the set of freed chunks (might be empty)
+     */
+    private Set<Chunk> applyFreedPages(long storeVersion, long time) {
+        Set<Chunk> removedChunks = New.hashSet();
+        synchronized (freedPages) {
+            while (true) {
+                ArrayList<Chunk> modified = New.arrayList();
+                for (Iterator<Long> it = freedPages.keySet().iterator(); it.hasNext();) {
+                    long v = it.next();
+                    if (v > storeVersion) {
+                        continue;
+                    }
+                    Map<Integer, Chunk> freed = freedPages.get(v);
+                    for (Chunk f : freed.values()) {
+                        Chunk c = chunks.get(f.id);
+                        c.maxLengthLive += f.maxLengthLive;
+                        c.pageCountLive += f.pageCountLive;
+                        if (c.pageCountLive < 0) {
+                            throw DataUtils.newIllegalStateException(
+                                    "Corrupt page count {0}", c.pageCountLive);
+                        }
+                        if (c.maxLengthLive < 0) {
+                            throw DataUtils.newIllegalStateException(
+                                    "Corrupt max length {0}", c.maxLengthLive);
+                        }
+                        if (c.pageCount == 0 && c.maxLengthLive > 0) {
+                            throw DataUtils.newIllegalStateException(
+                                    "Corrupt max length {0}", c.maxLengthLive);
+                        }
+                        modified.add(c);
+                    }
+                    it.remove();
                 }
-                Map<Integer, Chunk> freed = freedChunks.get(v);
-                for (Chunk f : freed.values()) {
-                    Chunk c = chunks.get(f.id);
-                    c.maxLengthLive += f.maxLengthLive;
-                    if (c.maxLengthLive < 0) {
-                        throw DataUtils.newIllegalStateException(
-                                "Corrupt max length {0}", c.maxLengthLive);
+                for (Chunk c : modified) {
+                    if (c.maxLengthLive == 0) {
+                        if (canOverwriteChunk(c, time)) {
+                            removedChunks.add(c);
+                            chunks.remove(c.id);
+                            meta.remove("chunk." + c.id);
+                        } else {
+                            // remove this chunk in the next save operation
+                            registerFreePage(storeVersion + 1, c.id, 0, 0);
+                        }
+                    } else {
+                        meta.put("chunk." + c.id, c.asString());
                     }
                 }
-                it.remove();
+                if (modified.size() == 0) {
+                    break;
+                }
             }
         }
+        return removedChunks;
     }
 
     /**
@@ -1267,18 +1295,23 @@ public class MVStore {
             // can be re-used
             version = currentStoreVersion;
         }
-        synchronized (freedChunks) {
-            HashMap<Integer, Chunk>freed = freedChunks.get(version);
+        registerFreePage(version, c.id, DataUtils.getPageMaxLength(pos), 1);
+    }
+    
+    private void registerFreePage(long version, int chunkId, long maxLengthLive, int pageCount) {
+        synchronized (freedPages) {
+            HashMap<Integer, Chunk>freed = freedPages.get(version);
             if (freed == null) {
                 freed = New.hashMap();
-                freedChunks.put(version, freed);
+                freedPages.put(version, freed);
             }
-            Chunk f = freed.get(c.id);
+            Chunk f = freed.get(chunkId);
             if (f == null) {
-                f = new Chunk(c.id);
-                freed.put(c.id, f);
+                f = new Chunk(chunkId);
+                freed.put(chunkId, f);
             }
-            f.maxLengthLive -= DataUtils.getPageMaxLength(pos);
+            f.maxLengthLive -= maxLengthLive;
+            f.pageCountLive -= pageCount;
         }
     }
 
@@ -1492,8 +1525,8 @@ public class MVStore {
             chunks.clear();
             freeSpaceList.clear();
             maps.clear();
-            synchronized (freedChunks) {
-                freedChunks.clear();
+            synchronized (freedPages) {
+                freedPages.clear();
             }
             currentVersion = version;
             metaChanged = false;
@@ -1506,10 +1539,10 @@ public class MVStore {
             m.rollbackTo(version);
         }
         for (long v = currentVersion; v >= version; v--) {
-            if (freedChunks.size() == 0) {
+            if (freedPages.size() == 0) {
                 break;
             }
-            freedChunks.remove(v);
+            freedPages.remove(v);
         }
         meta.rollbackTo(version);
         metaChanged = false;
@@ -1556,8 +1589,8 @@ public class MVStore {
     }
 
     private void revertTemp(long storeVersion) {
-        synchronized (freedChunks) {
-            for (Iterator<Long> it = freedChunks.keySet().iterator(); it.hasNext();) {
+        synchronized (freedPages) {
+            for (Iterator<Long> it = freedPages.keySet().iterator(); it.hasNext();) {
                 long v = it.next();
                 if (v > storeVersion) {
                     continue;

h2/src/main/org/h2/mvstore/Page.java

@@ -814,11 +814,15 @@ public class Page {
                 ^ DataUtils.getCheckValue(start)
                 ^ DataUtils.getCheckValue(pageLength);
         buff.putShort(start + 4, (short) check);
-        this.pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
+        if (pos != 0) {
+            throw DataUtils.newIllegalStateException("Page already stored");
+        }
+        pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
         long max = DataUtils.getPageMaxLength(pos);
         chunk.maxLength += max;
         chunk.maxLengthLive += max;
         chunk.pageCount++;
+        chunk.pageCountLive++;
         return buff;
     }
 

h2/src/main/org/h2/mvstore/TransactionStore.java

@@ -90,6 +90,7 @@ public class TransactionStore {
         MVMapConcurrent.Builder<long[], Object[]> builder = 
                 new MVMapConcurrent.Builder<long[], Object[]>().
                 valueType(valueType);
+        // TODO escape other map names, to avoid conflicts
         undoLog = store.openMap("undoLog", builder);
         init();
     }
@@ -220,8 +221,9 @@ public class TransactionStore {
     void commit(Transaction t, long maxLogId) {
         store.incrementVersion();
         for (long logId = 0; logId < maxLogId; logId++) {
-            Object[] op = undoLog.get(new long[] {
-                    t.getId(), logId });
+            long[] undoKey = new long[] {
+                    t.getId(), logId };
+            Object[] op = undoLog.get(undoKey);
             int opType = (Integer) op[0];
             if (opType == Transaction.OP_REMOVE) {
                 int mapId = (Integer) op[1];
@@ -238,7 +240,7 @@ public class TransactionStore {
                     map.remove(key);
                 }
             }
-            undoLog.remove(logId);
+            undoLog.remove(undoKey);
         }
         openTransactions.remove(t.getId());
         openTransactionMap.remove(t.getId());
@@ -858,6 +860,7 @@ public class TransactionStore {
         }
 
         public void removeMap() {
+            // TODO remove in a transaction
             mapWrite.removeMap();
         }
 

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -116,7 +116,8 @@ public class MVTableEngine implements TableEngine {
      * @param store the store
      */
     static void store(MVStore store) {
-        store.compact(50);
+        int test;
+//        store.compact(50);
         store.store();
     }
 

h2/src/test/org/h2/test/TestAll.java

@@ -203,6 +203,9 @@ java org.pitest.mutationtest.MutationCoverageReport
 --targetTests org.h2.test.store.TestStreamStore
 --sourceDirs src/test,src/tools
 
+Dump heap on out of memory:
+-XX:+HeapDumpOnOutOfMemoryError
+
 Random test:
 java15
 cd h2database/h2/bin
@@ -536,6 +539,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         ssl = false;
         traceLevelFile = 0;
         test();
+        testUnit();
 
         big = false;
         cipher = "AES";

h2/src/test/org/h2/test/TestBase.java

@@ -600,9 +600,9 @@ public abstract class TestBase {
      */
     public void assertEquals(java.util.Date expected, java.util.Date actual) {
         if (expected != actual && !expected.equals(actual)) {
-            final DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
-            final SimpleTimeZone GMT_TIMEZONE = new SimpleTimeZone(0, "Z");
-            df.setTimeZone(GMT_TIMEZONE);
+            DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+            SimpleTimeZone gmt = new SimpleTimeZone(0, "Z");
+            df.setTimeZone(gmt);
             fail("Expected: " + df.format(expected) + " actual: " + df.format(actual));
         }
     }

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -8,7 +8,6 @@ package org.h2.test.store;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Random;
@@ -22,7 +21,6 @@ import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
-import org.h2.util.New;
 
 /**
  * Tests the MVStore.
@@ -35,7 +33,10 @@ public class TestMVStore extends TestBase {
      * @param a ignored
      */
     public static void main(String... a) throws Exception {
-        TestBase.createCaller().init().test();
+        TestBase test = TestBase.createCaller().init();
+        test.config.traceTest = true;
+        test.config.big = true;
+        test.test();
     }
 
     public void test() throws Exception {
@@ -61,7 +62,6 @@ public class TestMVStore extends TestBase {
         testIterateOldVersion();
         testObjects();
         testExample();
-        testIterateOverChanges();
         testOpenStoreCloseLoop();
         testVersion();
         testTruncateFile();
@@ -79,10 +79,10 @@ public class TestMVStore extends TestBase {
         testIterate();
         testCloseTwice();
         testSimple();
-        // this test will run out of memory on a 32-bit VM before it gets to the part we need it to test
-        if ("64".equals(System.getProperty("sun.arch.data.model"))) { 
-            testLargerThan2G();
-        }
+        
+        // longer running tests
+        
+        testLargerThan2G();
     }
 
     private void testAtomicOperations() {
@@ -705,39 +705,6 @@ public class TestMVStore extends TestBase {
         }
     }
 
-    private void testIterateOverChanges() {
-        String fileName = getBaseDir() + "/testIterate.h3";
-        FileUtils.delete(fileName);
-        MVStore s = openStore(fileName);
-        s.setPageSize(6);
-        MVMap<Integer, String> m = s.openMap("data");
-        for (int i = 0; i < 100; i++) {
-            m.put(i, "Hi");
-        }
-        s.incrementVersion();
-        s.store();
-        for (int i = 20; i < 40; i++) {
-            assertEquals("Hi", m.put(i, "Hello"));
-        }
-        long old = s.getCurrentVersion();
-        s.incrementVersion();
-        for (int i = 10; i < 15; i++) {
-            m.put(i, "Hallo");
-        }
-        m.put(50, "Hallo");
-        for (int i = 90; i < 93; i++) {
-            assertEquals("Hi", m.remove(i));
-        }
-        assertEquals(null, m.put(100, "Hallo"));
-        Iterator<Integer> it = m.changeIterator(old);
-        ArrayList<Integer> list = New.arrayList();
-        while (it.hasNext()) {
-            list.add(it.next());
-        }
-        assertEquals("[10, 11, 12, 13, 14, 50, 100, 90, 91, 92]", list.toString());
-        s.close();
-    }
-
     private void testOldVersion() {
         MVStore s;
         for (int op = 0; op <= 1; op++) {
@@ -1031,8 +998,8 @@ public class TestMVStore extends TestBase {
         assertEquals("name:data", m.get("map." + id));
         assertTrue(m.get("root.1").length() > 0);
         assertTrue(m.containsKey("chunk.1"));
-        assertEquals("id:1,length:246,maxLength:224,maxLengthLive:0," +
-                "metaRoot:274877910922,pageCount:2," +
+        assertEquals("id:1,length:263,maxLength:288,maxLengthLive:288," +
+                "metaRoot:274877910924,pageCount:2,pageCountLive:2," +
                 "start:8192,time:0,version:1", m.get("chunk.1"));
 
         assertTrue(m.containsKey("chunk.2"));
@@ -1397,30 +1364,36 @@ public class TestMVStore extends TestBase {
         s.close();
     }
     
-    private void testLargerThan2G() {
+    private void testLargerThan2G() throws IOException {
+        if (!config.big) {
+            return;
+        }
         String fileName = getBaseDir() + "/testLargerThan2G.h3";
         FileUtils.delete(fileName);
-        
-        MVStore store = new MVStore.Builder().cacheSize(800).fileName(fileName).writeBufferSize(0).writeDelay(-1)
-                .open();
-
-        MVMap<String, String> map = store.openMap("test");
-        int totalWrite = 15000000;
-        int lineStored = 0;
-        while (lineStored <= totalWrite) {
-            lineStored++;
-            String actualKey = lineStored + " just for length length length    " + lineStored;
-            String value = "Just a a string that is kinda long long long but not too much much much much much much much much much   "
-                    + lineStored;
-
-            map.put(actualKey, value);
-
-            if (lineStored % 1000000 == 0) {
-                store.store();
+        MVStore store = new MVStore.Builder().cacheSize(16).
+                fileName(fileName).open();
+        MVMap<Integer, String> map = store.openMap("test");
+        long last = System.currentTimeMillis();
+        String data = new String(new char[2500]).replace((char) 0, 'x');
+        for (int i = 0;; i++) {
+            map.put(i, data);
+            if (i % 10000 == 0) {
+                long version = store.commit();
+                store.setRetainVersion(version);
+                long time = System.currentTimeMillis();
+                if (time - last > 2000) {
+                    long mb = store.getFile().size() / 1024 / 1024;
+                    trace(mb + "/4500");
+                    if (mb > 4500) {
+                        break;
+                    }
+                    last = time;
+                }
             }
         }
-        store.store();
+        store.commit();
         store.close();
+        FileUtils.delete(fileName);
     }
 
     /**

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -36,7 +36,7 @@ public class TestMVTableEngine extends TestBase {
     public void test() throws Exception {
         testEncryption();
         testReadOnly();
-        // testReuseDiskSpace();
+        testReuseDiskSpace();
         testDataTypes();
         testLocking();
         testSimple();
@@ -87,8 +87,8 @@ public class TestMVTableEngine extends TestBase {
                 ";DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
         Connection conn;
         Statement stat;
-        long maxSizeHalf = 0;
-        for (int i = 0; i < 10; i++) {
+        long maxSize = 0;
+        for (int i = 0; i < 20; i++) {
             conn = getConnection(dbName);
             for (MVTableEngine.Store s : MVTableEngine.getStores()) {
                 s.getStore().setRetentionTime(0);
@@ -100,10 +100,10 @@ public class TestMVTableEngine extends TestBase {
             conn.close();
             long size = FileUtils.size(getBaseDir() + "/mvstore"
                     + Constants.SUFFIX_MV_FILE);
-            if (i < 6) {
-                maxSizeHalf = Math.max(size, maxSizeHalf);
-            } else if (size > maxSizeHalf) {
-                fail(i + " size: " + size + " max: " + maxSizeHalf);
+            if (i < 10) {
+                maxSize = (int) (Math.max(size, maxSize) * 1.1);
+            } else if (size > maxSize) {
+                fail(i + " size: " + size + " max: " + maxSize);
             }
         }
     }