A persistent tree map (work in progress)

h2/src/tools/org/h2/dev/store/btree/BtreeMap.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.dev.store.btree;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+/**
+ * A stored map.
+ *
+ * @param <K> the key class
+ * @param <V> the value class
+ */
+public class BtreeMap<K, V> {
+
+    private final BtreeMapStore store;
+    private final String name;
+    private final KeyType keyType;
+    private final ValueType valueType;
+    private Page root;
+
+    private BtreeMap(BtreeMapStore store, String name, Class<K> keyClass, Class<V> valueClass) {
+        this.store = store;
+        this.name = name;
+        if (keyClass == Integer.class) {
+            keyType = new IntegerType();
+        } else if (keyClass == String.class) {
+            keyType = new StringType();
+        } else {
+            throw new RuntimeException("Unsupported key class " + keyClass.toString());
+        }
+        if (valueClass == Integer.class) {
+            valueType = new IntegerType();
+        } else if (valueClass == String.class) {
+            valueType = new StringType();
+        } else {
+            throw new RuntimeException("Unsupported value class " + keyClass.toString());
+        }
+    }
+
+    /**
+     * Get the class with the given tag name.
+     *
+     * @param name the tag name
+     * @return the class
+     */
+    static Class<?> getClass(String name) {
+        if (name.equals("i")) {
+            return Integer.class;
+        } else if (name.equals("s")) {
+            return String.class;
+        }
+        throw new RuntimeException("Unknown class name " + name);
+    }
+
+    /**
+     * Open a map.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     * @param store the tree store
+     * @param name the name of the map
+     * @param keyClass the key class
+     * @param valueClass the value class
+     * @return the map
+     */
+    static <K, V> BtreeMap<K, V> open(BtreeMapStore store, String name, Class<K> keyClass, Class<V> valueClass) {
+        return new BtreeMap<K, V>(store, name, keyClass, valueClass);
+    }
+
+    /**
+     * Store a key-value pair.
+     *
+     * @param key the key
+     * @param data the value
+     */
+    public void put(K key, V data) {
+        if (!isChanged()) {
+            store.markChanged(name, this);
+        }
+        root = Page.put(this, root, key, data);
+    }
+
+    /**
+     * Get a value.
+     *
+     * @param key the key
+     * @return the value
+     */
+    @SuppressWarnings("unchecked")
+    public V get(K key) {
+        if (root == null) {
+            return null;
+        }
+        return (V) root.find(key);
+    }
+
+    /**
+     * Remove a key-value pair.
+     *
+     * @param key the key
+     */
+    public void remove(K key) {
+        if (!isChanged()) {
+            store.markChanged(name, this);
+        }
+        root = Page.remove(root, key);
+    }
+
+    /**
+     * Was this map changed.
+     *
+     * @return true if yes
+     */
+    boolean isChanged() {
+        return root != null && root.getId() < 0;
+    }
+
+    /**
+     * A value type.
+     */
+    static interface ValueType {
+
+        /**
+         * Get the length in bytes.
+         *
+         * @param obj the object
+         * @return the length
+         */
+        int length(Object obj);
+
+        /**
+         * Write the object.
+         *
+         * @param buff the target buffer
+         * @param x the value
+         */
+        void write(ByteBuffer buff, Object x);
+
+        /**
+         * Read an object.
+         *
+         * @param buff the source buffer
+         * @return the object
+         */
+        Object read(ByteBuffer buff);
+
+        /**
+         * Get the tag name of the class.
+         *
+         * @return the tag name
+         */
+        String getName();
+
+    }
+
+    /**
+     * A key type.
+     */
+    static interface KeyType extends ValueType {
+
+        /**
+         * Compare two keys.
+         *
+         * @param a the first key
+         * @param b the second key
+         * @return -1 if the first key is smaller, 1 if larger, and 0 if equal
+         */
+        int compare(Object a, Object b);
+    }
+
+    /**
+     * Compare two keys.
+     *
+     * @param a the first key
+     * @param b the second key
+     * @return -1 if the first key is smaller, 1 if bigger, 0 if equal
+     */
+    int compare(Object a, Object b) {
+        return keyType.compare(a, b);
+    }
+
+    /**
+     * An integer type.
+     */
+    static class IntegerType implements KeyType {
+
+        public int compare(Object a, Object b) {
+            return ((Integer) a).compareTo((Integer) b);
+        }
+
+        public int length(Object obj) {
+            return getVarIntLen((Integer) obj);
+        }
+
+        public Integer read(ByteBuffer buff) {
+            return readVarInt(buff);
+        }
+
+        public void write(ByteBuffer buff, Object x) {
+            writeVarInt(buff, (Integer) x);
+        }
+
+        public String getName() {
+            return "i";
+        }
+
+    }
+
+    /**
+     * A string type.
+     */
+    static class StringType implements KeyType {
+
+        public int compare(Object a, Object b) {
+            return a.toString().compareTo(b.toString());
+        }
+
+        public int length(Object obj) {
+            try {
+                byte[] bytes = obj.toString().getBytes("UTF-8");
+                return getVarIntLen(bytes.length) + bytes.length;
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public String read(ByteBuffer buff) {
+            int len = readVarInt(buff);
+            byte[] bytes = new byte[len];
+            buff.get(bytes);
+            try {
+                return new String(bytes, "UTF-8");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public void write(ByteBuffer buff, Object x) {
+            try {
+                byte[] bytes = x.toString().getBytes("UTF-8");
+                writeVarInt(buff, bytes.length);
+                buff.put(bytes);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public String getName() {
+            return "s";
+        }
+
+    }
+
+    /**
+     * Get the key type.
+     *
+     * @return the key type
+     */
+    KeyType getKeyType() {
+        return keyType;
+    }
+
+    /**
+     * Get the value type.
+     *
+     * @return the value type
+     */
+    ValueType getValueType() {
+        return valueType;
+    }
+
+    long getTransaction() {
+        return store.getTransaction();
+    }
+
+    /**
+     * Register a page and get the next temporary page id.
+     *
+     * @param p the new page
+     * @return the page id
+     */
+    long registerTempPage(Page p) {
+        return store.registerTempPage(p);
+    }
+
+    /**
+     * Read a node.
+     *
+     * @param id the node id
+     * @return the node
+     */
+    Page readPage(long id) {
+        return store.readPage(this, id);
+    }
+
+    /**
+     * Remove a node.
+     *
+     * @param id the node id
+     */
+    void removePage(long id) {
+        store.removePage(id);
+    }
+
+    /**
+     * Set the position of the root page.
+     *
+     * @param rootPos the position
+     */
+    void setRoot(long rootPos) {
+        root = readPage(rootPos);
+    }
+
+    /**
+     * Iterate over all keys.
+     *
+     * @param from the first key to return
+     * @return the iterator
+     */
+    public Iterator<K> keyIterator(K from) {
+        return new Cursor(root, from);
+    }
+
+    /**
+     * A cursor to iterate over elements in ascending order.
+     */
+    class Cursor implements Iterator<K> {
+        Page current;
+        ArrayList<Page> parents = new ArrayList<Page>();
+
+        Cursor(Page root, K from) {
+            min(root, from);
+        }
+
+        private void min(Page p, K key) {
+            int todo;
+//            while (n != null) {
+//                int compare = key == null ? -1 : n.compare(key);
+//                if (compare == 0) {
+//                    current = n;
+//                    return;
+//                } else if (compare > 0) {
+//                    n = n.getRight();
+//                } else {
+//                    parents.add(n);
+//                    n = n.getLeft();
+//                }
+//            }
+//            if (parents.size() == 0) {
+//                current = null;
+//                return;
+//            }
+//            current = parents.remove(parents.size() - 1);
+        }
+
+        @SuppressWarnings("unchecked")
+        public K next() {
+            int todo;
+            return null;
+//            Page c = current;
+//            if (c != null) {
+//                fetchNext();
+//            }
+//            return c == null ? null : (K) c.getKey();
+        }
+
+        private void fetchNext() {
+            int todo;
+//            Page r = current.getRight();
+//            if (r != null) {
+//                min(r, null);
+//                return;
+//            }
+//            if (parents.size() == 0) {
+//                current = null;
+//                return;
+//            }
+//            current = parents.remove(parents.size() - 1);
+        }
+
+        public boolean hasNext() {
+            return current != null;
+        }
+
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * Get the root node.
+     *
+     * @return the root node
+     */
+    Page getRoot() {
+        return root;
+    }
+
+    /**
+     * Get the map name.
+     *
+     * @return the name
+     */
+    String getName() {
+        return name;
+    }
+
+    /**
+     * Read a variable size int.
+     *
+     * @param buff the source buffer
+     * @return the value
+     */
+    static int readVarInt(ByteBuffer buff) {
+        int b = buff.get();
+        if (b >= 0) {
+            return b;
+        }
+        // a separate function so that this one can be inlined
+        return readVarIntRest(buff, b);
+    }
+
+    private static int readVarIntRest(ByteBuffer buff, int b) {
+        int x = b & 0x7f;
+        b = buff.get();
+        if (b >= 0) {
+            return x | (b << 7);
+        }
+        x |= (b & 0x7f) << 7;
+        b = buff.get();
+        if (b >= 0) {
+            return x | (b << 14);
+        }
+        x |= (b & 0x7f) << 14;
+        b = buff.get();
+        if (b >= 0) {
+            return x | b << 21;
+        }
+        x |= ((b & 0x7f) << 21) | (buff.get() << 28);
+        return x;
+    }
+
+    /**
+     * Get the length of the variable size int.
+     *
+     * @param x the value
+     * @return the length in bytes
+     */
+    static int getVarIntLen(int x) {
+        if ((x & (-1 << 7)) == 0) {
+            return 1;
+        } else if ((x & (-1 << 14)) == 0) {
+            return 2;
+        } else if ((x & (-1 << 21)) == 0) {
+            return 3;
+        } else if ((x & (-1 << 28)) == 0) {
+            return 4;
+        }
+        return 5;
+    }
+
+    /**
+     * Write a variable size int.
+     *
+     * @param buff the target buffer
+     * @param x the value
+     */
+    static void writeVarInt(ByteBuffer buff, int x) {
+        while ((x & ~0x7f) != 0) {
+            buff.put((byte) (0x80 | (x & 0x7f)));
+            x >>>= 7;
+        }
+        buff.put((byte) x);
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/BtreeMapStore.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.dev.store.btree;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.TreeMap;
+import org.h2.dev.store.FilePathCache;
+import org.h2.store.fs.FilePath;
+import org.h2.util.New;
+import org.h2.util.SmallLRUCache;
+import org.h2.util.StringUtils;
+
+/*
+
+file format:
+
+header
+header
+[ chunk ] *
+
+header:
+# H3 store #
+pageSize=4096
+r=1
+
+chunk:
+'d' [id] [metaRootPos] data ...
+
+todo:
+
+- garbage collection
+
+- use page checksums
+
+- compress chunks
+
+- encode length in pos (1=32, 2=128, 3=512,...)
+
+- don't use any 't' blocks
+
+- floating header (avoid duplicate header)
+    for each chunk, store chunk (a counter)
+    for each page, store chunk id and offset to root
+    for each chunk, store position of expected next chunks
+
+*/
+
+/**
+ * A persistent storage for tree maps.
+ */
+public class BtreeMapStore {
+
+    private final String fileName;
+    private FileChannel file;
+    private int pageSize = 4 * 1024;
+    private long rootBlockStart;
+    private HashMap<Long, Page> cache = SmallLRUCache.newInstance(50000);
+    private ArrayList<Page> temp = New.arrayList();
+    private TreeMap<Integer, Block> blocks = new TreeMap<Integer, Block>();
+    private BtreeMap<String, String> meta;
+    private HashMap<String, BtreeMap<?, ?>> maps = New.hashMap();
+    private HashMap<String, BtreeMap<?, ?>> mapsChanged = New.hashMap();
+
+    // TODO use an int instead? (with rollover to 0)
+    private long transaction;
+
+    private int lastBlockId;
+
+    private int loadCount;
+
+    private BtreeMapStore(String fileName) {
+        this.fileName = fileName;
+    }
+
+    /**
+     * Open a tree store.
+     *
+     * @param fileName the file name
+     * @return the store
+     */
+    public static BtreeMapStore open(String fileName) {
+        BtreeMapStore s = new BtreeMapStore(fileName);
+        s.open();
+        return s;
+    }
+
+    /**
+     * Open a map.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     * @param name the name of the map
+     * @param keyClass the key class
+     * @param valueClass the value class
+     * @return the map
+     */
+    public <K, V> BtreeMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
+        @SuppressWarnings("unchecked")
+        BtreeMap<K, V> m = (BtreeMap<K, V>) maps.get(name);
+        if (m == null) {
+            String root = meta.get("map." + name);
+            m = BtreeMap.open(this, name, keyClass, valueClass);
+            maps.put(name, m);
+            if (root != null) {
+                root = StringUtils.arraySplit(root, ',', false)[0];
+                if (!root.equals("0")) {
+                    m.setRoot(Long.parseLong(root));
+                }
+            }
+        }
+        return m;
+    }
+
+    /**
+     * Mark a map as changed.
+     *
+     * @param name the map name
+     * @param the map
+     */
+    void markChanged(String name, BtreeMap<?, ?> map) {
+        if (map != meta) {
+            mapsChanged.put(name, map);
+        }
+    }
+
+    private void open() {
+        meta = BtreeMap.open(this, "meta", String.class, String.class);
+        new File(fileName).getParentFile().mkdirs();
+        try {
+            log("file open");
+            file = FilePathCache.wrap(FilePath.get(fileName).open("rw"));
+            if (file.size() == 0) {
+                writeHeader();
+            } else {
+                readHeader();
+                readMeta();
+            }
+        } catch (Exception e) {
+            throw convert(e);
+        }
+    }
+
+    private void readMeta() {
+        long rootId = readMetaRootId(rootBlockStart);
+        lastBlockId = getBlockId(rootId);
+        Block b = new Block(lastBlockId);
+        b.start = rootBlockStart;
+        blocks.put(b.id, b);
+        meta.setRoot(rootId);
+        Iterator<String> it = meta.keyIterator("block.");
+        while (it.hasNext()) {
+            String s = it.next();
+            if (!s.startsWith("block.")) {
+                break;
+            }
+            b = Block.fromString(meta.get(s));
+            lastBlockId = Math.max(b.id, lastBlockId);
+            blocks.put(b.id, b);
+        }
+    }
+
+    private void writeHeader() {
+        try {
+            ByteBuffer header = ByteBuffer.wrap((
+                "# H2 1.5\n" +
+                "read-version: 1\n" +
+                "write-version: 1\n" +
+                "rootBlock: " + rootBlockStart + "\n" +
+                "transaction: " + transaction + "\n").getBytes());
+            file.position(0);
+            file.write(header);
+            file.position(pageSize);
+            file.write(header);
+        } catch (Exception e) {
+            throw convert(e);
+        }
+    }
+
+    private void readHeader() {
+        try {
+            file.position(0);
+            byte[] header = new byte[pageSize];
+            // TODO read fully; read both headers
+            file.read(ByteBuffer.wrap(header));
+            Properties prop = new Properties();
+            prop.load(new ByteArrayInputStream(header));
+            rootBlockStart = Long.parseLong(prop.get("rootBlock").toString());
+            transaction = Long.parseLong(prop.get("transaction").toString());
+        } catch (Exception e) {
+            throw convert(e);
+        }
+    }
+
+    public String toString() {
+        return "cache size: " + cache.size() + " loadCount: " + loadCount + "\n" + blocks.toString().replace('\n', ' ');
+    }
+
+    private static RuntimeException convert(Exception e) {
+        throw new RuntimeException("Exception: " + e, e);
+    }
+
+    /**
+     * Close the file.
+     */
+    public void close() {
+        store();
+        if (file != null) {
+            try {
+                log("file close");
+                file.close();
+            } catch (Exception e) {
+                file = null;
+                throw convert(e);
+            }
+        }
+    }
+
+    private long getPosition(long pageId) {
+        Block b = getBlock(pageId);
+        if (b == null) {
+            throw new RuntimeException("Block " + getBlockId(pageId) + " not found");
+        }
+        long pos = b.start;
+        pos += (int) (pageId & Integer.MAX_VALUE);
+        return pos;
+    }
+
+    private long getId(int blockId, int offset) {
+        return ((long) blockId << 32) | offset;
+    }
+
+    /**
+     * Persist all changes to disk.
+     */
+    public void store() {
+        if (!meta.isChanged() && mapsChanged.size() == 0) {
+            // TODO truncate file if empty
+            return;
+        }
+        commit();
+
+        // the length estimate is not correct,
+        // as we don't know the exact positions and entry counts
+        int lenEstimate = 1 + 8;
+        for (BtreeMap<?, ?> m : mapsChanged.values()) {
+            meta.put("map." + m.getName(), String.valueOf(Long.MAX_VALUE) +
+                    "," + m.getKeyType().getName() + "," + m.getValueType().getName());
+            lenEstimate += m.getRoot().lengthIncludingTempChildren();
+        }
+        int blockId = ++lastBlockId;
+        Block b = new Block(blockId);
+        b.start = Long.MAX_VALUE;
+        b.entryCount = Integer.MAX_VALUE;
+        b.liveCount = Integer.MAX_VALUE;
+        blocks.put(b.id, b);
+        for (Block x : blocks.values()) {
+            if (x.liveCount == 0) {
+                meta.remove("block." + x.id);
+            } else {
+                meta.put("block." + x.id, "temp " + x.toString());
+            }
+        }
+        // modifying the meta can itself affect the metadata
+        // TODO solve this in a better way
+        for (Block x : new ArrayList<Block>(blocks.values())) {
+            if (x.liveCount == 0) {
+                meta.remove("block." + x.id);
+                blocks.remove(x.id);
+            } else {
+                meta.put("block." + x.id, x.toString());
+            }
+        }
+        lenEstimate += meta.getRoot().lengthIncludingTempChildren();
+        b.length = lenEstimate;
+
+        long storePos = allocateBlock(lenEstimate);
+
+        long pageId = getId(blockId, 1 + 8);
+        for (BtreeMap<?, ?> m : mapsChanged.values()) {
+            Page r = m.getRoot();
+            long p = r == null ? 0 : pageId;
+            meta.put("map." + m.getName(), String.valueOf(p) + "," + m.getKeyType().getName() + "," + m.getValueType().getName());
+            if (r != null) {
+                pageId = r.updatePageIds(pageId);
+            }
+        }
+       int metaRootOffset = (int) (pageId - getId(blockId, 0));
+
+        // add a dummy entry so the count is correct
+        meta.put("block." + b.id, b.toString());
+        int count = 0;
+        for (BtreeMap<?, ?> m : mapsChanged.values()) {
+            count += m.getRoot().countTemp();
+        }
+        count += meta.getRoot().countTemp();
+
+        b.start = storePos;
+        b.entryCount = count;
+        b.liveCount = b.entryCount;
+
+        meta.put("block." + b.id, b.toString());
+
+        pageId = meta.getRoot().updatePageIds(pageId);
+        int len = (int) (pageId - getId(blockId, 0));
+
+        ByteBuffer buff = ByteBuffer.allocate(len);
+        buff.put((byte) 'd');
+        buff.putInt(b.id);
+        buff.putInt(metaRootOffset);
+        for (BtreeMap<?, ?> m : mapsChanged.values()) {
+            m.getRoot().storeTemp(buff);
+        }
+        meta.getRoot().storeTemp(buff);
+        if (buff.hasRemaining()) {
+            throw new RuntimeException("remaining: " + buff.remaining());
+        }
+        buff.rewind();
+        try {
+            file.position(storePos);
+            file.write(buff);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        rootBlockStart = storePos;
+        writeHeader();
+        mapsChanged.clear();
+        temp.clear();
+    }
+
+    private long allocateBlock(long length) {
+        BitSet set = new BitSet();
+        set.set(0);
+        set.set(1);
+        for (Block b : blocks.values()) {
+            if (b.start == Long.MAX_VALUE) {
+                continue;
+            }
+            int first = (int) (b.start / pageSize);
+            int last = (int) ((b.start + b.length) / pageSize);
+            set.set(first, last +1);
+        }
+        int required = (int) (length / pageSize) + 1;
+        for (int i = 0; i < set.size(); i++) {
+            if (!set.get(i)) {
+                boolean ok = true;
+                for (int j = 1; j <= required; j++) {
+                    if (set.get(i + j)) {
+                        ok = false;
+                        break;
+                    }
+                }
+                if (ok) {
+                    return i * pageSize;
+                }
+            }
+        }
+        return set.size() * pageSize;
+    }
+
+    /**
+     * Get the current transaction number.
+     *
+     * @return the transaction number
+     */
+    long getTransaction() {
+        return transaction;
+    }
+
+    /**
+     * Register a page and get the next temporary page id.
+     *
+     * @param p the new page
+     * @return the page id
+     */
+    long registerTempPage(Page p) {
+        temp.add(p);
+        return -temp.size();
+    }
+
+    /**
+     * Commit the current transaction.
+     *
+     * @return the transaction id
+     */
+    public long commit() {
+        return ++transaction;
+    }
+
+    private long readMetaRootId(long blockStart) {
+        try {
+            file.position(blockStart);
+            ByteBuffer buff = ByteBuffer.wrap(new byte[16]);
+            file.read(buff);
+            buff.rewind();
+            if (buff.get() != 'd') {
+                throw new RuntimeException("File corrupt");
+            }
+            int blockId = buff.getInt();
+            int offset = buff.getInt();
+            return getId(blockId, offset);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Try to reduce the file size. Blocks with a low number of live items will
+     * be re-written.
+     */
+    public void compact() {
+        if(true)throw new RuntimeException("not implemented yet");
+        if (blocks.size() <= 1) {
+            return;
+        }
+        long liveCountTotal = 0, entryCountTotal = 0;
+        for (Block b : blocks.values()) {
+            entryCountTotal += b.entryCount;
+            liveCountTotal += b.liveCount;
+        }
+        int averageEntryCount = (int) (entryCountTotal / blocks.size());
+        if (entryCountTotal == 0) {
+            return;
+        }
+        int percentTotal = (int) (100 * liveCountTotal / entryCountTotal);
+        if (percentTotal > 80) {
+            return;
+        }
+        ArrayList<Block> old = New.arrayList();
+        for (Block b : blocks.values()) {
+            int age = lastBlockId - b.id + 1;
+            b.collectPriority = b.getFillRate() / age;
+            old.add(b);
+        }
+        Collections.sort(old, new Comparator<Block>() {
+            public int compare(Block o1, Block o2) {
+                return new Integer(o1.collectPriority).compareTo(o2.collectPriority);
+            }
+        });
+        int moveCount = 0;
+        Block move = null;
+        for (Block b : old) {
+            if (moveCount + b.liveCount > averageEntryCount) {
+                break;
+            }
+            log(" block " + b.id + " " + b.getFillRate() + "% full; prio=" + b.collectPriority);
+            moveCount += b.liveCount;
+            move = b;
+        }
+        boolean remove = false;
+        for (Iterator<Block> it = old.iterator(); it.hasNext();) {
+            Block b = it.next();
+            if (move == b) {
+                remove = true;
+            } else if (remove) {
+                it.remove();
+            }
+        }
+        long oldMetaRootId = readMetaRootId(move.start);
+        long offset = getPosition(oldMetaRootId);
+        log("  meta:" + move.id + "/" + offset);
+        BtreeMap<String, String> oldMeta = BtreeMap.open(this, "old-meta", String.class, String.class);
+        oldMeta.setRoot(oldMetaRootId);
+        Iterator<String> it = oldMeta.keyIterator(null);
+        ArrayList<Integer> oldBlocks = New.arrayList();
+        while (it.hasNext()) {
+            String k = it.next();
+            String v = oldMeta.get(k);
+            log("    " + k + " " + v.replace('\n', ' '));
+            if (k.startsWith("block.")) {
+                String s = oldMeta.get(k);
+                Block b = Block.fromString(s);
+                if (!blocks.containsKey(b.id)) {
+                    oldBlocks.add(b.id);
+                    blocks.put(b.id, b);
+                }
+                continue;
+            }
+            if (!k.startsWith("map.")) {
+                continue;
+            }
+            k = k.substring("map.".length());
+            if (!maps.containsKey(k)) {
+                continue;
+            }
+            String[] d = StringUtils.arraySplit(v, ',', false);
+            Class<?> kt = BtreeMap.getClass(d[1]);
+            Class<?> vt = BtreeMap.getClass(d[2]);
+            BtreeMap<?, ?> oldData = BtreeMap.open(this, "old-" + k, kt, vt);
+            long oldDataRoot = Long.parseLong(d[0]);
+            oldData.setRoot(oldDataRoot);
+            @SuppressWarnings("unchecked")
+            BtreeMap<Object, Object> data = (BtreeMap<Object, Object>) maps.get(k);
+            Iterator<?> dataIt = oldData.keyIterator(null);
+            while (dataIt.hasNext()) {
+                Object o = dataIt.next();
+                int todo;
+                Page p = null; // data.getNode(o);
+                if (p == null) {
+                    // was removed later - ignore
+                } else if (p.getId() < 0) {
+                    // temporarily changed - ok
+                } else {
+                    Block b = getBlock(p.getId());
+                    if (old.contains(b)) {
+                        log("       move key:" + o + " block:" + b.id);
+                        data.remove(o);
+                        int todo2;
+                        // data.put(o, n.getData());
+                    }
+                }
+            }
+        }
+        for (int o : oldBlocks) {
+            blocks.remove(o);
+        }
+    }
+
+    /**
+     * Read a page.
+     *
+     * @param map the map
+     * @param id the page id
+     * @return the page
+     */
+    Page readPage(BtreeMap<?, ?> map, long id) {
+        if (id < 0) {
+            return temp.get((int) (-id - 1));
+        }
+        Page p = cache.get(id);
+        if (p == null) {
+            try {
+                long pos = getPosition(id);
+                file.position(pos);
+                ByteBuffer buff = ByteBuffer.wrap(new byte[1024]);
+                // TODO read fully; read only required bytes
+                do {
+                    int len = file.read(buff);
+                    if (len < 0) {
+                        break;
+                    }
+                } while (buff.remaining() > 0);
+                buff.rewind();
+                p = Page.read(map, id, buff);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            cache.put(id, p);
+        }
+        return p;
+    }
+
+    /**
+     * Remove a page.
+     *
+     * @param id the page id
+     */
+    void removePage(long id) {
+        if (id > 0) {
+            if (getBlock(id).liveCount == 0) {
+                throw new RuntimeException("Negative live count: " + id);
+            }
+            getBlock(id).liveCount--;
+        }
+    }
+
+    private int getBlockId(long pageId) {
+        return (int) (pageId >>> 32);
+    }
+
+    private Block getBlock(long pageId) {
+        return blocks.get(getBlockId(pageId));
+    }
+
+    /**
+     * A block of data.
+     */
+    static class Block implements Comparable<Block> {
+        public int collectPriority;
+        int id;
+        long start;
+        long length;
+        int entryCount;
+        int liveCount;
+
+        Block(int id) {
+            this.id = id;
+        }
+
+        /**
+         * Build a block from the given string.
+         *
+         * @param s the string
+         * @return the block
+         */
+        static Block fromString(String s) {
+            Block b = new Block(0);
+            Properties prop = new Properties();
+            try {
+                prop.load(new ByteArrayInputStream(s.getBytes("UTF-8")));
+                b.id = Integer.parseInt(prop.get("id").toString());
+                b.start = Long.parseLong(prop.get("start").toString());
+                b.length = Long.parseLong(prop.get("length").toString());
+                b.entryCount = Integer.parseInt(prop.get("entryCount").toString());
+                b.liveCount = Integer.parseInt(prop.get("liveCount").toString());
+                return b;
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public int getFillRate() {
+            return entryCount == 0 ? 0 : 100 * liveCount / entryCount;
+        }
+
+        public int compareTo(Block o) {
+            return start == o.start ? 0 : start < o.start ? -1 : 1;
+        }
+
+        public int hashCode() {
+            return id;
+        }
+
+        public boolean equals(Object o) {
+            return o instanceof Block && ((Block) o).id == id;
+        }
+
+        public String toString() {
+            return
+                "id:" + id + "\n" +
+                "start:" + start + "\n" +
+                "length:" + length + "\n" +
+                "entryCount:" + entryCount + "\n" +
+                "liveCount:" + liveCount + "\n";
+        }
+
+    }
+
+    /**
+     * Log the string, if logging is enabled.
+     *
+     * @param string the string to log
+     */
+    public void log(String string) {
+         // System.out.println(string);
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/Page.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.dev.store.btree;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A btree page implementation.
+ */
+class Page {
+
+    private static final int MAX_SIZE = 4;
+
+    private final BtreeMap<?, ?> map;
+    private long id;
+    private long storedId;
+    private long transaction;
+    private Object[] keys;
+    private Object[] values;
+    private long[] children;
+
+    private Page(BtreeMap<?, ?> map) {
+        this.map = map;
+    }
+
+    /**
+     * Create a new page.
+     *
+     * @param map the map
+     * @param key the keys
+     * @param values the values
+     * @return the page
+     */
+    static Page create(BtreeMap<?, ?> map, Object[] keys, Object[] values, long[] children) {
+        Page p = new Page(map);
+        p.keys = keys;
+        p.values = values;
+        p.children = children;
+        p.transaction = map.getTransaction();
+        p.id = map.registerTempPage(p);
+        return p;
+    }
+
+    /**
+     * Read a page.
+     *
+     * @param map the map
+     * @param id the page id
+     * @param buff the source buffer
+     * @return the page
+     */
+    static Page read(BtreeMap<?, ?> map, long id, ByteBuffer buff) {
+        Page p = new Page(map);
+        p.id = id;
+        p.read(buff);
+        return p;
+    }
+
+    private Page copyOnWrite() {
+        long t = map.getTransaction();
+        if (transaction == t) {
+            return this;
+        }
+        map.removePage(id);
+        Page p2 = create(map, keys, values, children);
+        p2.transaction = t;
+        return p2;
+    }
+
+    public String toString() {
+        StringBuilder buff = new StringBuilder();
+        buff.append("nodeId: ").append(id).append("\n");
+        for (int i = 0; i <= keys.length; i++) {
+            if (i > 0) {
+                buff.append(" ");
+            }
+            if (children != null) {
+                buff.append("[" + children[i] + "]");
+            }
+            if (i < keys.length) {
+                buff.append(keys[i]);
+                if (values != null) {
+                    buff.append(':');
+                    buff.append(values[i]);
+                }
+            }
+        }
+        return buff.toString();
+    }
+
+    /**
+     * Get the page id.
+     *
+     * @return the page id
+     */
+    long getId() {
+        return id;
+    }
+
+    /**
+     * Set the page id.
+     *
+     * @param id the new id
+     */
+    void setId(long id) {
+        this.id = id;
+    }
+
+    Object find(Object key) {
+        int x = findKey(key);
+        if (children != null) {
+            if (x < 0) {
+                x = -x - 1;
+            } else {
+                x++;
+            }
+            Page p = map.readPage(children[x]);
+            return p.find(key);
+        }
+        if (x >= 0) {
+            return values[x];
+        }
+        return null;
+    }
+
+    private int findKey(Object key) {
+        int low = 0, high = keys.length - 1;
+        while (low <= high) {
+            int x = (low + high) >>> 1;
+            int compare = map.compare(key, keys[x]);
+            if (compare > 0) {
+                low = x + 1;
+            } else if (compare < 0) {
+                high = x - 1;
+            } else {
+                return x;
+            }
+        }
+        return -(low + 1);
+    }
+
+    private int size() {
+        return keys.length;
+    }
+
+    private boolean isLeaf() {
+        return children == null;
+    }
+
+    private Page split(int at) {
+        int a = at, b = keys.length - a;
+        Object[] aKeys = new Object[a];
+        Object[] bKeys = new Object[b];
+        System.arraycopy(keys, 0, aKeys, 0, a);
+        System.arraycopy(keys, a, bKeys, 0, b);
+        keys = aKeys;
+        Object[] aValues = new Object[a];
+        Object[] bValues = new Object[b];
+        System.arraycopy(values, 0, aValues, 0, a);
+        System.arraycopy(values, a, bValues, 0, b);
+        values = aValues;
+        long[] bChildren;
+        if (children == null) {
+            bChildren = null;
+        } else {
+            a = children.length / 2;
+            b = children.length - a;
+            long[] aChildren = new long[a];
+            bChildren = new long[b];
+            System.arraycopy(children, 0, aChildren, 0, a);
+            System.arraycopy(children, a, bChildren, 0, b);
+            children = aChildren;
+        }
+        Page newPage = create(map, bKeys, bValues, bChildren);
+        return newPage;
+    }
+
+    /**
+     * Add or replace the key-value pair.
+     *
+     * @param map the map
+     * @param p the page
+     * @param key the key
+     * @param data the value
+     * @return the root page
+     */
+    static Page put(BtreeMap<?, ?> map, Page p, Object key, Object value) {
+        if (p == null) {
+            Object[] keys = { key };
+            Object[] values = { value };
+            p = create(map, keys, values, null);
+            return p;
+        }
+        p = p.copyOnWrite();
+        Page top = p;
+        Page parent = null;
+        int parentIndex = 0;
+        while (true) {
+            if (parent != null) {
+                parent.children[parentIndex] = p.id;
+            }
+            int index = p.findKey(key);
+            if (p.isLeaf()) {
+                if (index >= 0) {
+                    p.values[index] = value;
+                } else {
+                    index = -index - 1;
+                    p.insert(index, key, value, 0);
+                    if (p.size() >= MAX_SIZE) {
+                        int pos = p.size() / 2;
+                        Object k = p.keys[pos];
+                        Page split = p.split(pos);
+                        if (parent == null) {
+                            Object[] keys = { k };
+                            long[] children = { p.getId(), split.getId() };
+                            Page newRoot = create(map, keys, null, children);
+                            return newRoot;
+                        }
+                        parent.insert(parentIndex, k, null, split.getId());
+                    }
+                }
+                break;
+            }
+            if (index < 0) {
+                index = -index - 1;
+            }
+            parent = p;
+            parentIndex = index;
+            p = map.readPage(p.children[index]);
+            p = p.copyOnWrite();
+        }
+        return top;
+    }
+
+
+    /**
+     * Remove a key-value pair.
+     *
+     * @param p the root node
+     * @param key the key
+     * @return the new root node
+     */
+    static Page remove(Page p, Object key) {
+        // TODO avoid separate lookup
+        if (p.find(key) == null) {
+            return p;
+        }
+        p = p.copyOnWrite();
+        Page top = p;
+        Page parent = null;
+        int parentIndex = 0;
+        while (true) {
+            if (parent != null) {
+                parent.children[parentIndex] = p.id;
+            }
+            int index = p.findKey(key);
+            if (p.isLeaf()) {
+                if (index >= 0) {
+                    p.remove(index);
+                } else {
+                    // not found?
+                    throw new RuntimeException("Not found: " + key);
+                }
+                if (p.size() == 0) {
+                    if (parent != null) {
+                        parent.remove(parentIndex);
+                        // TODO recursive, or on the way down
+                    }
+                }
+                break;
+            }
+            if (index < 0) {
+                index = -index - 1;
+            }
+            parent = p;
+            parentIndex = index;
+            p = p.map.readPage(p.children[index]);
+            p = p.copyOnWrite();
+        }
+        return top;
+    }
+
+    /**
+     * Remove a key.
+     *
+     * @param key the key
+     * @return the new page or null if the page is now empty
+     */
+    private Page remove(Object key) {
+        int index = findKey(key);
+        if (isLeaf()) {
+            if (index < 0) {
+                // not found
+                return this;
+            }
+        }
+
+        Page p = copyOnWrite();
+        p = p.remove(key);
+        return p;
+    }
+
+    void insert(int index, Object key, Object value, long child) {
+        Object[] newKeys = new Object[keys.length + 1];
+        copyWithGap(keys, newKeys, keys.length, index);
+        newKeys[index] = key;
+        keys = newKeys;
+        if (values != null) {
+            Object[] newValues = new Object[values.length + 1];
+            copyWithGap(values, newValues, values.length, index);
+            newValues[index] = value;
+            values = newValues;
+        }
+        if (children != null) {
+            long[] newChildren = new long[children.length + 1];
+            copyWithGap(children, newChildren, children.length, index);
+            newChildren[index] = child;
+            children = newChildren;
+        }
+    }
+
+    void remove(int index) {
+        Object[] newKeys = new Object[keys.length - 1];
+        copyExcept(keys, newKeys, keys.length, index);
+        keys = newKeys;
+        if (values != null) {
+            Object[] newValues = new Object[values.length - 1];
+            copyExcept(values, newValues, values.length, index);
+            values = newValues;
+        }
+        if (children != null) {
+            long[] newChildren = new long[children.length - 1];
+            copyExcept(children, newChildren, children.length, index);
+            children = newChildren;
+        }
+    }
+
+//        int x = findKey(key);
+//        if (x >= 0) {
+//            return values[x];
+//        }
+//        x = -x - 1;
+//        Page p = map.readPage(children[x]);
+//        return p.find(key);
+//        return null;
+//    }
+
+    private void read(ByteBuffer buff) {
+        boolean node = buff.get() == 1;
+        if (node) {
+            int len = BtreeMap.readVarInt(buff);
+            children = new long[len];
+            keys = new Object[len - 1];
+            for (int i = 0; i < len; i++) {
+                children[i] = buff.getLong();
+                if (i < keys.length) {
+                    keys[i] = map.getKeyType().read(buff);
+                }
+            }
+        } else {
+            int len = BtreeMap.readVarInt(buff);
+            keys = new Object[len];
+            values = new Object[len];
+            for (int i = 0; i < len; i++) {
+                keys[i] = map.getKeyType().read(buff);
+                values[i] = map.getValueType().read(buff);
+            }
+        }
+    }
+
+    /**
+     * Store the page.
+     *
+     * @param buff the target buffer
+     */
+    void write(ByteBuffer buff) {
+        if (children != null) {
+            buff.put((byte) 1);
+            int size = children.length;
+            BtreeMap.writeVarInt(buff, size);
+            for (int i = 0; i < size; i++) {
+                long c = map.readPage(children[i]).storedId;
+                buff.putLong(c);
+                if (i < keys.length) {
+                    map.getKeyType().write(buff, keys[i]);
+                }
+            }
+        } else {
+            buff.put((byte) 0);
+            int size = keys.length;
+            BtreeMap.writeVarInt(buff, size);
+            for (int i = 0; i < size; i++) {
+                map.getKeyType().write(buff, keys[i]);
+                map.getValueType().write(buff, values[i]);
+            }
+        }
+    }
+
+    /**
+     * Get the length in bytes, including temporary children.
+     *
+     * @return the length
+     */
+    int lengthIncludingTempChildren() {
+        int len = length();
+        if (children != null) {
+            int size = children.length;
+            for (int i = 0; i < size; i++) {
+                long c = children[i];
+                if (c < 0) {
+                    len += map.readPage(c).lengthIncludingTempChildren();
+                }
+            }
+        }
+        return len;
+    }
+
+    long updatePageIds(long pageId) {
+        this.storedId = pageId;
+        pageId += length();
+        if (children != null) {
+            int size = children.length;
+            for (int i = 0; i < size; i++) {
+                long c = children[i];
+                if (c < 0) {
+                    pageId = map.readPage(c).updatePageIds(pageId);
+                }
+            }
+        }
+        return pageId;
+    }
+
+    long storeTemp(ByteBuffer buff) {
+        write(buff);
+        if (children != null) {
+            int size = children.length;
+            for (int i = 0; i < size; i++) {
+                long c = children[i];
+                if (c < 0) {
+                    children[i] = map.readPage(c).storeTemp(buff);
+                }
+            }
+        }
+        this.id = storedId;
+        return id;
+    }
+
+    int countTemp() {
+        int count = 1;
+        if (children != null) {
+            int size = children.length;
+            for (int i = 0; i < size; i++) {
+                long c = children[i];
+                if (c < 0) {
+                    count += map.readPage(c).countTemp();
+                }
+            }
+        }
+        return count;
+    }
+
+    /**
+     * Get the length in bytes.
+     *
+     * @return the length
+     */
+    int length() {
+        int len = 1;
+        if (children != null) {
+            int size = children.length;
+            len += BtreeMap.getVarIntLen(size);
+            for (int i = 0; i < size; i++) {
+                len += 8;
+                if (i < keys.length) {
+                    len += map.getKeyType().length(keys[i]);
+                }
+            }
+        } else {
+            int size = keys.length;
+            len += BtreeMap.getVarIntLen(size);
+            for (int i = 0; i < size; i++) {
+                len += map.getKeyType().length(keys[i]);
+                len += map.getValueType().length(values[i]);
+            }
+        }
+        return len;
+    }
+
+    private static void copyWithGap(Object src, Object dst, int oldSize, int gapIndex) {
+        if (gapIndex > 0) {
+            System.arraycopy(src, 0, dst, 0, gapIndex);
+        }
+        if (gapIndex < oldSize) {
+            System.arraycopy(src, gapIndex, dst, gapIndex + 1, oldSize - gapIndex);
+        }
+    }
+
+    private static void copyExcept(Object src, Object dst, int oldSize, int removeIndex) {
+        if (removeIndex > 0 && oldSize > 0) {
+            System.arraycopy(src, 0, dst, 0, removeIndex);
+        }
+        if (removeIndex < oldSize) {
+            System.arraycopy(src, removeIndex + 1, dst, removeIndex, oldSize - removeIndex - 1);
+        }
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/package.html

+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License, Version 1.0,
+and under the Eclipse Public License, Version 1.0
+(http://h2database.com/html/license.html).
+Initial Developer: H2 Group
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /><title>
+Javadoc package documentation
+</title></head><body style="font: 9pt/130% Tahoma, Arial, Helvetica, sans-serif; font-weight: normal;"><p>
+
+A persistent storage for tree maps.
+
+</p></body></html>
\ No newline at end of file