A persistent tree map (work in progress).

h2/src/tools/org/h2/dev/store/btree/IntegerType.java → h2/src/test/org/h2/test/store/IntegerType.java

@@ -4,9 +4,11 @@
  * (http://h2database.com/html/license.html).
  * Initial Developer: H2 Group
  */
-package org.h2.dev.store.btree;
+package org.h2.test.store;
 
 import java.nio.ByteBuffer;
+import org.h2.dev.store.btree.DataType;
+import org.h2.dev.store.btree.DataUtils;
 
 /**
  * An integer type.
@@ -21,6 +23,14 @@ class IntegerType implements DataType {
         return DataUtils.getVarIntLen((Integer) obj);
     }
 
+    public int getMaxLength(Object obj) {
+        return DataUtils.MAX_VAR_INT_LEN;
+    }
+
+    public int getMemory(Object obj) {
+        return 20;
+    }
+
     public Integer read(ByteBuffer buff) {
         return DataUtils.readVarInt(buff);
     }

h2/src/test/org/h2/test/store/RowType.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.store;
+
+import java.nio.ByteBuffer;
+import org.h2.dev.store.btree.DataType;
+import org.h2.dev.store.btree.DataTypeFactory;
+import org.h2.dev.store.btree.DataUtils;
+import org.h2.util.StringUtils;
+
+/**
+ * A row type.
+ */
+public class RowType implements DataType {
+
+    private final DataType[] types;
+
+    private RowType(DataType[] types) {
+        this.types = types;
+    }
+
+    public int compare(Object a, Object b) {
+        if (a == b) {
+            return 0;
+        }
+        Object[] ax = (Object[]) a;
+        Object[] bx = (Object[]) b;
+        int al = ax.length;
+        int bl = bx.length;
+        int len = Math.min(al, bl);
+        for (int i = 0; i < len; i++) {
+            int comp = types[i].compare(ax[i], bx[i]);
+            if (comp != 0) {
+                return comp;
+            }
+        }
+        if (len < al) {
+            return -1;
+        } else if (len < bl) {
+            return 1;
+        }
+        return 0;
+    }
+
+    public int length(Object obj) {
+        Object[] x = (Object[]) obj;
+        int len = x.length;
+        int result = DataUtils.getVarIntLen(len);
+        for (int i = 0; i < len; i++) {
+            result += types[i].length(x[i]);
+        }
+        return result;
+    }
+
+    public int getMaxLength(Object obj) {
+        Object[] x = (Object[]) obj;
+        int len = x.length;
+        int result = DataUtils.MAX_VAR_INT_LEN;
+        for (int i = 0; i < len; i++) {
+            result += types[i].getMaxLength(x[i]);
+        }
+        return result;
+    }
+
+    public int getMemory(Object obj) {
+        Object[] x = (Object[]) obj;
+        int len = x.length;
+        int memory = 0;
+        for (int i = 0; i < len; i++) {
+            memory += types[i].getMemory(x[i]);
+        }
+        return memory;
+    }
+
+    public Object[] read(ByteBuffer buff) {
+        int len = DataUtils.readVarInt(buff);
+        Object[] x = new Object[len];
+        for (int i = 0; i < len; i++) {
+            x[i] = types[i].read(buff);
+        }
+        return x;
+    }
+
+    public void write(ByteBuffer buff, Object obj) {
+        Object[] x = (Object[]) obj;
+        int len = x.length;
+        DataUtils.writeVarInt(buff, len);
+        for (int i = 0; i < len; i++) {
+            types[i].write(buff, x[i]);
+        }
+    }
+
+    public String asString() {
+        StringBuilder buff = new StringBuilder();
+        buff.append('r');
+        buff.append('(');
+        for (int i = 0; i < types.length; i++) {
+            if (i > 0) {
+                buff.append(',');
+            }
+            buff.append(types[i].asString());
+        }
+        buff.append(')');
+        return buff.toString();
+    }
+
+    static RowType fromString(String t, DataTypeFactory factory) {
+        if (!t.startsWith("r(") || !t.endsWith(")")) {
+            throw new RuntimeException("Unknown type: " + t);
+        }
+        t = t.substring(2, t.length() - 1);
+        String[] array = StringUtils.arraySplit(t, ',', false);
+        DataType[] types = new DataType[array.length];
+        for (int i = 0; i < array.length; i++) {
+            types[i] = factory.fromString(array[i]);
+        }
+        return new RowType(types);
+    }
+
+}

h2/src/test/org/h2/test/store/TestTreeMapStore.java

@@ -29,6 +29,7 @@ public class TestTreeMapStore extends TestBase {
     }
 
     public void test() throws Exception {
+        testLargeImport();
         testBtreeStore();
         testDefragment();
         testReuseSpace();
@@ -38,10 +39,39 @@ public class TestTreeMapStore extends TestBase {
         testSimple();
     }
 
+    private void testLargeImport() throws Exception {
+        String fileName = getBaseDir() + "/testCsv.h3";
+        int len = 10000;
+        for (int j = 0; j < 5; j++) {
+            FileUtils.delete(fileName);
+            BtreeMapStore s = openStore(fileName);
+            s.setMaxPageSize(40);
+            RowType rowType = RowType.fromString("r(i,,)", new TestTypeFactory());
+            BtreeMap<Integer, Object[]> m = s.openMap("data", new IntegerType(), rowType);
+            int i = 0;
+            // long t = System.currentTimeMillis();
+            for (; i < len;) {
+              Object[] o = new Object[3];
+              o[0] = i;
+              o[1] = "Hello";
+              o[2] = "World";
+              m.put(i, o);
+              i++;
+              if (i % 10000 == 0) {
+                  s.store();
+              }
+            }
+            s.store();
+            s.close();
+            // System.out.println("store time " + (System.currentTimeMillis() - t));
+            // System.out.println("store size " + FileUtils.size(fileName));
+        }
+    }
+
     private void testBtreeStore() {
         String fileName = getBaseDir() + "/testBtreeStore.h3";
         FileUtils.delete(fileName);
-        BtreeMapStore s = BtreeMapStore.open(fileName);
+        BtreeMapStore s = openStore(fileName);
         BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
         int count = 2000;
         // Profiler p = new Profiler();
@@ -64,8 +94,9 @@ public class TestTreeMapStore extends TestBase {
         for (int i = 1; i < count; i++) {
             assertEquals("hello " + i, m.get(i));
         }
+        s.store();
         s.close();
-        s = BtreeMapStore.open(fileName);
+        s = openStore(fileName);
         m = s.openMap("data", Integer.class, String.class);
         assertNull(m.get(0));
         for (int i = 1; i < count; i++) {
@@ -87,7 +118,7 @@ public class TestTreeMapStore extends TestBase {
         FileUtils.delete(fileName);
         long initialLength = 0;
         for (int j = 0; j < 20; j++) {
-            BtreeMapStore s = BtreeMapStore.open(fileName);
+            BtreeMapStore s = openStore(fileName);
             BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
             for (int i = 0; i < 10; i++) {
                 m.put(j + i, "Hello " + j);
@@ -103,9 +134,9 @@ public class TestTreeMapStore extends TestBase {
                 assertTrue("initial: " + initialLength + " len: " + len, len <= initialLength * 3);
             }
         }
-        long len = FileUtils.size(fileName);
+        // long len = FileUtils.size(fileName);
         // System.out.println("len0: " + len);
-        BtreeMapStore s = BtreeMapStore.open(fileName);
+        BtreeMapStore s = openStore(fileName);
         BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
         for (int i = 0; i < 100; i++) {
             m.remove(i);
@@ -113,13 +144,13 @@ public class TestTreeMapStore extends TestBase {
         s.store();
         s.compact();
         s.close();
-        len = FileUtils.size(fileName);
+        // len = FileUtils.size(fileName);
         // System.out.println("len1: " + len);
-        s = BtreeMapStore.open(fileName);
+        s = openStore(fileName);
         m = s.openMap("data", Integer.class, String.class);
         s.compact();
         s.close();
-        len = FileUtils.size(fileName);
+        // len = FileUtils.size(fileName);
         // System.out.println("len2: " + len);
     }
 
@@ -127,14 +158,15 @@ public class TestTreeMapStore extends TestBase {
         String fileName = getBaseDir() + "/testReuseSpace.h3";
         FileUtils.delete(fileName);
         long initialLength = 0;
-        for (int j = 0; j < 10; j++) {
-            BtreeMapStore s = BtreeMapStore.open(fileName);
+        for (int j = 0; j < 20; j++) {
+            BtreeMapStore s = openStore(fileName);
             BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
             for (int i = 0; i < 10; i++) {
                 m.put(i, "Hello");
             }
             s.store();
             for (int i = 0; i < 10; i++) {
+                assertEquals("Hello", m.get(i));
                 m.remove(i);
             }
             s.store();
@@ -151,23 +183,25 @@ public class TestTreeMapStore extends TestBase {
     private void testRandom() {
         String fileName = getBaseDir() + "/testRandom.h3";
         FileUtils.delete(fileName);
-        BtreeMapStore s = BtreeMapStore.open(fileName);
+        BtreeMapStore s = openStore(fileName);
         BtreeMap<Integer, Integer> m = s.openMap("data", Integer.class, Integer.class);
         TreeMap<Integer, Integer> map = new TreeMap<Integer, Integer>();
         Random r = new Random(1);
-        int operationCount = 1000;
-        int maxValue = 20;
+        int operationCount = 10000;
+        int maxValue = 30;
         for (int i = 0; i < operationCount; i++) {
             int k = r.nextInt(maxValue);
             int v = r.nextInt();
             boolean compareAll;
             switch (r.nextInt(3)) {
             case 0:
+                log(i + ": put " + k + " = " + v);
                 m.put(k, v);
                 map.put(k, v);
                 compareAll = true;
                 break;
             case 1:
+                log(i + ": remove " + k);
                 m.remove(k);
                 map.remove(k);
                 compareAll = true;
@@ -185,10 +219,12 @@ public class TestTreeMapStore extends TestBase {
             }
             if (compareAll) {
                 Iterator<Integer> it = m.keyIterator(null);
-                Iterator<Integer> it2 = map.keySet().iterator();
-                while (it2.hasNext()) {
+                Iterator<Integer> itExpected = map.keySet().iterator();
+                while (itExpected.hasNext()) {
                     assertTrue(it.hasNext());
-                    assertEquals(it2.next(), it.next());
+                    Integer expected = itExpected.next();
+                    Integer got = it.next();
+                    assertEquals(expected, got);
                 }
                 assertFalse(it.hasNext());
             }
@@ -199,7 +235,7 @@ public class TestTreeMapStore extends TestBase {
     private void testKeyValueClasses() {
         String fileName = getBaseDir() + "/testKeyValueClasses.h3";
         FileUtils.delete(fileName);
-        BtreeMapStore s = BtreeMapStore.open(fileName);
+        BtreeMapStore s = openStore(fileName);
         BtreeMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
         is.put(1, "Hello");
         BtreeMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
@@ -220,8 +256,9 @@ public class TestTreeMapStore extends TestBase {
         } catch (RuntimeException e) {
             // expected
         }
+        s.store();
         s.close();
-        s = BtreeMapStore.open(fileName);
+        s = openStore(fileName);
         is = s.openMap("intString", Integer.class, String.class);
         assertEquals("Hello", is.get(1));
         ii = s.openMap("intInt", Integer.class, Integer.class);
@@ -236,7 +273,7 @@ public class TestTreeMapStore extends TestBase {
     private void testIterate() {
         String fileName = getBaseDir() + "/testIterate.h3";
         FileUtils.delete(fileName);
-        BtreeMapStore s = BtreeMapStore.open(fileName);
+        BtreeMapStore s = openStore(fileName);
         BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
         Iterator<Integer> it = m.keyIterator(null);
         assertFalse(it.hasNext());
@@ -269,7 +306,7 @@ public class TestTreeMapStore extends TestBase {
     private void testSimple() {
         String fileName = getBaseDir() + "/testSimple.h3";
         FileUtils.delete(fileName);
-        BtreeMapStore s = BtreeMapStore.open(fileName);
+        BtreeMapStore s = openStore(fileName);
         BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
         for (int i = 0; i < 3; i++) {
             m.put(i, "hello " + i);
@@ -281,10 +318,10 @@ public class TestTreeMapStore extends TestBase {
         for (int i = 1; i < 3; i++) {
             assertEquals("hello " + i, m.get(i));
         }
-
+        s.store();
         s.close();
 
-        s = BtreeMapStore.open(fileName);
+        s = openStore(fileName);
         m = s.openMap("data", Integer.class, String.class);
         assertNull(m.get(0));
         for (int i = 1; i < 3; i++) {
@@ -293,4 +330,19 @@ public class TestTreeMapStore extends TestBase {
         s.close();
     }
 
+    private static BtreeMapStore openStore(String fileName) {
+        BtreeMapStore store = BtreeMapStore.open(fileName, new TestTypeFactory());
+        store.setMaxPageSize(10);
+        return store;
+    }
+
+    /**
+     * Log the message.
+     *
+     * @param msg the message
+     */
+    private static void log(String msg) {
+        // System.out.println(msg);
+    }
+
 }

h2/src/test/org/h2/test/store/TestTypeFactory.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License, Version
+ * 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html). Initial Developer: H2 Group
+ */
+package org.h2.test.store;
+
+import org.h2.dev.store.btree.DataType;
+import org.h2.dev.store.btree.DataTypeFactory;
+import org.h2.dev.store.btree.StringType;
+
+/**
+ * A data type factory.
+ */
+public class TestTypeFactory implements DataTypeFactory {
+
+    public DataType fromString(String s) {
+        if (s.length() == 0) {
+            return new StringType();
+        }
+        switch (s.charAt(0)) {
+        case 'i':
+            return new IntegerType();
+        case 'r':
+            return RowType.fromString(s, this);
+        }
+        throw new RuntimeException("Unknown data type " + s);
+    }
+
+    public DataType getDataType(Class<?> objectClass) {
+        if (objectClass == Integer.class) {
+            return new IntegerType();
+        }
+        throw new RuntimeException("Unsupported object class " + objectClass.toString());
+    }
+
+}

h2/src/test/org/h2/test/store/package.html

+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License, Version 1.0,
+and under the Eclipse Public License, Version 1.0
+(http://h2database.com/html/license.html).
+Initial Developer: H2 Group
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /><title>
+Javadoc package documentation
+</title></head><body style="font: 9pt/130% Tahoma, Arial, Helvetica, sans-serif; font-weight: normal;"><p>
+
+This package contains tests for the map store.
+
+</p></body></html>
\ No newline at end of file

h2/src/tools/org/h2/dev/store/btree/BtreeMap.java

@@ -17,13 +17,15 @@ import java.util.Iterator;
 public class BtreeMap<K, V> {
 
     private final BtreeMapStore store;
+    private final long id;
     private final String name;
     private final DataType keyType;
     private final DataType valueType;
     private Page root;
 
-    private BtreeMap(BtreeMapStore store, String name, DataType keyType, DataType valueType) {
+    private BtreeMap(BtreeMapStore store, long id, String name, DataType keyType, DataType valueType) {
         this.store = store;
+        this.id = id;
         this.name = name;
         this.keyType = keyType;
         this.valueType = valueType;
@@ -40,8 +42,8 @@ public class BtreeMap<K, V> {
      * @param valueClass the value class
      * @return the map
      */
-    static <K, V> BtreeMap<K, V> open(BtreeMapStore store, String name, DataType keyType, DataType valueType) {
-        return new BtreeMap<K, V>(store, name, keyType, valueType);
+    static <K, V> BtreeMap<K, V> open(BtreeMapStore store, long id, String name, DataType keyType, DataType valueType) {
+        return new BtreeMap<K, V>(store, id, name, keyType, valueType);
     }
 
     /**
@@ -51,9 +53,7 @@ public class BtreeMap<K, V> {
      * @param data the value
      */
     public void put(K key, V data) {
-        if (!isChanged()) {
-            store.markChanged(name, this);
-        }
+        markChanged();
         root = Page.put(this, root, key, data);
     }
 
@@ -77,23 +77,32 @@ public class BtreeMap<K, V> {
      * @param key the key
      * @return the value, or null if not found
      */
-    public Page getPage(K key) {
+    Page getPage(K key) {
         if (root == null) {
             return null;
         }
         return root.findPage(key);
     }
 
+    /**
+     * Remove all entries.
+     */
+    public void clear() {
+        if (root != null) {
+            markChanged();
+            root.removeAllRecursive();
+            root = null;
+        }
+    }
+
     /**
      * Remove a key-value pair.
      *
      * @param key the key
      */
     public void remove(K key) {
-        if (!isChanged()) {
-            store.markChanged(name, this);
-        }
         if (root != null) {
+            markChanged();
             root = Page.remove(root, key);
         }
     }
@@ -107,6 +116,12 @@ public class BtreeMap<K, V> {
         return root != null && root.getId() < 0;
     }
 
+    private void markChanged() {
+        if (!isChanged()) {
+            store.markChanged(name, this);
+        }
+    }
+
     /**
      * Compare two keys.
      *
@@ -206,4 +221,12 @@ public class BtreeMap<K, V> {
         return name;
     }
 
+    int getMaxPageSize() {
+        return store.getMaxPageSize();
+    }
+
+    long getId() {
+        return id;
+    }
+
 }

h2/src/tools/org/h2/dev/store/btree/BtreeMapStore.java

@@ -6,8 +6,8 @@
  */
 package org.h2.dev.store.btree;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.StringReader;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
@@ -17,7 +17,6 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Properties;
-import java.util.TreeMap;
 import org.h2.dev.store.FilePathCache;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
@@ -35,21 +34,21 @@ header
 
 header:
 # H3 store #
-pageSize=4096
-r=1
+blockSize=4096
 
 chunk:
-'d' [id] [metaRootPos] data ...
+1 byte: 'c'
+4 bytes: length
+4 bytes: chunk id (an incrementing number)
+4 bytes: metaRootPos (relative to the chunk start)
+data ...
 
 todo:
 
 - garbage collection
-
 - use page checksums
-
 - compress chunks
-
-- encode length in pos (1=32, 2=128, 3=512,...)
+- possibly encode the length in pos (1=32, 2=128, 3=512,...)
 
 - floating header (avoid duplicate header)
     for each chunk, store chunk (a counter)
@@ -66,27 +65,35 @@ public class BtreeMapStore {
     private static final StringType STRING_TYPE = new StringType();
 
     private final String fileName;
+    private final DataTypeFactory typeFactory;
+
     private FileChannel file;
-    private int pageSize = 4 * 1024;
-    private long rootBlockStart;
-    private HashMap<Long, Page> cache = SmallLRUCache.newInstance(50000);
-    private ArrayList<Page> temp = New.arrayList();
-    private TreeMap<Integer, Block> blocks = new TreeMap<Integer, Block>();
-    private BtreeMap<String, String> meta;
+    private int blockSize = 4 * 1024;
+    private long rootChunkPos;
+    private HashMap<Long, Page> cache = SmallLRUCache.newInstance(5000);
+    private HashMap<Long, Page> temp = New.hashMap();
+    private HashMap<Integer, Chunk> chunks = New.hashMap();
     private HashMap<String, BtreeMap<?, ?>> maps = New.hashMap();
     private HashMap<String, BtreeMap<?, ?>> mapsChanged = New.hashMap();
+    private BtreeMap<String, String> meta;
 
     // TODO use an int instead? (with rollover to 0)
     private long transaction;
+    private long tempPageId;
+
+    private int lastChunkId;
+
+    // TODO use bit set, and integer instead of long
+    private long lastMapId;
 
-    private int lastBlockId;
+    private int maxPageSize = 30;
 
-    // TODO support quota (per map, per storage)
+    // TODO support reading metadata to support quota (per map, per storage)
     // TODO support r-tree
-    // TODO support triggers and events (possibly on a different layer)
 
-    private BtreeMapStore(String fileName) {
+    private BtreeMapStore(String fileName, DataTypeFactory typeFactory) {
         this.fileName = fileName;
+        this.typeFactory = typeFactory;
     }
 
     /**
@@ -96,7 +103,18 @@ public class BtreeMapStore {
      * @return the store
      */
     public static BtreeMapStore open(String fileName) {
-        BtreeMapStore s = new BtreeMapStore(fileName);
+        return open(fileName, null);
+    }
+
+    /**
+     * Open a tree store.
+     *
+     * @param fileName the file name
+     * @param typeFactory the type factory
+     * @return the store
+     */
+    public static BtreeMapStore open(String fileName, DataTypeFactory typeFactory) {
+        BtreeMapStore s = new BtreeMapStore(fileName, typeFactory);
         s.open();
         return s;
     }
@@ -107,43 +125,74 @@ public class BtreeMapStore {
      * @param <K> the key type
      * @param <V> the value type
      * @param name the name of the map
-     * @param keyClass the key class
-     * @param valueClass the value class
+     * @param keyType the key type
+     * @param valueType the value type
      * @return the map
      */
-    public <K, V> BtreeMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
+    public <K, V> BtreeMap<K, V> openMap(String name, DataType keyType, DataType valueType) {
         @SuppressWarnings("unchecked")
         BtreeMap<K, V> m = (BtreeMap<K, V>) maps.get(name);
         if (m == null) {
-            String root = meta.get("root." + name);
-            DataType keyType = getDataType(keyClass);
-            DataType valueType = getDataType(valueClass);
-            m = BtreeMap.open(this, name, keyType, valueType);
-            maps.put(name, m);
-            if (root == null) {
-                String info = m.getKeyType().asString() + "/" + m.getValueType().asString();
-                meta.put("map." + name, info);
+            String identifier = meta.get("map." + name);
+            long id;
+            String root;
+            if (identifier == null) {
+                id = ++lastMapId;
+                String types = id + "/" + keyType.asString() + "/" + valueType.asString();
+                meta.put("map." + name, types);
+                root = null;
             } else {
-                if (!root.equals("0")) {
+                String types = meta.get("map." + name);
+                String[] idTypeList = StringUtils.arraySplit(types, '/', false);
+                id = Long.parseLong(idTypeList[0]);
+                keyType = getDataType(idTypeList[1]);
+                valueType = getDataType(idTypeList[2]);
+                root = meta.get("root." + id);
+            }
+            m = BtreeMap.open(this, id, name, keyType, valueType);
+            maps.put(name, m);
+            if (root != null && !"0".equals(root)) {
                 m.setRoot(Long.parseLong(root));
             }
         }
-        }
         return m;
     }
 
+    /**
+     * Open a map.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     * @param name the name of the map
+     * @param keyClass the key class
+     * @param valueClass the value class
+     * @return the map
+     */
+    public <K, V> BtreeMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
+        DataType keyType = getDataType(keyClass);
+        DataType valueType = getDataType(valueClass);
+        return openMap(name, keyType, valueType);
+    }
+
     private DataType getDataType(Class<?> clazz) {
         if (clazz == String.class) {
             return STRING_TYPE;
         }
-        return DataTypeFactory.getDataType(clazz);
+        return getTypeFactory().getDataType(clazz);
     }
 
     private DataType getDataType(String s) {
         if (s.equals("")) {
             return STRING_TYPE;
         }
-        return DataTypeFactory.fromString(s);
+        return getTypeFactory().fromString(s);
+    }
+
+    private DataTypeFactory getTypeFactory() {
+        if (typeFactory == null) {
+            throw new RuntimeException("No data type factory set");
+        }
+        return typeFactory;
     }
 
     /**
@@ -159,7 +208,7 @@ public class BtreeMapStore {
     }
 
     private void open() {
-        meta = BtreeMap.open(this, "meta", STRING_TYPE, STRING_TYPE);
+        meta = BtreeMap.open(this, 0, "meta", STRING_TYPE, STRING_TYPE);
         FileUtils.createDirectories(FileUtils.getParent(fileName));
         try {
             log("file open");
@@ -176,21 +225,24 @@ public class BtreeMapStore {
     }
 
     private void readMeta() {
-        long rootId = readMetaRootId(rootBlockStart);
-        lastBlockId = getBlockId(rootId);
-        Block b = new Block(lastBlockId);
-        b.start = rootBlockStart;
-        blocks.put(b.id, b);
-        meta.setRoot(rootId);
-        Iterator<String> it = meta.keyIterator("block.");
+        Chunk header = readChunkHeader(rootChunkPos);
+        lastChunkId = header.id;
+        chunks.put(header.id, header);
+        meta.setRoot(getId(header.id, header.metaRootOffset));
+        Iterator<String> it = meta.keyIterator("chunk.");
         while (it.hasNext()) {
             String s = it.next();
-            if (!s.startsWith("block.")) {
+            if (!s.startsWith("chunk.")) {
                 break;
             }
-            b = Block.fromString(meta.get(s));
-            lastBlockId = Math.max(b.id, lastBlockId);
-            blocks.put(b.id, b);
+            Chunk c = Chunk.fromString(meta.get(s));
+            if (c.id == header.id) {
+                c.start = header.start;
+                c.length = header.length;
+                c.metaRootOffset = header.metaRootOffset;
+            }
+            lastChunkId = Math.max(c.id, lastChunkId);
+            chunks.put(c.id, c);
         }
     }
 
@@ -198,13 +250,15 @@ public class BtreeMapStore {
         try {
             ByteBuffer header = ByteBuffer.wrap((
                 "# H2 1.5\n" +
-                "read-version: 1\n" +
-                "write-version: 1\n" +
-                "rootBlock: " + rootBlockStart + "\n" +
-                "transaction: " + transaction + "\n").getBytes());
+                "versionRead:1\n" +
+                "versionWrite:1\n" +
+                "blockSize:" + blockSize + "\n" +
+                "rootChunk:" + rootChunkPos + "\n" +
+                "lastMapId:" + lastMapId + "\n" +
+                "transaction:" + transaction + "\n").getBytes("UTF-8"));
             file.position(0);
             file.write(header);
-            file.position(pageSize);
+            file.position(blockSize);
             file.write(header);
         } catch (Exception e) {
             throw convert(e);
@@ -214,13 +268,14 @@ public class BtreeMapStore {
     private void readHeader() {
         try {
             file.position(0);
-            byte[] header = new byte[pageSize];
+            byte[] header = new byte[blockSize];
             // TODO read fully; read both headers
             file.read(ByteBuffer.wrap(header));
             Properties prop = new Properties();
-            prop.load(new ByteArrayInputStream(header));
-            rootBlockStart = Long.parseLong(prop.get("rootBlock").toString());
+            prop.load(new StringReader(new String(header, "UTF-8")));
+            rootChunkPos = Long.parseLong(prop.get("rootChunk").toString());
             transaction = Long.parseLong(prop.get("transaction").toString());
+            lastMapId = Long.parseLong(prop.get("lastMapId").toString());
         } catch (Exception e) {
             throw convert(e);
         }
@@ -234,7 +289,6 @@ public class BtreeMapStore {
      * Close the file. Uncommitted changes are ignored.
      */
     public void close() {
-        store();
         if (file != null) {
             try {
                 log("file close");
@@ -246,18 +300,18 @@ public class BtreeMapStore {
         }
     }
 
-    private long getPosition(long pageId) {
-        Block b = getBlock(pageId);
-        if (b == null) {
-            throw new RuntimeException("Block " + getBlockId(pageId) + " not found");
+    private long getPosition(long posId) {
+        Chunk c = getChunk(posId);
+        if (c == null) {
+            throw new RuntimeException("Chunk " + getChunkId(posId) + " not found");
         }
-        long pos = b.start;
-        pos += (int) (pageId & Integer.MAX_VALUE);
+        long pos = c.start;
+        pos += (int) (posId & Integer.MAX_VALUE);
         return pos;
     }
 
-    private static long getId(int blockId, int offset) {
-        return ((long) blockId << 32) | offset;
+    private static long getId(int chunkId, int offset) {
+        return ((long) chunkId << 32) | offset;
     }
 
     /**
@@ -272,95 +326,74 @@ public class BtreeMapStore {
         }
         long trans = commit();
 
-        // the length estimate is not correct,
-        // as we don't know the exact positions and entry counts
-        int lenEstimate = 1 + 8;
-        for (BtreeMap<?, ?> m : mapsChanged.values()) {
-            meta.put("root." + m.getName(), String.valueOf(Long.MAX_VALUE));
-            Page p = m.getRoot();
-            if (p != null) {
-                lenEstimate += p.lengthIncludingTempChildren();
-            }
-        }
-        int blockId = ++lastBlockId;
-        Block b = new Block(blockId);
-        b.start = Long.MAX_VALUE;
-        b.entryCount = Integer.MAX_VALUE;
-        b.liveCount = Integer.MAX_VALUE;
-        blocks.put(b.id, b);
-        for (Block x : blocks.values()) {
+        int chunkId = ++lastChunkId;
+
+        Chunk c = new Chunk(chunkId);
+        c.entryCount = Integer.MAX_VALUE;
+        c.liveCount = Integer.MAX_VALUE;
+        c.start = Long.MAX_VALUE;
+        c.length = Long.MAX_VALUE;
+        chunks.put(c.id, c);
+        meta.put("chunk." + c.id, c.toString());
+        ArrayList<Integer> removedChunks = New.arrayList();
+        for (Chunk x : chunks.values()) {
             if (x.liveCount == 0) {
-                meta.remove("block." + x.id);
+                meta.remove("chunk." + x.id);
+                removedChunks.add(x.id);
             } else {
-                meta.put("block." + x.id, "temp-" + x.toString());
-            }
+                meta.put("chunk." + x.id, x.toString());
             }
-        // modifying the meta can itself affect the metadata
-        // TODO solve this in a better way
-        ArrayList<Integer> removedBlocks = New.arrayList();
-        for (Block x : new ArrayList<Block>(blocks.values())) {
-            if (x.liveCount == 0) {
-                meta.remove("block." + x.id);
-                removedBlocks.add(x.id);
-            } else {
-                meta.put("block." + x.id, x.toString());
-            }
-        }
-        lenEstimate += meta.getRoot().lengthIncludingTempChildren();
-        b.length = lenEstimate;
-
-        blocks.remove(b.id);
-        long storePos = allocateBlock(lenEstimate);
-        blocks.put(b.id, b);
-
-        for (int id : removedBlocks) {
-            blocks.remove(id);
-        }
-
-        long pageId = getId(blockId, 1 + 8);
-        for (BtreeMap<?, ?> m : mapsChanged.values()) {
-            Page r = m.getRoot();
-            long p = r == null ? 0 : pageId;
-            meta.put("root." + m.getName(), "" + p);
-            if (r != null) {
-                pageId = r.updatePageIds(pageId);
         }
+        for (int x : removedChunks) {
+            chunks.remove(x);
         }
-        int metaRootOffset = (int) (pageId - getId(blockId, 0));
-        // add a dummy entry so the count is correct
-        meta.put("block." + b.id, b.toString());
         int count = 0;
+        int maxLength = 1 + 4 + 4 + 4;
         for (BtreeMap<?, ?> m : mapsChanged.values()) {
             Page p = m.getRoot();
             if (p != null) {
-                count += p.countTemp();
+                maxLength += p.getMaxLengthTempRecursive();
+                count += p.countTempRecursive();
+                meta.put("root." + m.getId(), String.valueOf(Long.MAX_VALUE));
+            } else {
+                meta.put("root." + m.getId(), "0");
             }
         }
-        count += meta.getRoot().countTemp();
-
-        b.start = storePos;
-        b.entryCount = count;
-        b.liveCount = b.entryCount;
-
-        meta.put("block." + b.id, b.toString());
+        maxLength += meta.getRoot().getMaxLengthTempRecursive();
+        count += meta.getRoot().countTempRecursive();
 
-        pageId = meta.getRoot().updatePageIds(pageId);
-        int len = (int) (pageId - getId(blockId, 0));
-
-        ByteBuffer buff = ByteBuffer.allocate(len);
-        buff.put((byte) 'd');
-        buff.putInt(b.id);
-        buff.putInt(metaRootOffset);
+        ByteBuffer buff = ByteBuffer.allocate(maxLength);
+        // need to patch the header later
+        buff.put((byte) 'c');
+        buff.putInt(0);
+        buff.putInt(0);
+        buff.putInt(0);
+        long idOffset = getId(chunkId, 0);
         for (BtreeMap<?, ?> m : mapsChanged.values()) {
             Page p = m.getRoot();
             if (p != null) {
-                p.storeTemp(buff);
-            }
+                long root = p.writeTempRecursive(buff, idOffset);
+                meta.put("root." + m.getId(), "" + root);
             }
-        meta.getRoot().storeTemp(buff);
-        if (buff.hasRemaining()) {
-            throw new RuntimeException("remaining: " + buff.remaining());
         }
+
+        // fix metadata
+        c.entryCount = count;
+        c.liveCount = count;
+        meta.put("chunk." + c.id, c.toString());
+
+        meta.getRoot().writeTempRecursive(buff, idOffset);
+
+        buff.flip();
+        int length = buff.limit();
+        long storePos = allocateChunk(length);
+        int rootOffset = (int) (meta.getRoot().getId() - idOffset);
+
+        buff.rewind();
+        buff.put((byte) 'c');
+        buff.putInt(length);
+        buff.putInt(chunkId);
+        buff.putInt(rootOffset);
         buff.rewind();
         try {
             file.position(storePos);
@@ -368,26 +401,33 @@ public class BtreeMapStore {
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        rootBlockStart = storePos;
+        rootChunkPos = storePos;
         writeHeader();
         mapsChanged.clear();
         temp.clear();
+        tempPageId = 0;
+
+        // update the start position and length
+        c.start = storePos;
+        c.length = length;
+        meta.put("chunk." + c.id, c.toString());
+
         return trans;
     }
 
-    private long allocateBlock(long length) {
+    private long allocateChunk(long length) {
         BitSet set = new BitSet();
         set.set(0);
         set.set(1);
-        for (Block b : blocks.values()) {
-            if (b.start == Long.MAX_VALUE) {
+        for (Chunk c : chunks.values()) {
+            if (c.start == Long.MAX_VALUE) {
                 continue;
             }
-            int first = (int) (b.start / pageSize);
-            int last = (int) ((b.start + b.length) / pageSize);
+            int first = (int) (c.start / blockSize);
+            int last = (int) ((c.start + c.length) / blockSize);
             set.set(first, last +1);
         }
-        int required = (int) (length / pageSize) + 1;
+        int required = (int) (length / blockSize) + 1;
         for (int i = 0; i < set.size(); i++) {
             if (!set.get(i)) {
                 boolean ok = true;
@@ -398,11 +438,11 @@ public class BtreeMapStore {
                     }
                 }
                 if (ok) {
-                    return i * pageSize;
+                    return i * blockSize;
                 }
             }
         }
-        return set.size() * pageSize;
+        return set.size() * blockSize;
     }
 
     /**
@@ -421,8 +461,9 @@ public class BtreeMapStore {
      * @return the page id
      */
     long registerTempPage(Page p) {
-        temp.add(p);
-        return -temp.size();
+        long id = --tempPageId;
+        temp.put(id, p);
+        return id;
     }
 
     /**
@@ -434,37 +475,42 @@ public class BtreeMapStore {
         return ++transaction;
     }
 
-    private long readMetaRootId(long blockStart) {
+    private Chunk readChunkHeader(long pos) {
         try {
-            file.position(blockStart);
+            file.position(pos);
             ByteBuffer buff = ByteBuffer.wrap(new byte[16]);
             file.read(buff);
             buff.rewind();
-            if (buff.get() != 'd') {
+            if (buff.get() != 'c') {
                 throw new RuntimeException("File corrupt");
             }
-            int blockId = buff.getInt();
+            int length = buff.getInt();
+            int chunkId = buff.getInt();
             int offset = buff.getInt();
-            return getId(blockId, offset);
+            Chunk c = new Chunk(chunkId);
+            c.start = pos;
+            c.length = length;
+            c.metaRootOffset = offset;
+            return c;
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
     }
 
     /**
-     * Try to reduce the file size. Blocks with a low number of live items will
+     * Try to reduce the file size. Chunks with a low number of live items will
      * be re-written.
      */
     public void compact() {
-        if (blocks.size() <= 1) {
+        if (chunks.size() <= 1) {
             return;
         }
         long liveCountTotal = 0, entryCountTotal = 0;
-        for (Block b : blocks.values()) {
-            entryCountTotal += b.entryCount;
-            liveCountTotal += b.liveCount;
+        for (Chunk c : chunks.values()) {
+            entryCountTotal += c.entryCount;
+            liveCountTotal += c.liveCount;
         }
-        int averageEntryCount = (int) (entryCountTotal / blocks.size());
+        int averageEntryCount = (int) (entryCountTotal / chunks.size());
         if (entryCountTotal == 0) {
             return;
         }
@@ -472,52 +518,51 @@ public class BtreeMapStore {
         if (percentTotal > 80) {
             return;
         }
-        ArrayList<Block> old = New.arrayList();
-        for (Block b : blocks.values()) {
-            int age = lastBlockId - b.id + 1;
-            b.collectPriority = b.getFillRate() / age;
-            old.add(b);
+        ArrayList<Chunk> old = New.arrayList();
+        for (Chunk c : chunks.values()) {
+            int age = lastChunkId - c.id + 1;
+            c.collectPriority = c.getFillRate() / age;
+            old.add(c);
         }
-        Collections.sort(old, new Comparator<Block>() {
-            public int compare(Block o1, Block o2) {
+        Collections.sort(old, new Comparator<Chunk>() {
+            public int compare(Chunk o1, Chunk o2) {
                 return new Integer(o1.collectPriority).compareTo(o2.collectPriority);
             }
         });
         int moveCount = 0;
-        Block move = null;
-        for (Block b : old) {
-            if (moveCount + b.liveCount > averageEntryCount) {
+        Chunk move = null;
+        for (Chunk c : old) {
+            if (moveCount + c.liveCount > averageEntryCount) {
                 break;
             }
-            log(" block " + b.id + " " + b.getFillRate() + "% full; prio=" + b.collectPriority);
-            moveCount += b.liveCount;
-            move = b;
+            log(" chunk " + c.id + " " + c.getFillRate() + "% full; prio=" + c.collectPriority);
+            moveCount += c.liveCount;
+            move = c;
         }
         boolean remove = false;
-        for (Iterator<Block> it = old.iterator(); it.hasNext();) {
-            Block b = it.next();
-            if (move == b) {
+        for (Iterator<Chunk> it = old.iterator(); it.hasNext();) {
+            Chunk c = it.next();
+            if (move == c) {
                 remove = true;
             } else if (remove) {
                 it.remove();
             }
         }
-        long oldMetaRootId = readMetaRootId(move.start);
-        long offset = getPosition(oldMetaRootId);
-        log("  meta:" + move.id + "/" + offset + " start: " + move.start);
-        BtreeMap<String, String> oldMeta = BtreeMap.open(this, "old-meta", STRING_TYPE, STRING_TYPE);
-        oldMeta.setRoot(oldMetaRootId);
+        Chunk header = readChunkHeader(move.start);
+        log("  meta:" + move.id + "/" + header.metaRootOffset + " start: " + move.start);
+        BtreeMap<String, String> oldMeta = BtreeMap.open(this, 0, "old-meta", STRING_TYPE, STRING_TYPE);
+        oldMeta.setRoot(getId(header.id, header.metaRootOffset));
         Iterator<String> it = oldMeta.keyIterator(null);
-        ArrayList<Integer> oldBlocks = New.arrayList();
+        ArrayList<Integer> oldChunks = New.arrayList();
         while (it.hasNext()) {
             String k = it.next();
             String s = oldMeta.get(k);
             log("    " + k + " " + s.replace('\n', ' '));
-            if (k.startsWith("block.")) {
-                Block b = Block.fromString(s);
-                if (!blocks.containsKey(b.id)) {
-                    oldBlocks.add(b.id);
-                    blocks.put(b.id, b);
+            if (k.startsWith("chunk.")) {
+                Chunk c = Chunk.fromString(s);
+                if (!chunks.containsKey(c.id)) {
+                    oldChunks.add(c.id);
+                    chunks.put(c.id, c);
                 }
                 continue;
             }
@@ -528,11 +573,12 @@ public class BtreeMapStore {
             if (!maps.containsKey(k)) {
                 continue;
             }
-            String[] types = StringUtils.arraySplit(s, '/', false);
-            DataType kt = getDataType(types[0]);
-            DataType vt = getDataType(types[1]);
-            long oldDataRoot = Long.parseLong(oldMeta.get("root." + k));
-            BtreeMap<?, ?> oldData = BtreeMap.open(this, "old-" + k, kt, vt);
+            String[] idTypesList = StringUtils.arraySplit(s, '/', false);
+            long id = Long.parseLong(idTypesList[0]);
+            DataType kt = getDataType(idTypesList[1]);
+            DataType vt = getDataType(idTypesList[2]);
+            long oldDataRoot = Long.parseLong(oldMeta.get("root." + id));
+            BtreeMap<?, ?> oldData = BtreeMap.open(this, id, "old-" + k, kt, vt);
             if (oldDataRoot == 0) {
                 // no rows
             } else {
@@ -549,9 +595,9 @@ public class BtreeMapStore {
                         // temporarily changed - ok
                         // TODO move old data if changed temporarily?
                     } else {
-                        Block b = getBlock(p.getId());
-                        if (old.contains(b)) {
-                            log("       move key:" + o + " block:" + b.id);
+                        Chunk c = getChunk(p.getId());
+                        if (old.contains(c)) {
+                            log("       move key:" + o + " chunk:" + c.id);
                             Object value = data.get(o);
                             data.remove(o);
                             data.put(o, value);
@@ -560,8 +606,8 @@ public class BtreeMapStore {
                 }
             }
         }
-        for (int o : oldBlocks) {
-            blocks.remove(o);
+        for (int o : oldChunks) {
+            chunks.remove(o);
         }
     }
 
@@ -574,7 +620,7 @@ public class BtreeMapStore {
      */
     Page readPage(BtreeMap<?, ?> map, long id) {
         if (id < 0) {
-            return temp.get((int) (-id - 1));
+            return temp.get(id);
         }
         Page p = cache.get(id);
         if (p == null) {
@@ -606,19 +652,25 @@ public class BtreeMapStore {
      */
     void removePage(long id) {
         if (id > 0) {
-            if (getBlock(id).liveCount == 0) {
+            cache.remove(id);
+            if (getChunk(id).liveCount == 0) {
                 throw new RuntimeException("Negative live count: " + id);
             }
-            getBlock(id).liveCount--;
+            getChunk(id).liveCount--;
+        } else {
+            temp.remove(id);
+            if (temp.size() == 0) {
+                tempPageId = 0;
+            }
         }
     }
 
-    private static int getBlockId(long pageId) {
-        return (int) (pageId >>> 32);
+    private static int getChunkId(long pos) {
+        return (int) (pos >>> 32);
     }
 
-    private Block getBlock(long pageId) {
-        return blocks.get(getBlockId(pageId));
+    private Chunk getChunk(long pos) {
+        return chunks.get(getChunkId(pos));
     }
 
     /**
@@ -631,4 +683,12 @@ public class BtreeMapStore {
         // System.out.println(string);
     }
 
+    public void setMaxPageSize(int maxPageSize) {
+        this.maxPageSize = maxPageSize;
+    }
+
+    int getMaxPageSize() {
+        return maxPageSize;
+    }
+
 }

h2/src/tools/org/h2/dev/store/btree/Block.java → h2/src/tools/org/h2/dev/store/btree/Chunk.java

@@ -11,12 +11,12 @@ import java.io.IOException;
 import java.util.Properties;
 
 /**
- * A block of data.
+ * A chunk of data, containing one or multiple pages
  */
-class Block {
+class Chunk {
 
     /**
-     * The block id.
+     * The chunk id.
      */
     int id;
 
@@ -26,7 +26,7 @@ class Block {
     long start;
 
     /**
-     * The length in bytes.
+     * The length in bytes (may be larger than the actual value).
      */
     long length;
 
@@ -45,7 +45,12 @@ class Block {
      */
     int collectPriority;
 
-    Block(int id) {
+    /**
+     * The offset of the meta root.
+     */
+    int metaRootOffset;
+
+    Chunk(int id) {
         this.id = id;
     }
 
@@ -55,17 +60,18 @@ class Block {
      * @param s the string
      * @return the block
      */
-    static Block fromString(String s) {
-        Block b = new Block(0);
+    static Chunk fromString(String s) {
+        Chunk c = new Chunk(0);
         Properties prop = new Properties();
         try {
             prop.load(new ByteArrayInputStream(s.getBytes("UTF-8")));
-            b.id = Integer.parseInt(prop.get("id").toString());
-            b.start = Long.parseLong(prop.get("start").toString());
-            b.length = Long.parseLong(prop.get("length").toString());
-            b.entryCount = Integer.parseInt(prop.get("entryCount").toString());
-            b.liveCount = Integer.parseInt(prop.get("liveCount").toString());
-            return b;
+            c.id = Integer.parseInt(prop.get("id").toString());
+            c.start = Long.parseLong(prop.get("start").toString());
+            c.length = Long.parseLong(prop.get("length").toString());
+            c.entryCount = Integer.parseInt(prop.get("entryCount").toString());
+            c.liveCount = Integer.parseInt(prop.get("liveCount").toString());
+            c.metaRootOffset = Integer.parseInt(prop.get("metaRoot").toString());
+            return c;
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
@@ -80,7 +86,7 @@ class Block {
     }
 
     public boolean equals(Object o) {
-        return o instanceof Block && ((Block) o).id == id;
+        return o instanceof Chunk && ((Chunk) o).id == id;
     }
 
     public String toString() {
@@ -89,7 +95,8 @@ class Block {
             "start:" + start + "\n" +
             "length:" + length + "\n" +
             "entryCount:" + entryCount + "\n" +
-            "liveCount:" + liveCount + "\n";
+            "liveCount:" + liveCount + "\n" +
+            "metaRoot:" + metaRootOffset + "\n";
     }
 
 }

h2/src/tools/org/h2/dev/store/btree/Cursor.java

@@ -11,8 +11,11 @@ import java.util.Iterator;
 
 /**
  * A cursor to iterate over elements in ascending order.
+ *
+ * @param <K> the key type
  */
 class Cursor<K> implements Iterator<K> {
+
     private ArrayList<CursorPos> parents = new ArrayList<CursorPos>();
     private K current;
 

h2/src/tools/org/h2/dev/store/btree/DataType.java

@@ -23,20 +23,37 @@ public interface DataType {
     int compare(Object a, Object b);
 
     /**
-     * Get the length in bytes.
+     * Get the length in bytes used to store an object.
      *
      * @param obj the object
      * @return the length
      */
     int length(Object obj);
 
+    /**
+     * Get the maximum length in bytes used to store an object. In many cases,
+     * this method can be faster than calculating the exact length.
+     *
+     * @param obj the object
+     * @return the maximum length
+     */
+    int getMaxLength(Object obj);
+
+    /**
+     * Estimate the used memory in bytes.
+     *
+     * @param obj the object
+     * @return the used memory
+     */
+    int getMemory(Object obj);
+
     /**
      * Write the object.
      *
      * @param buff the target buffer
      * @param x the value
      */
-    void write(ByteBuffer buff, Object x);
+    void write(ByteBuffer buff, Object obj);
 
     /**
      * Read an object.

h2/src/tools/org/h2/dev/store/btree/DataTypeFactory.java

@@ -6,34 +6,18 @@
  */
 package org.h2.dev.store.btree;
 
-import java.io.IOException;
-import java.io.StringReader;
-
 /**
  * A factory for data types.
  */
-public class DataTypeFactory {
+public interface DataTypeFactory {
 
     /**
      * Read the data type.
      *
-     * @param buff the buffer
+     * @param s the string
      * @return the type
      */
-    static DataType fromString(String s) {
-        StringReader r = new StringReader(s);
-        char c;
-        try {
-            c = (char) r.read();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        switch (c) {
-        case 'i':
-            return new IntegerType();
-        }
-        throw new RuntimeException("Unknown data type " + c);
-    }
+    DataType fromString(String s);
 
     /**
      * Get the data type object for the given class.
@@ -41,13 +25,6 @@ public class DataTypeFactory {
      * @param objectClass the class
      * @return the data type object
      */
-    static DataType getDataType(Class<?> objectClass) {
-        if (objectClass == Integer.class) {
-            return new IntegerType();
-        } else if (objectClass == String.class) {
-            return new StringType();
-        }
-        throw new RuntimeException("Unsupported object class " + objectClass.toString());
-    }
+    DataType getDataType(Class<?> objectClass);
 
 }

h2/src/tools/org/h2/dev/store/btree/DataUtils.java

@@ -13,13 +13,18 @@ import java.nio.ByteBuffer;
  */
 public class DataUtils {
 
+    /**
+     * The maximum length of a variable size int.
+     */
+    public static final int MAX_VAR_INT_LEN = 5;
+
     /**
      * Get the length of the variable size int.
      *
      * @param x the value
      * @return the length in bytes
      */
-    static int getVarIntLen(int x) {
+    public static int getVarIntLen(int x) {
         if ((x & (-1 << 7)) == 0) {
             return 1;
         } else if ((x & (-1 << 14)) == 0) {
@@ -55,7 +60,7 @@ public class DataUtils {
      * @param buff the source buffer
      * @return the value
      */
-    static int readVarInt(ByteBuffer buff) {
+    public static int readVarInt(ByteBuffer buff) {
         int b = buff.get();
         if (b >= 0) {
             return b;
@@ -111,7 +116,7 @@ public class DataUtils {
      * @param buff the target buffer
      * @param x the value
      */
-    static void writeVarInt(ByteBuffer buff, int x) {
+    public static void writeVarInt(ByteBuffer buff, int x) {
         while ((x & ~0x7f) != 0) {
             buff.put((byte) (0x80 | (x & 0x7f)));
             x >>>= 7;
@@ -133,4 +138,22 @@ public class DataUtils {
         buff.put((byte) x);
     }
 
+    static void copyWithGap(Object src, Object dst, int oldSize, int gapIndex) {
+        if (gapIndex > 0) {
+            System.arraycopy(src, 0, dst, 0, gapIndex);
+        }
+        if (gapIndex < oldSize) {
+            System.arraycopy(src, gapIndex, dst, gapIndex + 1, oldSize - gapIndex);
+        }
+    }
+
+    static void copyExcept(Object src, Object dst, int oldSize, int removeIndex) {
+        if (removeIndex > 0 && oldSize > 0) {
+            System.arraycopy(src, 0, dst, 0, removeIndex);
+        }
+        if (removeIndex < oldSize) {
+            System.arraycopy(src, removeIndex + 1, dst, removeIndex, oldSize - removeIndex - 1);
+        }
+    }
+
 }

h2/src/tools/org/h2/dev/store/btree/Dump.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.dev.store.btree;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Arrays;
+import java.util.Properties;
+import org.h2.store.fs.FilePath;
+import org.h2.store.fs.FileUtils;
+
+/**
+ * Convert a database file to a human-readable text dump.
+ */
+public class Dump {
+
+    private static int blockSize = 4 * 1024;
+
+    public static void main(String... args) {
+        String fileName = "test.h3";
+        for (int i = 0; i < args.length; i++) {
+            if ("-file".equals(args[i])) {
+                fileName = args[++i];
+            }
+        }
+        dump(fileName);
+    }
+
+    public static void dump(String fileName) {
+        if (!FileUtils.exists(fileName)) {
+            System.out.println("File not found: " + fileName);
+            return;
+        }
+        FileChannel file = null;
+        try {
+            file = FilePath.get(fileName).open("r");
+            long fileLength = file.size();
+            file.position(0);
+            byte[] header = new byte[blockSize];
+            file.read(ByteBuffer.wrap(header));
+            Properties prop = new Properties();
+            prop.load(new ByteArrayInputStream(header));
+            prop.load(new StringReader(new String(header, "UTF-8")));
+            System.out.println("file " + fileName);
+            System.out.println("    length " + fileLength);
+            System.out.println("    " + prop);
+            ByteBuffer block = ByteBuffer.wrap(new byte[16]);
+            for (long pos = 0; pos < fileLength;) {
+                file.position(pos);
+                block.rewind();
+                FileUtils.readFully(file,  block);
+                block.rewind();
+                if (block.get() != 'c') {
+                    pos += blockSize;
+                    continue;
+                }
+                int length = block.getInt();
+                int chunkId = block.getInt();
+                int metaRootOffset = block.getInt();
+                System.out.println("    chunk " + chunkId + " at " + pos +
+                        " length " + length + " offset " + metaRootOffset);
+                ByteBuffer chunk = ByteBuffer.allocate(length);
+                file.position(pos);
+                FileUtils.readFully(file, chunk);
+                int p = block.position();
+                pos = (pos + length + blockSize) / blockSize * blockSize;
+                length -= p;
+                while (length > 0) {
+                    chunk.position(p);
+                    int len = chunk.getInt();
+                    long mapId = chunk.getLong();
+                    int type = chunk.get();
+                    int count = DataUtils.readVarInt(chunk);
+                    if (type == 1) {
+                        long[] children = new long[count];
+                        for (int i = 0; i < count; i++) {
+                            children[i] = chunk.getLong();
+                        }
+                        System.out.println("        map " + mapId + " at " + p + " node, " + count + " children: " + Arrays.toString(children));
+                    } else {
+                        System.out.println("        map " + mapId + " at " + p + " leaf, " + count + " rows");
+                    }
+                    p += len;
+                    length -= len;
+                }
+            }
+        } catch (IOException e) {
+            System.out.println("ERROR: " + e);
+        } finally {
+            try {
+                file.close();
+            } catch (IOException e) {
+                // ignore
+            }
+        }
+        System.out.println();
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/Page.java

@@ -10,19 +10,20 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 
 /**
- * A btree page implementation.
+ * A btree page (a node or a leaf).
+ * <p>
+ * For nodes, the key at a given index is larger than the largest key of the
+ * child at the same index.
  */
-class Page {
-
-    private static final int MAX_SIZE = 20;
+public class Page {
 
     private final BtreeMap<?, ?> map;
     private long id;
-    private long storedId;
     private long transaction;
     private Object[] keys;
     private Object[] values;
     private long[] children;
+    private int cachedCompare;
 
     private Page(BtreeMap<?, ?> map) {
         this.map = map;
@@ -57,7 +58,7 @@ class Page {
      */
     static Page read(BtreeMap<?, ?> map, long id, ByteBuffer buff) {
         Page p = new Page(map);
-        p.id = p.storedId = id;
+        p.id = id;
         p.read(buff);
         return p;
     }
@@ -72,6 +73,7 @@ class Page {
         map.removePage(id);
         Page p2 = create(map, keys, values, children);
         p2.transaction = t;
+        p2.cachedCompare = cachedCompare;
         return p2;
     }
 
@@ -153,18 +155,39 @@ class Page {
 
     private int findKey(Object key) {
         int low = 0, high = keys.length - 1;
+        int x = cachedCompare - 1;
+        if (x < 0 || x > high) {
+            x = (low + high) >>> 1;
+        }
         while (low <= high) {
-            int x = (low + high) >>> 1;
             int compare = map.compare(key, keys[x]);
             if (compare > 0) {
                 low = x + 1;
             } else if (compare < 0) {
                 high = x - 1;
             } else {
+                cachedCompare = x + 1;
                 return x;
             }
+            x = (low + high) >>> 1;
         }
+        cachedCompare = low;
         return -(low + 1);
+
+        // regular binary search (without caching)
+        // int low = 0, high = keys.length - 1;
+        // while (low <= high) {
+        //     int x = (low + high) >>> 1;
+        //     int compare = map.compare(key, keys[x]);
+        //     if (compare > 0) {
+        //         low = x + 1;
+        //     } else if (compare < 0) {
+        //         high = x - 1;
+        //     } else {
+        //         return x;
+        //     }
+        // }
+        // return -(low + 1);
     }
 
     /**
@@ -176,8 +199,8 @@ class Page {
      */
     static void min(Page p, ArrayList<CursorPos> parents, Object key) {
         while (p != null) {
-            int x = key == null ? 0 : p.findKey(key);
             if (p.children != null) {
+                int x = key == null ? -1 : p.findKey(key);
                 if (x < 0) {
                     x = -x - 1;
                 } else {
@@ -189,6 +212,7 @@ class Page {
                 parents.add(c);
                 p = p.map.readPage(p.children[x]);
             } else {
+                int x = key == null ? 0 : p.findKey(key);
                 if (x < 0) {
                     x = -x - 1;
                 }
@@ -207,7 +231,7 @@ class Page {
      * @param parents the stack of parent page positions
      * @return the next key
      */
-    public static Object nextKey(ArrayList<CursorPos> parents) {
+    static Object nextKey(ArrayList<CursorPos> parents) {
         if (parents.size() == 0) {
             return null;
         }
@@ -224,7 +248,7 @@ class Page {
                     return null;
                 }
                 p = parents.remove(parents.size() - 1);
-                index = p.index++;
+                index = ++p.index;
                 if (index < p.page.children.length) {
                     parents.add(p);
                     Page x = p.page;
@@ -302,7 +326,7 @@ class Page {
                 parent.children[parentIndex] = p.id;
             }
             if (!p.isLeaf()) {
-                if (p.keyCount() >= MAX_SIZE) {
+                if (p.keyCount() >= map.getMaxPageSize()) {
                     // TODO almost duplicate code
                     int pos = p.keyCount() / 2;
                     Object k = p.keys[pos];
@@ -331,7 +355,7 @@ class Page {
                 }
                 index = -index - 1;
                 p.insert(index, key, value, 0);
-                if (p.keyCount() >= MAX_SIZE) {
+                if (p.keyCount() >= map.getMaxPageSize()) {
                     int pos = p.keyCount() / 2;
                     Object k = p.keys[pos];
                     Page split = p.splitLeaf(pos);
@@ -347,6 +371,8 @@ class Page {
             }
             if (index < 0) {
                 index = -index - 1;
+            } else {
+                index++;
             }
             parent = p;
             parentIndex = index;
@@ -356,6 +382,18 @@ class Page {
         return top;
     }
 
+    /**
+     * Remove this page and all child pages.
+     */
+    void removeAllRecursive() {
+        if (children != null) {
+            for (long c : children) {
+                map.readPage(c).removeAllRecursive();
+            }
+        }
+        map.removePage(id);
+    }
+
     /**
      * Remove a key-value pair.
      *
@@ -368,6 +406,7 @@ class Page {
         if (p.isLeaf()) {
             if (index >= 0) {
                 if (p.keyCount() == 1) {
+                    p.map.removePage(p.id);
                     return null;
                 }
                 p = p.copyOnWrite();
@@ -380,6 +419,8 @@ class Page {
         // node
         if (index < 0) {
             index = -index - 1;
+        } else {
+            index++;
         }
         Page c = p.map.readPage(p.children[index]);
         Page c2 = remove(c, key);
@@ -390,7 +431,8 @@ class Page {
             p = p.copyOnWrite();
             p.remove(index);
             if (p.keyCount() == 0) {
-                p = c2;
+                p.map.removePage(p.id);
+                p = p.map.readPage(p.children[0]);
             }
         } else {
             p = p.copyOnWrite();
@@ -401,18 +443,18 @@ class Page {
 
     private void insert(int index, Object key, Object value, long child) {
         Object[] newKeys = new Object[keys.length + 1];
-        copyWithGap(keys, newKeys, keys.length, index);
+        DataUtils.copyWithGap(keys, newKeys, keys.length, index);
         newKeys[index] = key;
         keys = newKeys;
         if (values != null) {
             Object[] newValues = new Object[values.length + 1];
-            copyWithGap(values, newValues, values.length, index);
+            DataUtils.copyWithGap(values, newValues, values.length, index);
             newValues[index] = value;
             values = newValues;
         }
         if (children != null) {
             long[] newChildren = new long[children.length + 1];
-            copyWithGap(children, newChildren, children.length, index + 1);
+            DataUtils.copyWithGap(children, newChildren, children.length, index + 1);
             newChildren[index + 1] = child;
             children = newChildren;
         }
@@ -420,21 +462,28 @@ class Page {
 
     private void remove(int index) {
         Object[] newKeys = new Object[keys.length - 1];
-        copyExcept(keys, newKeys, keys.length, index);
+        int keyIndex = index >= keys.length ? index - 1 : index;
+        DataUtils.copyExcept(keys, newKeys, keys.length, keyIndex);
         keys = newKeys;
         if (values != null) {
             Object[] newValues = new Object[values.length - 1];
-            copyExcept(values, newValues, values.length, index);
+            DataUtils.copyExcept(values, newValues, values.length, index);
             values = newValues;
         }
         if (children != null) {
             long[] newChildren = new long[children.length - 1];
-            copyExcept(children, newChildren, children.length, index);
+            DataUtils.copyExcept(children, newChildren, children.length, index);
             children = newChildren;
         }
     }
 
     private void read(ByteBuffer buff) {
+        // len
+        buff.getInt();
+        long id = buff.getLong();
+        if (id != map.getId()) {
+            throw new RuntimeException("Page map id missmatch, expected " + map.getId() + " got " + id);
+        }
         boolean node = buff.get() == 1;
         if (node) {
             int len = DataUtils.readVarInt(buff);
@@ -442,9 +491,9 @@ class Page {
             keys = new Object[len - 1];
             for (int i = 0; i < len; i++) {
                 children[i] = buff.getLong();
-                if (i < keys.length) {
-                    keys[i] = map.getKeyType().read(buff);
             }
+            for (int i = 0; i < len - 1; i++) {
+                keys[i] = map.getKeyType().read(buff);
             }
         } else {
             int len = DataUtils.readVarInt(buff);
@@ -462,17 +511,19 @@ class Page {
      *
      * @param buff the target buffer
      */
-    void write(ByteBuffer buff) {
+    private void write(ByteBuffer buff) {
+        int pos = buff.position();
+        buff.putInt(0);
+        buff.putLong(map.getId());
         if (children != null) {
             buff.put((byte) 1);
             int len = children.length;
             DataUtils.writeVarInt(buff, len);
             for (int i = 0; i < len; i++) {
-                long c = map.readPage(children[i]).storedId;
-                buff.putLong(c);
-                if (i < keys.length) {
-                    map.getKeyType().write(buff, keys[i]);
+                buff.putLong(children[i]);
             }
+            for (int i = 0; i < len - 1; i++) {
+                map.getKeyType().write(buff, keys[i]);
             }
         } else {
             buff.put((byte) 0);
@@ -483,66 +534,62 @@ class Page {
                 map.getValueType().write(buff, values[i]);
             }
         }
+        int len = buff.position() - pos;
+        buff.putInt(pos, len);
     }
 
     /**
-     * Get the length in bytes, including temporary children.
-     *
-     * @return the length
-     */
-    int lengthIncludingTempChildren() {
-        int byteCount = length();
-        if (children != null) {
-            int len = children.length;
-            for (int i = 0; i < len; i++) {
-                long c = children[i];
-                if (c < 0) {
-                    byteCount += map.readPage(c).lengthIncludingTempChildren();
-                }
-            }
-        }
-        return byteCount;
-    }
-
-    /**
-     * Update the page ids recursively.
+     * Get the maximum length in bytes to store temporary records, recursively.
      *
      * @param pageId the new page id
      * @return the next page id
      */
-    long updatePageIds(long pageId) {
-        this.storedId = pageId;
-        pageId += length();
+    int getMaxLengthTempRecursive() {
+        int maxLength = 4 + 8 + 1;
         if (children != null) {
             int len = children.length;
+            maxLength += DataUtils.MAX_VAR_INT_LEN;
+            maxLength += 8 * len;
+            for (int i = 0; i < len - 1; i++) {
+                maxLength += map.getKeyType().getMaxLength(keys[i]);
+            }
             for (int i = 0; i < len; i++) {
                 long c = children[i];
                 if (c < 0) {
-                    pageId = map.readPage(c).updatePageIds(pageId);
+                    maxLength += map.readPage(c).getMaxLengthTempRecursive();
                 }
             }
+        } else {
+            int len = keys.length;
+            maxLength += DataUtils.MAX_VAR_INT_LEN;
+            for (int i = 0; i < len; i++) {
+                maxLength += map.getKeyType().getMaxLength(keys[i]);
+                maxLength += map.getValueType().getMaxLength(values[i]);
+            }
         }
-        return pageId;
+        return maxLength;
     }
 
     /**
-     * Store this page.
+     * Store this page and all children that are changed,
+     * in reverse order, and update the id and child ids.
      *
      * @param buff the target buffer
+     * @param idOffset the offset of the id
      * @return the page id
      */
-    long storeTemp(ByteBuffer buff) {
-        write(buff);
+    long writeTempRecursive(ByteBuffer buff, long idOffset) {
         if (children != null) {
             int len = children.length;
             for (int i = 0; i < len; i++) {
                 long c = children[i];
                 if (c < 0) {
-                    children[i] = map.readPage(c).storeTemp(buff);
+                    children[i] = map.readPage(c).writeTempRecursive(buff, idOffset);
                 }
             }
         }
-        this.id = storedId;
+        this.id = idOffset + buff.position();
+        write(buff);
         return id;
     }
 
@@ -551,63 +598,18 @@ class Page {
      *
      * @return the number of temporary pages
      */
-    int countTemp() {
+    int countTempRecursive() {
         int count = 1;
         if (children != null) {
             int size = children.length;
             for (int i = 0; i < size; i++) {
                 long c = children[i];
                 if (c < 0) {
-                    count += map.readPage(c).countTemp();
+                    count += map.readPage(c).countTempRecursive();
                 }
             }
         }
         return count;
     }
 
-    /**
-     * Get the length in bytes.
-     *
-     * @return the length
-     */
-    int length() {
-        int byteCount = 1;
-        if (children != null) {
-            int len = children.length;
-            byteCount += DataUtils.getVarIntLen(len);
-            for (int i = 0; i < len; i++) {
-                byteCount += 8;
-                if (i < keys.length) {
-                    byteCount += map.getKeyType().length(keys[i]);
-                }
-            }
-        } else {
-            int len = keys.length;
-            byteCount += DataUtils.getVarIntLen(len);
-            for (int i = 0; i < len; i++) {
-                byteCount += map.getKeyType().length(keys[i]);
-                byteCount += map.getValueType().length(values[i]);
-            }
-        }
-        return byteCount;
-    }
-
-    private static void copyWithGap(Object src, Object dst, int oldSize, int gapIndex) {
-        if (gapIndex > 0) {
-            System.arraycopy(src, 0, dst, 0, gapIndex);
-        }
-        if (gapIndex < oldSize) {
-            System.arraycopy(src, gapIndex, dst, gapIndex + 1, oldSize - gapIndex);
-        }
-    }
-
-    private static void copyExcept(Object src, Object dst, int oldSize, int removeIndex) {
-        if (removeIndex > 0 && oldSize > 0) {
-            System.arraycopy(src, 0, dst, 0, removeIndex);
-        }
-        if (removeIndex < oldSize) {
-            System.arraycopy(src, removeIndex + 1, dst, removeIndex, oldSize - removeIndex - 1);
-        }
-    }
-
 }

h2/src/tools/org/h2/dev/store/btree/StringType.java

@@ -11,39 +11,67 @@ import java.nio.ByteBuffer;
 /**
  * A string type.
  */
-class StringType implements DataType {
+public class StringType implements DataType {
 
     public int compare(Object a, Object b) {
         return a.toString().compareTo(b.toString());
     }
 
     public int length(Object obj) {
-        try {
-            byte[] bytes = obj.toString().getBytes("UTF-8");
-            return DataUtils.getVarIntLen(bytes.length) + bytes.length;
-        } catch (Exception e) {
-            throw new RuntimeException(e);
+        int plus = 0;
+        String s = obj.toString();
+        int len = s.length();
+        for (int i = 0; i < len; i++) {
+            char c = s.charAt(i);
+            if (c >= 0x800) {
+                plus += 2;
+            } else if (c >= 0x80) {
+                plus++;
             }
         }
+        return DataUtils.getVarIntLen(len) + len + plus;
+    }
+
+    public int getMaxLength(Object obj) {
+        return DataUtils.MAX_VAR_INT_LEN + obj.toString().length() * 3;
+    }
+
+    public int getMemory(Object obj) {
+        return obj.toString().length() * 2 + 48;
+    }
 
     public String read(ByteBuffer buff) {
         int len = DataUtils.readVarInt(buff);
-        byte[] bytes = new byte[len];
-        buff.get(bytes);
-        try {
-            return new String(bytes, "UTF-8");
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public void write(ByteBuffer buff, Object x) {
-        try {
-            byte[] bytes = x.toString().getBytes("UTF-8");
-            DataUtils.writeVarInt(buff, bytes.length);
-            buff.put(bytes);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
+        char[] chars = new char[len];
+        for (int i = 0; i < len; i++) {
+            int x = buff.get() & 0xff;
+            if (x < 0x80) {
+                chars[i] = (char) x;
+            } else if (x >= 0xe0) {
+                chars[i] = (char) (((x & 0xf) << 12) + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f));
+            } else {
+                chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f));
+            }
+        }
+        return new String(chars);
+    }
+
+    public void write(ByteBuffer buff, Object obj) {
+        String s = obj.toString();
+        int len = s.length();
+        DataUtils.writeVarInt(buff, len);
+        for (int i = 0; i < len; i++) {
+            int c = s.charAt(i);
+            if (c < 0x80) {
+                buff.put((byte) c);
+            } else if (c >= 0x800) {
+                buff.put((byte) (0xe0 | (c >> 12)));
+                buff.put((byte) (((c >> 6) & 0x3f)));
+                buff.put((byte) (c & 0x3f));
+            } else {
+                buff.put((byte) (0xc0 | (c >> 6)));
+                buff.put((byte) (c & 0x3f));
+            }
         }
     }
 

h2/src/tools/org/h2/dev/store/tree/StoredMap.java

@@ -132,7 +132,7 @@ public class StoredMap<K, V> {
     /**
      * A value type.
      */
-    static interface ValueType {
+    interface ValueType {
 
         /**
          * Get the length in bytes.
@@ -170,7 +170,7 @@ public class StoredMap<K, V> {
     /**
      * A key type.
      */
-    static interface KeyType extends ValueType {
+    interface KeyType extends ValueType {
 
         /**
          * Compare two keys.