A persistent tree map (work in progress).

h2/src/test/org/h2/test/store/TestBtreeMapStore.java

@@ -29,6 +29,7 @@ public class TestBtreeMapStore extends TestBase {
     }
 
     public void test() {
+        testFastDelete();
         testRollbackInMemory();
         testRollbackStored();
         testMeta();
@@ -42,6 +43,31 @@ public class TestBtreeMapStore extends TestBase {
         testSimple();
     }
 
+    private void testFastDelete() {
+        String fileName = getBaseDir() + "/testMeta.h3";
+        FileUtils.delete(fileName);
+        BtreeMapStore s;
+        BtreeMap<Integer, String> m;
+        s = openStore(fileName);
+        s.setMaxPageSize(100);
+        m = s.openMap("data", Integer.class, String.class);
+        for (int i = 0; i < 1000; i++) {
+            m.put(i, "Hello World");
+        }
+        s.store();
+        assertEquals(3, s.getWriteCount());
+        s.close();
+
+        s = openStore(fileName);
+        m = s.openMap("data", Integer.class, String.class);
+        m.clear();
+        s.store();
+        // ensure only nodes are read, but not leaves
+        assertEquals(4, s.getReadCount());
+        assertEquals(2, s.getWriteCount());
+        s.close();
+    }
+
     private void testRollbackStored() {
         String fileName = getBaseDir() + "/testMeta.h3";
         FileUtils.delete(fileName);

h2/src/test/org/h2/test/store/TestDataUtils.java

@@ -44,13 +44,25 @@ public class TestDataUtils extends TestBase {
     }
 
     private void testPagePos() {
-        for (int chunkId = 0; chunkId < 67000000; chunkId += 670000) {
-            for (long offset = 0; offset < Integer.MAX_VALUE; offset += Integer.MAX_VALUE / 100) {
-                for (int length = 0; length < 2000000; length += 200000) {
-                    long pos = DataUtils.getPos(chunkId, (int) offset, length);
-                    assertEquals(chunkId, DataUtils.getChunkId(pos));
-                    assertEquals(offset, DataUtils.getOffset(pos));
-                    assertTrue(DataUtils.getMaxLength(pos) >= length);
+        assertEquals(0, DataUtils.PAGE_TYPE_LEAF);
+        assertEquals(1, DataUtils.PAGE_TYPE_NODE);
+        for (int i = 0; i < 67000000; i++) {
+            long pos = DataUtils.getPagePos(i, 3, 128, 1);
+            assertEquals(i, DataUtils.getPageChunkId(pos));
+            assertEquals(3, DataUtils.getPageOffset(pos));
+            assertEquals(128, DataUtils.getPageMaxLength(pos));
+            assertEquals(1, DataUtils.getPageType(pos));
+        }
+        for (int type = 0; type <= 1; type++) {
+            for (int chunkId = 0; chunkId < 67000000; chunkId += 670000) {
+                for (long offset = 0; offset < Integer.MAX_VALUE; offset += Integer.MAX_VALUE / 100) {
+                    for (int length = 0; length < 2000000; length += 200000) {
+                        long pos = DataUtils.getPagePos(chunkId, (int) offset, length, type);
+                        assertEquals(chunkId, DataUtils.getPageChunkId(pos));
+                        assertEquals(offset, DataUtils.getPageOffset(pos));
+                        assertTrue(DataUtils.getPageMaxLength(pos) >= length);
+                        assertTrue(DataUtils.getPageType(pos) == type);
+                    }
                 }
             }
         }
@@ -75,7 +87,7 @@ public class TestDataUtils extends TestBase {
             if (code > lastCode) {
                 lastCode = code;
             }
-            int max = DataUtils.getMaxLength(code);
+            int max = DataUtils.getPageMaxLength(code << 1);
             assertTrue(max >= i && max >= 32);
         }
     }

h2/src/tools/org/h2/dev/store/btree/BtreeMap.java

@@ -95,7 +95,7 @@ public class BtreeMap<K, V> {
     }
 
     /**
-     * Remove all entries, and remove the map. The map becomes invalid.
+     * Remove all entries, and close the map.
      */
     public void remove() {
         checkWrite();
@@ -103,10 +103,14 @@ public class BtreeMap<K, V> {
             root.removeAllRecursive();
         }
         store.removeMap(id);
+        close();
+    }
+
+    public void close() {
+        readOnly = true;
+        store = null;
         oldRoots.clear();
         root = null;
-        store = null;
-        readOnly = true;
     }
 
     public boolean isClosed() {
@@ -278,13 +282,6 @@ public class BtreeMap<K, V> {
         return buff.toString();
     }
 
-    public void close() {
-        readOnly = true;
-        store = null;
-        oldRoots.clear();
-        root = null;
-    }
-
     public int hashCode() {
         return id;
     }

h2/src/tools/org/h2/dev/store/btree/BtreeMapStore.java

@@ -38,7 +38,6 @@ header:
 blockSize=4096
 
 TODO:
-- keep page type (leaf/node) in pos to speed up large deletes
 - support fast range deletes
 - support custom pager for r-tree, kd-tree
 - need an 'end of chunk' marker to verify all data is written
@@ -57,6 +56,7 @@ TODO:
 - file header could be a regular chunk, end of file the second
 - possibly split chunk data into immutable and mutable
 - reduce minimum chunk size, speed up very small transactions
+- defragment: use total max length instead of page count (liveCount)
 
 */
 
@@ -107,6 +107,8 @@ public class BtreeMapStore {
     private Compressor compressor = new CompressLZF();
 
     private long currentVersion;
+    private int readCount;
+    private int writeCount;
 
     private BtreeMapStore(String fileName, DataTypeFactory typeFactory) {
         this.fileName = fileName;
@@ -310,6 +312,7 @@ public class BtreeMapStore {
                 "rootChunk:" + rootChunkStart + "\n" +
                 "lastMapId:" + lastMapId + "\n" +
                 "version:" + currentVersion + "\n").getBytes("UTF-8"));
+            writeCount++;
             file.position(0);
             file.write(header);
             file.position(blockSize);
@@ -321,8 +324,9 @@ public class BtreeMapStore {
 
     private void readHeader() {
         try {
-            file.position(0);
             byte[] header = new byte[blockSize];
+            readCount++;
+            file.position(0);
             // TODO read fully; read both headers
             file.read(ByteBuffer.wrap(header));
             Properties prop = new Properties();
@@ -365,16 +369,16 @@ public class BtreeMapStore {
     }
 
     private Chunk getChunk(long pos) {
-        return chunks.get(DataUtils.getChunkId(pos));
+        return chunks.get(DataUtils.getPageChunkId(pos));
     }
 
     private long getFilePosition(long pos) {
         Chunk c = getChunk(pos);
         if (c == null) {
-            throw new RuntimeException("Chunk " + DataUtils.getChunkId(pos) + " not found");
+            throw new RuntimeException("Chunk " + DataUtils.getPageChunkId(pos) + " not found");
         }
         long filePos = c.start;
-        filePos += DataUtils.getOffset(pos);
+        filePos += DataUtils.getPageOffset(pos);
         return filePos;
     }
 
@@ -467,6 +471,7 @@ public class BtreeMapStore {
         buff.putLong(meta.getRoot().getPos());
         buff.rewind();
         try {
+            writeCount++;
             file.position(filePos);
             file.write(buff);
         } catch (IOException e) {
@@ -575,6 +580,7 @@ public class BtreeMapStore {
 
     private Chunk readChunkHeader(long start) {
         try {
+            readCount++;
             file.position(start);
             ByteBuffer buff = ByteBuffer.wrap(new byte[32]);
             DataUtils.readFully(file, buff);
@@ -724,6 +730,7 @@ public class BtreeMapStore {
         Page p = cache.get(pos);
         if (p == null) {
             long filePos = getFilePosition(pos);
+            readCount++;
             p = Page.read(file, map, filePos, pos);
             cache.put(pos, p);
         }
@@ -775,6 +782,11 @@ public class BtreeMapStore {
         this.maxPageSize = maxPageSize;
     }
 
+    /**
+     * The maximum number of key-value pairs in a page.
+     *
+     * @return the maximum number of entries
+     */
     int getMaxPageSize() {
         return maxPageSize;
     }
@@ -914,4 +926,22 @@ public class BtreeMapStore {
         return currentVersion;
     }
 
+    /**
+     * Get the number of write operations since this store was opened.
+     *
+     * @return the number of write operations
+     */
+    public int getWriteCount() {
+        return writeCount;
+    }
+
+    /**
+     * Get the number of read operations since this store was opened.
+     *
+     * @return the number of read operations
+     */
+    public int getReadCount() {
+        return readCount;
+    }
+
 }

h2/src/tools/org/h2/dev/store/btree/DataUtils.java

@@ -15,6 +15,21 @@ import java.nio.channels.FileChannel;
  */
 public class DataUtils {
 
+    /**
+     * The type for leaf page.
+     */
+    public static final int PAGE_TYPE_LEAF = 0;
+
+    /**
+     * The type for node page.
+     */
+    public static final int PAGE_TYPE_NODE = 1;
+
+    /**
+     * The bit mask for compressed pages.
+     */
+    public static final int PAGE_COMPRESSED = 2;
+
     /**
      * The maximum length of a variable size int.
      */
@@ -211,8 +226,8 @@ public class DataUtils {
      * @param pos the position
      * @return the chunk id
      */
-    public static int getChunkId(long pos) {
-        return (int) (pos >>> 37);
+    public static int getPageChunkId(long pos) {
+        return (int) (pos >>> 38);
     }
 
     /**
@@ -222,8 +237,8 @@ public class DataUtils {
      * @param pos the position
      * @return the maximum length
      */
-    public static int getMaxLength(long pos) {
-        int code = (int) (pos & 31);
+    public static int getPageMaxLength(long pos) {
+        int code = (int) ((pos >> 1) & 31);
         if (code == 31) {
             return Integer.MAX_VALUE;
         }
@@ -236,21 +251,37 @@ public class DataUtils {
      * @param pos the position
      * @return the offset
      */
-    public static int getOffset(long pos) {
-        return (int) (pos >> 5);
+    public static int getPageOffset(long pos) {
+        return (int) (pos >> 6);
+    }
+
+    /**
+     * Get the page type from the position.
+     *
+     * @param pos the position
+     * @return the page type (PAGE_TYPE_NODE or PAGE_TYPE_LEAF)
+     */
+    public static int getPageType(long pos) {
+        return ((int) pos) & 1;
     }
 
     /**
      * Get the position of this page. The following information is encoded in
-     * the position: the chunk id, the offset, and the maximum length.
+     * the position: the chunk id, the offset, the maximum length, and the type
+     * (node or leaf).
      *
      * @param chunkId the chunk id
      * @param offset the offset
      * @param length the length
+     * @param type the page type (1 for node, 0 for leaf)
      * @return the position
      */
-    public static long getPos(int chunkId, int offset, int length) {
-        return ((long) chunkId << 37) | ((long) offset << 5) | encodeLength(length);
+    public static long getPagePos(int chunkId, int offset, int length, int type) {
+        long pos = (long) chunkId << 38;
+        pos |= (long) offset << 6;
+        pos |= encodeLength(length) << 1;
+        pos |= type;
+        return pos;
     }
 
     /**

h2/src/tools/org/h2/dev/store/btree/Dump.java

@@ -12,7 +12,6 @@ import java.io.PrintWriter;
 import java.io.StringReader;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
-import java.util.Arrays;
 import java.util.Properties;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
@@ -68,7 +67,7 @@ public class Dump {
             writer.println("file " + fileName);
             writer.println("    length " + fileLength);
             writer.println("    " + prop);
-            ByteBuffer block = ByteBuffer.wrap(new byte[16]);
+            ByteBuffer block = ByteBuffer.wrap(new byte[32]);
             for (long pos = 0; pos < fileLength;) {
                 file.position(pos);
                 block.rewind();
@@ -78,34 +77,33 @@ public class Dump {
                     pos += blockSize;
                     continue;
                 }
-                int length = block.getInt();
+                int chunkLength = block.getInt();
                 int chunkId = block.getInt();
-                int metaRootOffset = block.getInt();
+                long metaRootPos = block.getLong();
                 writer.println("    chunk " + chunkId + " at " + pos +
-                        " length " + length + " offset " + metaRootOffset);
-                ByteBuffer chunk = ByteBuffer.allocate(length);
+                        " length " + chunkLength + " root " + metaRootPos);
+                ByteBuffer chunk = ByteBuffer.allocate(chunkLength);
                 file.position(pos);
                 FileUtils.readFully(file, chunk);
                 int p = block.position();
-                pos = (pos + length + blockSize) / blockSize * blockSize;
-                length -= p;
-                while (length > 0) {
+                pos = (pos + chunkLength + blockSize) / blockSize * blockSize;
+                chunkLength -= p;
+                while (chunkLength > 0) {
                     chunk.position(p);
-                    int len = chunk.getInt();
-                    long mapId = chunk.getLong();
+                    int pageLength = chunk.getInt();
+                    // check value (ignored)
+                    chunk.getShort();
+                    long mapId = DataUtils.readVarInt(chunk);
+                    int len = DataUtils.readVarInt(chunk);
                     int type = chunk.get();
-                    int count = DataUtils.readVarInt(chunk);
-                    if (type == 1) {
-                        long[] children = new long[count];
-                        for (int i = 0; i < count; i++) {
-                            children[i] = chunk.getLong();
-                        }
-                        writer.println("        map " + mapId + " at " + p + " node, " + count + " children: " + Arrays.toString(children));
-                    } else {
-                        writer.println("        map " + mapId + " at " + p + " leaf, " + count + " rows");
-                    }
-                    p += len;
-                    length -= len;
+                    boolean compressed = (type & 2) != 0;
+                    boolean node = (type & 1) != 0;
+                    writer.println("        map " + mapId + " at " + p + " " +
+                            (node ? "node" : "leaf") + " " +
+                            (compressed ? "compressed" : "") + " " +
+                            "len: " + pageLength + " entries: " + len);
+                    p += pageLength;
+                    chunkLength -= pageLength;
                 }
             }
         } catch (IOException e) {

h2/src/tools/org/h2/dev/store/btree/Page.java

@@ -21,6 +21,7 @@ import org.h2.compress.Compressor;
  * File format:
  * page length (including length): int
  * check value: short
+ * map id: varInt
  * number of keys: varInt
  * type: byte (0: leaf, 1: node; +2: compressed)
  * compressed: bytes saved (varInt)
@@ -71,7 +72,7 @@ public class Page {
      * @return the page
      */
     static Page read(FileChannel file, BtreeMap<?, ?> map, long filePos, long pos) {
-        int maxLength = DataUtils.getMaxLength(pos), length = maxLength;
+        int maxLength = DataUtils.getPageMaxLength(pos), length = maxLength;
         ByteBuffer buff;
         try {
             file.position(filePos);
@@ -88,8 +89,8 @@ public class Page {
         }
         Page p = new Page(map, 0);
         p.pos = pos;
-        int chunkId = DataUtils.getChunkId(pos);
-        int offset = DataUtils.getOffset(pos);
+        int chunkId = DataUtils.getPageChunkId(pos);
+        int offset = DataUtils.getPageOffset(pos);
         p.read(buff, chunkId, offset, maxLength);
         return p;
     }
@@ -427,7 +428,12 @@ public class Page {
     void removeAllRecursive() {
         if (children != null) {
             for (long c : children) {
-                map.readPage(c).removeAllRecursive();
+                int type = DataUtils.getPageType(c);
+                if (type == DataUtils.PAGE_TYPE_LEAF) {
+                    getStore().removePage(c);
+                } else {
+                    map.readPage(c).removeAllRecursive();
+                }
             }
         }
         getStore().removePage(pos);
@@ -524,6 +530,10 @@ public class Page {
             throw new RuntimeException("Length too large, expected =< " + maxLength + " got " + pageLength);
         }
         short check = buff.getShort();
+        int mapId = DataUtils.readVarInt(buff);
+        if (mapId != map.getId()) {
+            throw new RuntimeException("Error reading page, expected map " + map.getId() + " got " + mapId);
+        }
         int len = DataUtils.readVarInt(buff);
         int checkTest = DataUtils.getCheckValue(chunkId) ^
                 DataUtils.getCheckValue(map.getId()) ^
@@ -535,8 +545,8 @@ public class Page {
         }
         keys = new Object[len];
         int type = buff.get();
-        boolean node = (type & 1) != 0;
-        boolean compressed = (type & 2) != 0;
+        boolean node = (type & 1) == DataUtils.PAGE_TYPE_NODE;
+        boolean compressed = (type & DataUtils.PAGE_COMPRESSED) != 0;
         if (compressed) {
             Compressor compressor = map.getStore().getCompressor();
             int lenAdd = DataUtils.readVarInt(buff);
@@ -573,16 +583,17 @@ public class Page {
         int start = buff.position();
         buff.putInt(0);
         buff.putShort((byte) 0);
+        DataUtils.writeVarInt(buff,  map.getId());
         int len = keys.length;
         DataUtils.writeVarInt(buff, len);
         Compressor compressor = map.getStore().getCompressor();
-        int type = children != null ? 1 : 0;
+        int type = children != null ? DataUtils.PAGE_TYPE_NODE : DataUtils.PAGE_TYPE_LEAF;
         buff.put((byte) type);
         int compressStart = buff.position();
         for (int i = 0; i < len; i++) {
             map.getKeyType().write(buff, keys[i]);
         }
-        if (type == 1) {
+        if (type == DataUtils.PAGE_TYPE_NODE) {
             for (int i = 0; i < len + 1; i++) {
                 buff.putLong(children[i]);
             }
@@ -600,7 +611,7 @@ public class Page {
             int compLen = compressor.compress(exp, exp.length, comp, 0);
             if (compLen + DataUtils.getVarIntLen(compLen - expLen) < expLen) {
                 buff.position(compressStart - 1);
-                buff.put((byte) (type + 2));
+                buff.put((byte) (type + DataUtils.PAGE_COMPRESSED));
                 DataUtils.writeVarInt(buff, expLen - compLen);
                 buff.put(comp,  0, compLen);
             }
@@ -614,7 +625,7 @@ public class Page {
                 DataUtils.getCheckValue(pageLength) ^
                 DataUtils.getCheckValue(len);
         buff.putShort(start + 4, (short) check);
-        this.pos = DataUtils.getPos(chunkId, start, pageLength);
+        this.pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
     }
 
     /**
@@ -623,8 +634,8 @@ public class Page {
      * @return the next page id
      */
     int getMaxLengthTempRecursive() {
-        // length, check, key length, type
-        int maxLength = 4 + 2 + DataUtils.MAX_VAR_INT_LEN + 1;
+        // length, check, map id, key length, type
+        int maxLength = 4 + 2 + DataUtils.MAX_VAR_INT_LEN + DataUtils.MAX_VAR_INT_LEN + 1;
         int len = keys.length;
         for (int i = 0; i < len; i++) {
             maxLength += map.getKeyType().getMaxLength(keys[i]);