Shrink headers

h2/src/main/org/h2/mvstore/Chunk.java

@@ -15,14 +15,6 @@ import java.util.HashMap;
  * Chunks are page aligned (each page is usually 4096 bytes).
  * There are at most 67 million (2^26) chunks,
  * each chunk is at most 2 GB large.
- * Chunk format:
- * 1 byte: 'c'
- * 4 bytes: length
- * 4 bytes: chunk id (an incrementing number)
- * 4 bytes: pageCount
- * 8 bytes: metaRootPos
- * 8 bytes: maxLengthLive
- * [ Page ] *
  */
 public class Chunk {
 
@@ -37,14 +29,14 @@ public class Chunk {
     public final int id;
 
     /**
-     * The start position within the file.
+     * The start block number within the file.
      */
-    public long start;
+    public long block;
 
     /**
-     * The length in bytes.
+     * The length in number of blocks.
      */
-    public int length;
+    public int blocks;
 
     /**
      * The total number of pages in this chunk.
@@ -122,12 +114,20 @@ public class Chunk {
      *
      * @param buff the target buffer
      */
-    void writeHeader(WriteBuffer buff) {
+    void writeHeader(WriteBuffer buff, int minLength) {
+        long pos = buff.position();
         buff.put((byte) '{');
         buff.put(asString().getBytes(DataUtils.UTF8));
         buff.put((byte) '}');
+        while (buff.position() - pos < minLength - 1) {
             buff.put((byte) ' ');
         }
+        buff.put((byte) '\n');
+    }
+    
+    static String getMetaKey(int chunkId) {
+        return "chunk." + Integer.toHexString(chunkId);
+    }
 
     /**
      * Build a block from the given string.
@@ -137,17 +137,17 @@ public class Chunk {
      */
     public static Chunk fromString(String s) {
         HashMap<String, String> map = DataUtils.parseMap(s);
-        int id = Integer.parseInt(map.get("chunk"));
+        int id = Integer.parseInt(map.get("chunk"), 16);
         Chunk c = new Chunk(id);
-        c.start = Long.parseLong(map.get("start"));
-        c.length = Integer.parseInt(map.get("length"));
-        c.pageCount = Integer.parseInt(map.get("pageCount"));
-        c.pageCountLive = Integer.parseInt(map.get("pageCountLive"));
-        c.maxLength = Long.parseLong(map.get("maxLength"));
-        c.maxLengthLive = Long.parseLong(map.get("maxLengthLive"));
-        c.metaRootPos = Long.parseLong(map.get("metaRoot"));
-        c.time = Long.parseLong(map.get("time"));
-        c.version = Long.parseLong(map.get("version"));
+        c.block = Long.parseLong(map.get("block"), 16);
+        c.blocks = Integer.parseInt(map.get("blocks"), 16);
+        c.pageCount = Integer.parseInt(map.get("pages"), 16);
+        c.pageCountLive = DataUtils.parseHexInt(map.get("livePages"), c.pageCount);
+        c.maxLength = Long.parseLong(map.get("max"), 16);
+        c.maxLengthLive = DataUtils.parseHexLong(map.get("liveMax"), c.maxLength);
+        c.metaRootPos = Long.parseLong(map.get("root"), 16);
+        c.time = Long.parseLong(map.get("time"), 16);
+        c.version = Long.parseLong(map.get("version"), 16);
         return c;
     }
 
@@ -171,17 +171,22 @@ public class Chunk {
      * @return the string
      */
     public String asString() {
-        return
-                "chunk:" + id + "," +
-                "length:" + length + "," +
-                "maxLength:" + maxLength + "," +
-                "maxLengthLive:" + maxLengthLive + "," +
-                "metaRoot:" + metaRootPos + "," +
-                "pageCount:" + pageCount + "," +
-                "pageCountLive:" + pageCountLive + "," +
-                "start:" + start + "," +
-                "time:" + time + "," +
-                "version:" + version;
+        StringBuilder buff = new StringBuilder();
+        buff.append("chunk:").append(Integer.toHexString(id)).
+            append(",block:").append(Long.toHexString(block)).
+            append(",blocks:").append(Integer.toHexString(blocks));
+        if (maxLength != maxLengthLive) {
+            buff.append(",liveMax:").append(Long.toHexString(maxLengthLive));
+        }
+        if (pageCount != pageCountLive) {
+            buff.append(",livePages:").append(Integer.toHexString(pageCountLive));
+        }
+        buff.append(",max:").append(Long.toHexString(maxLength)).
+            append(",pages:").append(Integer.toHexString(pageCount)).
+            append(",root:").append(Long.toHexString(metaRootPos)).
+            append(",time:").append(Long.toHexString(time)).
+            append(",version:").append(Long.toHexString(version));
+        return buff.toString();
     }
 
     @Override

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -828,41 +828,42 @@ public class DataUtils {
     }
 
     /**
-     * Parse a string as a number.
+     * Parse a string as a hexadecimal number.
      *
      * @param x the number
      * @param defaultValue if x is null
      * @return the parsed value
      * @throws IllegalStateException if parsing fails
      */
-    public static long parseLong(String x, long defaultValue) {
+    public static long parseHexLong(String x, long defaultValue) {
         if (x == null) {
             return defaultValue;
         }
         try {
-            return Long.parseLong(x);
+            return Long.parseLong(x, 16);
         } catch (NumberFormatException e) {
             throw newIllegalStateException(ERROR_FILE_CORRUPT,
-                    "Error parsing the value {0} as a long", x, e);
+                    "Error parsing the value {0}", x, e);
         }
     }
     
     /**
-     * Try to parse a string as a number.
+     * Parse a string as a hexadecimal number.
      *
      * @param x the number
      * @param defaultValue if x is null
-     * @param errorValue if parsing fails
-     * @return the parsed value if parsing is possible
+     * @return the parsed value
+     * @throws IllegalStateException if parsing fails
      */
-    public static long parseLong(String x, long defaultValue, long errorValue) {
+    public static int parseHexInt(String x, int defaultValue) {
         if (x == null) {
             return defaultValue;
         }
         try {
-            return Long.parseLong(x);
+            return Integer.parseInt(x, 16);
         } catch (NumberFormatException e) {
-            return errorValue;
+            throw newIllegalStateException(ERROR_FILE_CORRUPT,
+                    "Error parsing the value {0}", x, e);
         }
     }
 

h2/src/main/org/h2/mvstore/MVMap.java

@@ -72,9 +72,8 @@ public class MVMap<K, V> extends AbstractMap<K, V>
      */
     protected void init(MVStore store, HashMap<String, String> config) {
         this.store = store;
-        this.id = Integer.parseInt(config.get("id"));
-        String x = config.get("createVersion");
-        this.createVersion = x == null ? 0 : Long.parseLong(x);
+        this.id = Integer.parseInt(config.get("id"), 16);
+        this.createVersion = DataUtils.parseHexLong(config.get("createVersion"), 0);
         this.writeVersion = store.getCurrentVersion();
     }
 
@@ -1038,8 +1037,8 @@ public class MVMap<K, V> extends AbstractMap<K, V>
         MVMap<K, V> m = new MVMap<K, V>(keyType, valueType);
         m.readOnly = true;
         HashMap<String, String> config = New.hashMap();
-        config.put("id", String.valueOf(id));
-        config.put("createVersion", String.valueOf(createVersion));
+        config.put("id", Integer.toHexString(id));
+        config.put("createVersion", Long.toHexString(createVersion));
         m.init(store, config);
         m.root = root;
         return m;
@@ -1098,7 +1097,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
             DataUtils.appendMap(buff, "name", name);
         }
         if (createVersion != 0) {
-            DataUtils.appendMap(buff, "createVersion", createVersion);
+            DataUtils.appendMap(buff, "createVersion", Long.toHexString(createVersion));
         }
         String type = getType();
         if (type != null) {

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -67,18 +67,21 @@ public class MVStoreTool {
             file = FilePath.get(fileName).open("r");
             long fileLength = file.size();
             pw.println("file " + fileName);
-            pw.println("    length " + fileLength);
+            pw.println("    length " + Long.toHexString(fileLength));
             ByteBuffer block = ByteBuffer.allocate(4096);
             for (long pos = 0; pos < fileLength;) {
                 block.rewind();
                 DataUtils.readFully(file, pos, block);
                 block.rewind();
+                if (block.get() != '{') {
+                    block.position(MVStore.BLOCK_SIZE - MVStore.STORE_HEADER_LENGTH);
                     if (block.get() != '{') {
                         continue;
                     }
+                }
                 byte headerType = block.get();
                 if (headerType == 'H') {
-                    pw.println("    store header at " + pos);
+                    pw.println("    store header at " + Long.toHexString(pos));
                     pw.println("    " + new String(block.array(), "UTF-8").trim());
                     pos += blockSize;
                     continue;
@@ -89,29 +92,32 @@ public class MVStoreTool {
                 }
                 block.position(0);
                 Chunk c = Chunk.fromHeader(block, pos);
-                int chunkLength = c.length;
+                int chunkLength = c.blocks * MVStore.BLOCK_SIZE;
                 pw.println("    " + c.toString());
                 ByteBuffer chunk = ByteBuffer.allocate(chunkLength);
                 DataUtils.readFully(file, pos, chunk);
                 int p = block.position();
-                pos = (pos + chunkLength + blockSize) / blockSize * blockSize;
-                chunkLength -= p;
-                while (chunkLength > 0) {
+                pos += chunkLength;
+                int remaining = c.pageCount;
+                while (remaining > 0) {
                     chunk.position(p);
                     int pageLength = chunk.getInt();
                     // check value (ignored)
                     chunk.getShort();
-                    long mapId = DataUtils.readVarInt(chunk);
+                    int mapId = DataUtils.readVarInt(chunk);
                     int len = DataUtils.readVarInt(chunk);
                     int type = chunk.get();
                     boolean compressed = (type & 2) != 0;
                     boolean node = (type & 1) != 0;
-                    pw.println("        map " + mapId + " at " + p + " " +
-                            (node ? "node" : "leaf") + " " +
-                            (compressed ? "compressed " : "") +
-                            "len: " + pageLength + " entries: " + len);
+                    pw.println(
+                            "        map " + Integer.toHexString(mapId) + 
+                            " at " + Long.toHexString(p) + " " +
+                            (node ? " node" : " leaf") + 
+                            (compressed ? " compressed" : "") +
+                            " len: " + Integer.toHexString(pageLength) + 
+                            " entries: " + Integer.toHexString(len));
                     p += pageLength;
-                    chunkLength -= pageLength;
+                    remaining--;
                     if (mapId == 0 && !compressed) {
                         String[] keys = new String[len];
                         for (int i = 0; i < len; i++) {
@@ -147,6 +153,12 @@ public class MVStoreTool {
                         }
                     }
                 }
+                chunk.position(chunk.limit() - MVStore.STORE_HEADER_LENGTH);
+                if (chunk.get() == '{' && chunk.get() == 'H') {
+                    pw.println("      store header");
+                    pw.println("      " + new String(chunk.array(), chunk.position() - 2, 
+                            MVStore.STORE_HEADER_LENGTH, "UTF-8").trim());
+                }
             }
         } catch (IOException e) {
             pw.println("ERROR: " + e);
@@ -165,8 +177,9 @@ public class MVStoreTool {
     }
 
     private static String getPosString(long pos) {
-        return "pos " + pos + ", chunk " + DataUtils.getPageChunkId(pos) +
-                ", offset " + DataUtils.getPageOffset(pos);
+        return "pos " + Long.toHexString(pos) + 
+                ", chunk " + Integer.toHexString(DataUtils.getPageChunkId(pos)) +
+                ", offset " + Integer.toHexString(DataUtils.getPageOffset(pos));
 
     }
 

h2/src/main/org/h2/mvstore/Page.java

@@ -260,13 +260,13 @@ public class Page {
     public String toString() {
         StringBuilder buff = new StringBuilder();
         buff.append("id: ").append(System.identityHashCode(this)).append('\n');
-        buff.append("pos: ").append(pos).append("\n");
+        buff.append("pos: ").append(Long.toHexString(pos)).append("\n");
         for (int i = 0; i <= keyCount; i++) {
             if (i > 0) {
                 buff.append(" ");
             }
             if (children != null) {
-                buff.append("[" + children[i] + "] ");
+                buff.append("[" + Long.toHexString(children[i]) + "] ");
             }
             if (i < keyCount) {
                 buff.append(keys[i]);

h2/src/test/org/h2/test/store/TestConcurrent.java

@@ -121,7 +121,8 @@ public class TestConcurrent extends TestMVStore {
                     chunkCount++;
                 }
             }
-            assertEquals(1, chunkCount);
+            // the chunk metadata is not yet written
+            assertEquals(0, chunkCount);
             s.close();
         }
         FileUtils.deleteRecursive("memFS:", false);

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -292,12 +292,12 @@ public class TestMVStore extends TestBase {
 
                 }).
                 open();
-        s.setAutoCommitDelay(10);
+        s.setAutoCommitDelay(50);
         MVMap<Integer, String> m;
         m = s.openMap("data");
         s.getFileStore().getFile().close();
         m.put(1, "Hello");
-        for (int i = 0; i < 100; i++) {
+        for (int i = 0; i < 200; i++) {
             if (exRef.get() != null) {
                 break;
             }
@@ -597,7 +597,7 @@ public class TestMVStore extends TestBase {
         }
         s.close();
         int[] expectedReadsForCacheSize = {
-                3407, 2590, 1924, 1440, 1096, 956, 918
+                3407, 2590, 1924, 1440, 1111, 956, 918
         };
         for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) {
             int cacheMB = 1 + 3 * cacheSize;
@@ -612,8 +612,10 @@ public class TestMVStore extends TestBase {
                     assertEquals(10240, x.length());
                 }
             }
-            assertEquals(expectedReadsForCacheSize[cacheSize],
-                    s.getFileStore().getReadCount());
+            long readCount = s.getFileStore().getReadCount();
+            int expected = expectedReadsForCacheSize[cacheSize];
+            assertTrue("reads: " + readCount + " expected: " + expected,
+                    Math.abs(100 - (100 * expected / readCount)) < 5);
             s.close();
         }
 
@@ -648,8 +650,8 @@ public class TestMVStore extends TestBase {
         MVStore s = openStore(fileName);
         long time = System.currentTimeMillis();
         assertEquals("1", s.getStoreHeader().get("format"));
-        long creationTime = Long.parseLong(s.getStoreHeader()
-                .get("creationTime"));
+        long creationTime = Long.parseLong(
+                s.getStoreHeader().get("created"), 16);
         assertTrue(Math.abs(time - creationTime) < 100);
         s.getStoreHeader().put("test", "123");
         MVMap<Integer, Integer> map = s.openMap("test");
@@ -684,7 +686,6 @@ public class TestMVStore extends TestBase {
             }
         }
         s.close();
-
         FilePath f = FilePath.get(fileName);
         int blockSize = 4 * 1024;
         // test corrupt file headers
@@ -692,7 +693,7 @@ public class TestMVStore extends TestBase {
             FileChannel fc = f.open("rw");
             if (i == 0) {
                 // corrupt the last block (the end header)
-                fc.truncate(fc.size() - 4096);
+                fc.write(ByteBuffer.allocate(256), fc.size() - 256);
             }
             ByteBuffer buff = ByteBuffer.allocate(4 * 1024);
             fc.read(buff, i);
@@ -1137,7 +1138,7 @@ public class TestMVStore extends TestBase {
         assertEquals(0, m.size());
         s.commit();
         // ensure only nodes are read, but not leaves
-        assertEquals(42, s.getFileStore().getReadCount());
+        assertEquals(40, s.getFileStore().getReadCount());
         assertEquals(1, s.getFileStore().getWriteCount());
         s.close();
     }
@@ -1295,7 +1296,6 @@ public class TestMVStore extends TestBase {
         data.put("2", "World");
         s.commit();
         assertEquals(1, s.getCurrentVersion());
-        assertTrue(m.containsKey("chunk.1"));
         assertFalse(m.containsKey("chunk.2"));
 
         assertEquals("[data]", s.getMapNames().toString());
@@ -1305,20 +1305,18 @@ public class TestMVStore extends TestBase {
 
         String id = s.getMetaMap().get("name.data");
         assertEquals("name:data", m.get("map." + id));
-        assertTrue(m.containsKey("chunk.1"));
         assertEquals("Hello", data.put("1", "Hallo"));
         s.commit();
         assertEquals("name:data", m.get("map." + id));
         assertTrue(m.get("root.1").length() > 0);
         assertTrue(m.containsKey("chunk.1"));
 
-        assertTrue(m.containsKey("chunk.2"));
         assertEquals(2, s.getCurrentVersion());
 
         s.rollbackTo(1);
         assertEquals("Hello", data.get("1"));
         assertEquals("World", data.get("2"));
-        assertTrue(m.containsKey("chunk.1"));
+        assertFalse(m.containsKey("chunk.1"));
         assertFalse(m.containsKey("chunk.2"));
 
         s.close();