MVStore: the file format was changed slightly.

h2/src/docsrc/html/changelog.html

@@ -20,6 +20,7 @@ Change Log
 <h2>Next Version (unreleased)</h2>
 <ul><li>Issue 545: Unnecessary duplicate code was removed. 
 </li><li>The profiler tool can now process files with full thread dumps.
+</li><li>MVStore: the file format was changed slightly.
 </li><li>MVStore mode: the CLOB and BLOB storage was re-implemented and is
     now much faster than with the PageStore (which is still the default storage).
 </li><li>MVStore mode: creating indexes is now much faster

h2/src/main/org/h2/mvstore/Chunk.java

@@ -15,7 +15,7 @@ import java.util.HashMap;
  * Chunks are page aligned (each page is usually 4096 bytes).
  * There are at most 67 million (2^26) chunks,
  * each chunk is at most 2 GB large.
- * File format:
+ * Chunk format:
  * 1 byte: 'c'
  * 4 bytes: length
  * 4 bytes: chunk id (an incrementing number)
@@ -26,6 +26,11 @@ import java.util.HashMap;
  */
 public class Chunk {
 
+    /**
+     * The maximum length of a chunk header, in bytes.
+     */
+    static final int MAX_HEADER_LENGTH = 1024;
+
     /**
      * The chunk id.
      */
@@ -93,26 +98,23 @@ public class Chunk {
      * @return the chunk
      */
     static Chunk fromHeader(ByteBuffer buff, long start) {
-        if (buff.get() != 'c') {
+        int pos = buff.position();
+        if (buff.get() != '{') {
             throw DataUtils.newIllegalStateException(
                     DataUtils.ERROR_FILE_CORRUPT,
                     "File corrupt reading chunk at position {0}", start);
         }
-        int length = buff.getInt();
-        int chunkId = buff.getInt();
-        int pageCount = buff.getInt();
-        long metaRootPos = buff.getLong();
-        long maxLength = buff.getLong();
-        long maxLengthLive = buff.getLong();
-        Chunk c = new Chunk(chunkId);
-        c.length = length;
-        c.pageCount = pageCount;
-        c.pageCountLive = pageCount;
-        c.start = start;
-        c.metaRootPos = metaRootPos;
-        c.maxLength = maxLength;
-        c.maxLengthLive = maxLengthLive;
-        return c;
+        byte[] data = new byte[Math.min(buff.remaining(), MAX_HEADER_LENGTH)];
+        // set the position to the start of the first page
+        buff.get(data);
+        for (int i = 0; i < data.length; i++) {
+            if (data[i] == '\n') {
+                buff.position(pos + i + 2);
+                break;
+            }
+        }
+        String s = new String(data, 0, data.length, DataUtils.UTF8);
+        return fromString(s);
     }
 
     /**
@@ -121,13 +123,10 @@ public class Chunk {
      * @param buff the target buffer
      */
     void writeHeader(WriteBuffer buff) {
-        buff.put((byte) 'c').
-            putInt(length).
-            putInt(id).
-            putInt(pageCount).
-            putLong(metaRootPos).
-            putLong(maxLength).
-            putLong(maxLengthLive);
+        buff.put((byte) '{');
+        buff.put(asString().getBytes(DataUtils.UTF8));
+        buff.put((byte) '}');
+        buff.put((byte) ' ');
     }
 
     /**
@@ -138,7 +137,7 @@ public class Chunk {
      */
     public static Chunk fromString(String s) {
         HashMap<String, String> map = DataUtils.parseMap(s);
-        int id = Integer.parseInt(map.get("id"));
+        int id = Integer.parseInt(map.get("chunk"));
         Chunk c = new Chunk(id);
         c.start = Long.parseLong(map.get("start"));
         c.length = Integer.parseInt(map.get("length"));
@@ -173,7 +172,7 @@ public class Chunk {
      */
     public String asString() {
         return
-                "id:" + id + "," +
+                "chunk:" + id + "," +
                 "length:" + length + "," +
                 "maxLength:" + maxLength + "," +
                 "maxLengthLive:" + maxLengthLive + "," +

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -570,7 +570,7 @@ public class DataUtils {
         }
         buff.append(key).append(':');
         String v = value.toString();
-        if (v.indexOf(',') < 0 && v.indexOf('\"') < 0) {
+        if (v.indexOf(',') < 0 && v.indexOf('\"') < 0 && v.indexOf('}') < 0) {
             buff.append(value);
         } else {
             buff.append('\"');
@@ -595,6 +595,9 @@ public class DataUtils {
     public static HashMap<String, String> parseMap(String s) {
         HashMap<String, String> map = New.hashMap();
         for (int i = 0, size = s.length(); i < size;) {
+            if (s.charAt(i) == '}') {
+                break;
+            }
             int startKey = i;
             i = s.indexOf(':', i);
             if (i < 0) {
@@ -607,6 +610,9 @@ public class DataUtils {
                 char c = s.charAt(i++);
                 if (c == ',') {
                     break;
+                } else if (c == '}') {
+                    i--;
+                    break;
                 } else if (c == '\"') {
                     while (i < size) {
                         c = s.charAt(i++);
@@ -641,7 +647,8 @@ public class DataUtils {
     public static int getFletcher32(byte[] bytes, int length) {
         int s1 = 0xffff, s2 = 0xffff;
         for (int i = 0; i < length;) {
-            for (int end = Math.min(i + 718, length); i < end;) {
+            // reduce after 360 words (each word is two bytes)
+            for (int end = Math.min(i + 720, length); i < end;) {
                 int x = ((bytes[i++] & 0xff) << 8) | (bytes[i++] & 0xff);
                 s2 += s1 += x;
             }
@@ -735,8 +742,8 @@ public class DataUtils {
             Object a = arguments[i];
             if (!(a instanceof Exception)) {
                 String s = a == null ? "null" : a.toString();
-                if (s.length() > 100) {
-                    s = s.substring(0, 100);
+                if (s.length() > 1000) {
+                    s = s.substring(0, 1000) + "...";
                 }
                 arguments[i] = s;
             }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -30,13 +30,15 @@ import org.h2.util.New;
 /*
 
 File format:
-header: (blockSize) bytes
-header: (blockSize) bytes
+
+store header: (blockSize) bytes
+store header: (blockSize) bytes
 [ chunk ] *
 (there are two headers for security at the beginning of the file,
-and there is a header after each chunk)
-header:
-H:3,...
+and there is a store header at the end of each chunk)
+
+store header:
+{H:2,...
 
 Format: 
 Current store header:
@@ -63,8 +65,6 @@ pageCountLive -> livePages, maxLength -> max, maxLengthLive -> liveMax,
 metaRootPos -> root (offset))
 +, if different: maxLive:1030,pagesLive:30
 
-compression: support multiple algorithms
-
 TODO:
 
 Documentation
@@ -148,8 +148,7 @@ MVStore:
     specially for large pages (when using the StreamStore)
 - StreamStore: split blocks similar to rsync crypto, where the split is made
     "if the sum of the past 8196 bytes divides by 4096 with zero remainder"
-- Compression: try using a bloom filter before trying to match
-- DataType: change to reading and writing arrays, not individual entries
+- Compression: try using a bloom filter (64 bit) before trying to match
 
 */
 
@@ -336,7 +335,6 @@ public class MVStore {
                 creationTime = 0;
                 creationTime = getTime();
                 lastCommitTime = creationTime;
-                storeHeader.put("H", "3");
                 storeHeader.put("blockSize", "" + BLOCK_SIZE);
                 storeHeader.put("format", "" + FORMAT_WRITE);
                 storeHeader.put("creationTime", "" + creationTime);
@@ -625,7 +623,7 @@ public class MVStore {
             HashMap<String, String> m;
             try {
                 m = DataUtils.parseMap(s);
-            } catch (IllegalArgumentException e) {
+            } catch (IllegalStateException e) {
                 continue;
             }
             String f = m.remove("fletcher");
@@ -664,7 +662,7 @@ public class MVStore {
     }
 
     private byte[] getStoreHeaderBytes() {
-        StringBuilder buff = new StringBuilder();
+        StringBuilder buff = new StringBuilder("{H:2");
         storeHeader.put("lastMapId", "" + lastMapId);
         storeHeader.put("chunk", "" + lastChunkId);
         storeHeader.put("rootChunk", "" + rootChunkStart);
@@ -673,6 +671,7 @@ public class MVStore {
         byte[] bytes = buff.toString().getBytes(DataUtils.UTF8);
         int checksum = DataUtils.getFletcher32(bytes, bytes.length / 2 * 2);
         DataUtils.appendMap(buff, "fletcher", Integer.toHexString(checksum));
+        buff.append("}\n");
         bytes = buff.toString().getBytes(DataUtils.UTF8);
         if (bytes.length > BLOCK_SIZE) {
             throw DataUtils.newIllegalStateException(
@@ -887,8 +886,11 @@ public class MVStore {
         }
         Chunk c;
         c = new Chunk(++lastChunkId);
+        c.pageCount = Integer.MAX_VALUE;
+        c.pageCountLive = Integer.MAX_VALUE;
         c.maxLength = Long.MAX_VALUE;
         c.maxLengthLive = Long.MAX_VALUE;
+        c.metaRootPos = Long.MAX_VALUE;
         c.start = Long.MAX_VALUE;
         c.length = Integer.MAX_VALUE;
         c.time = time;
@@ -924,6 +926,9 @@ public class MVStore {
         WriteBuffer buff = getWriteBuffer();
         // need to patch the header later
         c.writeHeader(buff);
+        long endHeader = buff.position();
+        c.pageCount = 0;
+        c.pageCountLive = 0;
         c.maxLength = 0;
         c.maxLengthLive = 0;
         for (MVMap<?, ?> m : changed) {
@@ -973,6 +978,10 @@ public class MVStore {
         c.metaRootPos = metaRoot.getPos();
         buff.position(0);
         c.writeHeader(buff);
+        while (buff.position() < endHeader - 1) {
+            buff.put((byte) ' ');
+        }
+        buff.put((byte) '\n');
         rootChunkStart = filePos;
         revertTemp(storeVersion);
 
@@ -1185,7 +1194,7 @@ public class MVStore {
     }
 
     private Chunk readChunkHeader(long start) {
-        ByteBuffer buff = fileStore.readFully(start, 40);
+        ByteBuffer buff = fileStore.readFully(start, Chunk.MAX_HEADER_LENGTH);
         return Chunk.fromHeader(buff, start);
     }
 

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -73,30 +73,24 @@ public class MVStoreTool {
                 block.rewind();
                 DataUtils.readFully(file, pos, block);
                 block.rewind();
-                int tag = block.get();
-                if (tag == 'H') {
-                    pw.println("    header at " + pos);
+                if (block.get() != '{') {
+                    continue;
+                }
+                byte headerType = block.get();
+                if (headerType == 'H') {
+                    pw.println("    store header at " + pos);
                     pw.println("    " + new String(block.array(), "UTF-8").trim());
                     pos += blockSize;
                     continue;
                 }
-                if (tag != 'c') {
+                if (headerType != 'c') {
                     pos += blockSize;
                     continue;
                 }
-                int chunkLength = block.getInt();
-                int chunkId = block.getInt();
-                int pageCount = block.getInt();
-                long metaRootPos = block.getLong();
-                long maxLength = block.getLong();
-                long maxLengthLive = block.getLong();
-                pw.println("    chunk " + chunkId +
-                        " at " + pos +
-                        " length " + chunkLength +
-                        " pageCount " + pageCount +
-                        " root " + getPosString(metaRootPos) +
-                        " maxLength " + maxLength +
-                        " maxLengthLive " + maxLengthLive);
+                block.position(0);
+                Chunk c = Chunk.fromHeader(block, pos);
+                int chunkLength = c.length;
+                pw.println("    " + c.toString());
                 ByteBuffer chunk = ByteBuffer.allocate(chunkLength);
                 DataUtils.readFully(file, pos, chunk);
                 int p = block.position();

h2/src/main/org/h2/mvstore/Page.java

@@ -758,11 +758,7 @@ public class Page {
             buff = ByteBuffer.allocate(l);
             compressor.expand(comp, 0, compLen, buff.array(), buff.arrayOffset(), l);
         }
-        DataType keyType = map.getKeyType();
-        for (int i = 0; i < len; i++) {
-            Object k = keyType.read(buff);
-            keys[i] = k;
-        }
+        map.getKeyType().read(buff, keys, len, true);
         if (node) {
             childCount = len + 1;
             children = new long[len + 1];
@@ -780,11 +776,7 @@ public class Page {
             totalCount = total;
         } else {
             values = new Object[len];
-            DataType valueType = map.getValueType();
-            for (int i = 0; i < len; i++) {
-                Object v = valueType.read(buff);
-                values[i] = v;
-            }
+            map.getValueType().read(buff, values, len, false);
             totalCount = len;
         }
         recalculateMemory();
@@ -807,10 +799,7 @@ public class Page {
             putVarInt(len).
             put((byte) type);
         int compressStart = buff.position();
-        DataType keyType = map.getKeyType();
-        for (int i = 0; i < len; i++) {
-            keyType.write(buff, keys[i]);
-        }
+        map.getKeyType().write(buff, keys, len, true);
         if (type == DataUtils.PAGE_TYPE_NODE) {
             for (int i = 0; i <= len; i++) {
                 buff.putLong(children[i]);
@@ -819,10 +808,7 @@ public class Page {
                 buff.putVarLong(counts[i]);
             }
         } else {
-            DataType valueType = map.getValueType();
-            for (int i = 0; i < len; i++) {
-                valueType.write(buff, values[i]);
-            }
+            map.getValueType().write(buff, values, len, false);
         }
         MVStore store = map.getStore();
         if (store.getCompress()) {

h2/src/main/org/h2/mvstore/db/TransactionStore.java

@@ -1509,6 +1509,20 @@ public class TransactionStore {
             return Long.signum(comp);
         }
         
+        @Override
+        public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+            for (int i = 0; i < len; i++) {
+                obj[i] = read(buff);
+            }
+        }
+        
+        @Override
+        public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+            for (int i = 0; i < len; i++) {
+                write(buff, obj[i]);
+            }
+        }
+
         @Override
         public void write(WriteBuffer buff, Object obj) {
             VersionedValue v = (VersionedValue) obj;
@@ -1578,6 +1592,20 @@ public class TransactionStore {
             return 0;
         }
         
+        @Override
+        public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+            for (int i = 0; i < len; i++) {
+                obj[i] = read(buff);
+            }
+        }
+        
+        @Override
+        public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+            for (int i = 0; i < len; i++) {
+                write(buff, obj[i]);
+            }
+        }
+
         @Override
         public void write(WriteBuffer buff, Object obj) {
             Object[] array = (Object[]) obj;

h2/src/main/org/h2/mvstore/db/ValueDataType.java

@@ -144,6 +144,20 @@ public class ValueDataType implements DataType {
         return v == null ? 0 : v.getMemory();
     }
     
+    @Override
+    public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            obj[i] = read(buff);
+        }
+    }
+    
+    @Override
+    public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            write(buff, obj[i]);
+        }
+    }
+
     @Override
     public Object read(ByteBuffer buff) {
         return readValue(buff);

h2/src/main/org/h2/mvstore/rtree/SpatialDataType.java

@@ -53,6 +53,20 @@ public class SpatialDataType implements DataType {
         return 40 + dimensions * 4;
     }
     
+    @Override
+    public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            obj[i] = read(buff);
+        }
+    }
+    
+    @Override
+    public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            write(buff, obj[i]);
+        }
+    }
+
     @Override
     public void write(WriteBuffer buff, Object obj) {
         SpatialKey k = (SpatialKey) obj;

h2/src/main/org/h2/mvstore/type/DataType.java

@@ -34,13 +34,23 @@ public interface DataType {
     int getMemory(Object obj);
 
     /**
-     * Write the object.
+     * Write an object.
      *
      * @param buff the target buffer
      * @param obj the value
      */
     void write(WriteBuffer buff, Object obj);
     
+    /**
+     * Write a list of objects.
+     *
+     * @param buff the target buffer
+     * @param obj the objects
+     * @param len the number of objects to write
+     * @param key whether the objects are keys
+     */
+    void write(WriteBuffer buff, Object[] obj, int len, boolean key);
+
     /**
      * Read an object.
      *
@@ -49,5 +59,15 @@ public interface DataType {
      */
     Object read(ByteBuffer buff);
     
+    /**
+     * Read a list of objects.
+     *
+     * @param buff the target buffer
+     * @param obj the objects
+     * @param len the number of objects to read
+     * @param key whether the objects are keys
+     */
+    void read(ByteBuffer buff, Object[] obj, int len, boolean key);
+
 }
 

h2/src/main/org/h2/mvstore/type/StringDataType.java

@@ -27,6 +27,20 @@ public class StringDataType implements DataType {
         return 24 + 2 * obj.toString().length();
     }
     
+    @Override
+    public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            obj[i] = read(buff);
+        }
+    }
+    
+    @Override
+    public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            write(buff, obj[i]);
+        }
+    }
+
     @Override
     public String read(ByteBuffer buff) {
         int len = DataUtils.readVarInt(buff);

h2/src/test/org/h2/test/store/RowDataType.java

@@ -59,6 +59,20 @@ public class RowDataType implements DataType {
         return memory;
     }
     
+    @Override
+    public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            obj[i] = read(buff);
+        }
+    }
+    
+    @Override
+    public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+        for (int i = 0; i < len; i++) {
+            write(buff, obj[i]);
+        }
+    }
+
     @Override
     public Object[] read(ByteBuffer buff) {
         int len = DataUtils.readVarInt(buff);

h2/src/test/org/h2/test/store/TestDataUtils.java

@@ -79,15 +79,17 @@ public class TestDataUtils extends TestBase {
         DataUtils.appendMap(buff,  "b", ",");
         DataUtils.appendMap(buff,  "c", "1,2");
         DataUtils.appendMap(buff,  "d", "\"test\"");
-        assertEquals(":,a:1,b:\",\",c:\"1,2\",d:\"\\\"test\\\"\"", buff.toString());
+        DataUtils.appendMap(buff,  "e", "}");
+        assertEquals(":,a:1,b:\",\",c:\"1,2\",d:\"\\\"test\\\"\",e:\"}\"", buff.toString());
 
         HashMap<String, String> m = DataUtils.parseMap(buff.toString());
-        assertEquals(5, m.size());
+        assertEquals(6, m.size());
         assertEquals("", m.get(""));
         assertEquals("1", m.get("a"));
         assertEquals(",", m.get("b"));
         assertEquals("1,2", m.get("c"));
         assertEquals("\"test\"", m.get("d"));
+        assertEquals("}", m.get("e"));
     }
 
     private void testMapRandomized() {

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -597,7 +597,7 @@ public class TestMVStore extends TestBase {
         }
         s.close();
         int[] expectedReadsForCacheSize = {
-                3406, 2590, 1924, 1440, 1102, 956, 918
+                3407, 2590, 1924, 1440, 1096, 956, 918
         };
         for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) {
             int cacheMB = 1 + 3 * cacheSize;
@@ -647,7 +647,7 @@ public class TestMVStore extends TestBase {
         String fileName = getBaseDir() + "/testFileHeader.h3";
         MVStore s = openStore(fileName);
         long time = System.currentTimeMillis();
-        assertEquals("3", s.getStoreHeader().get("H"));
+        assertEquals("1", s.getStoreHeader().get("format"));
         long creationTime = Long.parseLong(s.getStoreHeader()
                 .get("creationTime"));
         assertTrue(Math.abs(time - creationTime) < 100);