MVStore: the file format was changed slightly.

h2/src/main/org/h2/mvstore/Chunk.java

@@ -36,7 +36,7 @@ public class Chunk {
     /**
      * The length in number of blocks.
      */
-    public int blocks;
+    public int len;
 
     /**
      * The total number of pages in this chunk.
@@ -56,7 +56,7 @@ public class Chunk {
     /**
      * The sum of the max length of all pages that are in use.
      */
-    public long maxLengthLive;
+    public long maxLenLive;
 
     /**
      * The garbage collection priority.
@@ -78,6 +78,11 @@ public class Chunk {
      */
     public long time;
 
+    /**
+     * The last used map id.
+     */
+    public int mapId;
+
     Chunk(int id) {
         this.id = id;
     }
@@ -91,22 +96,23 @@ public class Chunk {
      */
     static Chunk fromHeader(ByteBuffer buff, long start) {
         int pos = buff.position();
-        if (buff.get() != '{') {
-            throw DataUtils.newIllegalStateException(
-                    DataUtils.ERROR_FILE_CORRUPT,
-                    "File corrupt reading chunk at position {0}", start);
-        }
         byte[] data = new byte[Math.min(buff.remaining(), MAX_HEADER_LENGTH)];
-        // set the position to the start of the first page
         buff.get(data);
-        for (int i = 0; i < data.length; i++) {
-            if (data[i] == '\n') {
-                buff.position(pos + i + 2);
-                break;
+        try {
+            for (int i = 0; i < data.length; i++) {
+                if (data[i] == '\n') {
+                    // set the position to the start of the first page
+                    buff.position(pos + i + 1);
+                    String s = new String(data, 0, i, DataUtils.LATIN).trim();
+                    return fromString(s);
+                }
             }
+        } catch (Exception e) {
+            // there could be various reasons
         }
-        String s = new String(data, 0, data.length, DataUtils.UTF8);
-        return fromString(s);
+        throw DataUtils.newIllegalStateException(
+                DataUtils.ERROR_FILE_CORRUPT,
+                "File corrupt reading chunk at position {0}", start);
     }
 
     /**
@@ -116,9 +122,7 @@ public class Chunk {
      */
     void writeHeader(WriteBuffer buff, int minLength) {
         long pos = buff.position();
-        buff.put((byte) '{');
-        buff.put(asString().getBytes(DataUtils.UTF8));
-        buff.put((byte) '}');
+        buff.put(asString().getBytes(DataUtils.LATIN));
         while (buff.position() - pos < minLength - 1) {
             buff.put((byte) ' ');
         }
@@ -137,22 +141,23 @@ public class Chunk {
      */
     public static Chunk fromString(String s) {
         HashMap<String, String> map = DataUtils.parseMap(s);
-        int id = Integer.parseInt(map.get("chunk"), 16);
+        int id = DataUtils.readHexInt(map, "chunk", 0);
         Chunk c = new Chunk(id);
-        c.block = Long.parseLong(map.get("block"), 16);
-        c.blocks = Integer.parseInt(map.get("blocks"), 16);
-        c.pageCount = Integer.parseInt(map.get("pages"), 16);
-        c.pageCountLive = DataUtils.parseHexInt(map.get("livePages"), c.pageCount);
-        c.maxLength = Long.parseLong(map.get("max"), 16);
-        c.maxLengthLive = DataUtils.parseHexLong(map.get("liveMax"), c.maxLength);
-        c.metaRootPos = Long.parseLong(map.get("root"), 16);
-        c.time = Long.parseLong(map.get("time"), 16);
-        c.version = Long.parseLong(map.get("version"), 16);
+        c.block = DataUtils.readHexLong(map, "block", 0);
+        c.len = DataUtils.readHexInt(map, "len", 0);
+        c.pageCount = DataUtils.readHexInt(map, "pages", 0);
+        c.pageCountLive = DataUtils.readHexInt(map, "livePages", c.pageCount);
+        c.mapId = Integer.parseInt(map.get("map"), 16);
+        c.maxLength = DataUtils.readHexLong(map, "max", 0);
+        c.maxLenLive = DataUtils.readHexLong(map, "liveMax", c.maxLength);
+        c.metaRootPos = DataUtils.readHexLong(map, "root", 0);
+        c.time = DataUtils.readHexLong(map, "time", 0);
+        c.version = DataUtils.readHexLong(map, "version", 0);
         return c;
     }
 
     public int getFillRate() {
-        return (int) (maxLength == 0 ? 0 : 100 * maxLengthLive / maxLength);
+        return (int) (maxLength == 0 ? 0 : 100 * maxLenLive / maxLength);
     }
 
     @Override
@@ -172,20 +177,21 @@ public class Chunk {
      */
     public String asString() {
         StringBuilder buff = new StringBuilder();
-        buff.append("chunk:").append(Integer.toHexString(id)).
-            append(",block:").append(Long.toHexString(block)).
-            append(",blocks:").append(Integer.toHexString(blocks));
-        if (maxLength != maxLengthLive) {
-            buff.append(",liveMax:").append(Long.toHexString(maxLengthLive));
+        DataUtils.appendMap(buff, "chunk", id);
+        DataUtils.appendMap(buff, "block", block);
+        DataUtils.appendMap(buff, "len", len);
+        if (maxLength != maxLenLive) {
+            DataUtils.appendMap(buff, "liveMax", maxLenLive);
         }
         if (pageCount != pageCountLive) {
-            buff.append(",livePages:").append(Integer.toHexString(pageCountLive));
+            DataUtils.appendMap(buff, "livePages", pageCountLive);
         }
-        buff.append(",max:").append(Long.toHexString(maxLength)).
-            append(",pages:").append(Integer.toHexString(pageCount)).
-            append(",root:").append(Long.toHexString(metaRootPos)).
-            append(",time:").append(Long.toHexString(time)).
-            append(",version:").append(Long.toHexString(version));
+        DataUtils.appendMap(buff, "map", mapId);
+        DataUtils.appendMap(buff, "max", maxLength);
+        DataUtils.appendMap(buff, "pages", pageCount);
+        DataUtils.appendMap(buff, "root", metaRootPos);
+        DataUtils.appendMap(buff, "time", time);
+        DataUtils.appendMap(buff, "version", version);
         return buff.toString();
     }
 

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -131,9 +131,14 @@ public class DataUtils {
     public static final int PAGE_MEMORY_CHILD = 16;
 
     /**
-     * Name of the character encoding format.
+     * The UTF-8 character encoding format.
      */
     public static final Charset UTF8 = Charset.forName("UTF-8");
+    
+    /**
+     * The ISO Latin character encoding format.
+     */
+    public static final Charset LATIN = Charset.forName("ISO-8859-1");
 
     /**
      * An 0-size byte array.
@@ -554,7 +559,7 @@ public class DataUtils {
         }
         return buff;
     }
-
+    
     /**
      * Append a key-value pair to the string builder. Keys may not contain a
      * colon. Values that contain a comma or a double quote are enclosed in
@@ -569,9 +574,16 @@ public class DataUtils {
             buff.append(',');
         }
         buff.append(key).append(':');
-        String v = value.toString();
-        if (v.indexOf(',') < 0 && v.indexOf('\"') < 0 && v.indexOf('}') < 0) {
-            buff.append(value);
+        String v;
+        if (value instanceof Long) {
+            v = Long.toHexString((Long) value);
+        } else if (value instanceof Integer) {
+            v = Integer.toHexString((Integer) value);
+        } else {
+            v = value.toString();
+        }
+        if (v.indexOf(',') < 0 && v.indexOf('\"') < 0) {
+            buff.append(v);
         } else {
             buff.append('\"');
             for (int i = 0, size = v.length(); i < size; i++) {
@@ -595,9 +607,6 @@ public class DataUtils {
     public static HashMap<String, String> parseMap(String s) {
         HashMap<String, String> map = New.hashMap();
         for (int i = 0, size = s.length(); i < size;) {
-            if (s.charAt(i) == '}') {
-                break;
-            }
             int startKey = i;
             i = s.indexOf(':', i);
             if (i < 0) {
@@ -610,9 +619,6 @@ public class DataUtils {
                 char c = s.charAt(i++);
                 if (c == ',') {
                     break;
-                } else if (c == '}') {
-                    i--;
-                    break;
                 } else if (c == '\"') {
                     while (i < size) {
                         c = s.charAt(i++);
@@ -828,42 +834,51 @@ public class DataUtils {
     }
 
     /**
-     * Parse a string as a hexadecimal number.
+     * Read a hex long value from a map.
      *
-     * @param x the number
-     * @param defaultValue if x is null
+     * @param map the map
+     * @param key the key
+     * @param defaultValue if the value is null
      * @return the parsed value
      * @throws IllegalStateException if parsing fails
      */
-    public static long parseHexLong(String x, long defaultValue) {
-        if (x == null) {
+    public static long readHexLong(HashMap<String, ? extends Object> map, String key, long defaultValue) {
+        Object v = map.get(key);
+        if (v == null) {
             return defaultValue;
+        } else if (v instanceof Long) {
+            return (Long) v;
         }
         try {
-            return Long.parseLong(x, 16);
+            return Long.parseLong((String) v, 16);
         } catch (NumberFormatException e) {
             throw newIllegalStateException(ERROR_FILE_CORRUPT,
-                    "Error parsing the value {0}", x, e);
+                    "Error parsing the value {0}", v, e);
         }
     }
     
     /**
-     * Parse a string as a hexadecimal number.
+     * Read a hex int value from a map.
      *
-     * @param x the number
-     * @param defaultValue if x is null
+     * @param map the map
+     * @param key the key
+     * @param defaultValue if the value is null
      * @return the parsed value
      * @throws IllegalStateException if parsing fails
      */
-    public static int parseHexInt(String x, int defaultValue) {
-        if (x == null) {
+    public static int readHexInt(HashMap<String, ? extends Object> map, String key, int defaultValue) {
+        Object v = map.get(key);
+        if (v == null) {
             return defaultValue;
+        } else if (v instanceof Integer) {
+            return (Integer) v;
         }
         try {
-            return Integer.parseInt(x, 16);
+            // support unsigned hex value
+            return (int) Long.parseLong((String) v, 16);
         } catch (NumberFormatException e) {
             throw newIllegalStateException(ERROR_FILE_CORRUPT,
-                    "Error parsing the value {0}", x, e);
+                    "Error parsing the value {0}", v, e);
         }
     }
 

h2/src/main/org/h2/mvstore/MVMap.java

@@ -70,10 +70,10 @@ public class MVMap<K, V> extends AbstractMap<K, V>
      * @param store the store
      * @param config the configuration
      */
-    protected void init(MVStore store, HashMap<String, String> config) {
+    protected void init(MVStore store, HashMap<String, Object> config) {
         this.store = store;
-        this.id = Integer.parseInt(config.get("id"), 16);
-        this.createVersion = DataUtils.parseHexLong(config.get("createVersion"), 0);
+        this.id = DataUtils.readHexInt(config, "id", 0);
+        this.createVersion = DataUtils.readHexLong(config, "createVersion", 0);
         this.writeVersion = store.getCurrentVersion();
     }
 
@@ -1036,9 +1036,9 @@ public class MVMap<K, V> extends AbstractMap<K, V>
     MVMap<K, V> openReadOnly() {
         MVMap<K, V> m = new MVMap<K, V>(keyType, valueType);
         m.readOnly = true;
-        HashMap<String, String> config = New.hashMap();
-        config.put("id", Integer.toHexString(id));
-        config.put("createVersion", Long.toHexString(createVersion));
+        HashMap<String, Object> config = New.hashMap();
+        config.put("id", id);
+        config.put("createVersion", createVersion);
         m.init(store, config);
         m.root = root;
         return m;
@@ -1097,7 +1097,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
             DataUtils.appendMap(buff, "name", name);
         }
         if (createVersion != 0) {
-            DataUtils.appendMap(buff, "createVersion", Long.toHexString(createVersion));
+            DataUtils.appendMap(buff, "createVersion", createVersion);
         }
         String type = getType();
         if (type != null) {

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -73,13 +73,7 @@ public class MVStoreTool {
                 block.rewind();
                 DataUtils.readFully(file, pos, block);
                 block.rewind();
-                if (block.get() != '{') {
-                    block.position(MVStore.BLOCK_SIZE - MVStore.STORE_HEADER_LENGTH);
-                    if (block.get() != '{') {
-                        continue;
-                    }
-                }
-                byte headerType = block.get();
+                int headerType = block.get();
                 if (headerType == 'H') {
                     pw.println("    store header at " + Long.toHexString(pos));
                     pw.println("    " + new String(block.array(), "UTF-8").trim());
@@ -92,12 +86,12 @@ public class MVStoreTool {
                 }
                 block.position(0);
                 Chunk c = Chunk.fromHeader(block, pos);
-                int chunkLength = c.blocks * MVStore.BLOCK_SIZE;
+                int length = c.len * MVStore.BLOCK_SIZE;
                 pw.println("    " + c.toString());
-                ByteBuffer chunk = ByteBuffer.allocate(chunkLength);
+                ByteBuffer chunk = ByteBuffer.allocate(length);
                 DataUtils.readFully(file, pos, chunk);
                 int p = block.position();
-                pos += chunkLength;
+                pos += length;
                 int remaining = c.pageCount;
                 while (remaining > 0) {
                     chunk.position(p);
@@ -153,12 +147,9 @@ public class MVStoreTool {
                         }
                     }
                 }
-                chunk.position(chunk.limit() - MVStore.STORE_HEADER_LENGTH);
-                if (chunk.get() == '{' && chunk.get() == 'H') {
-                    pw.println("      store header");
-                    pw.println("      " + new String(chunk.array(), chunk.position() - 2, 
-                            MVStore.STORE_HEADER_LENGTH, "UTF-8").trim());
-                }
+                chunk.position(chunk.limit() - MVStore.CHUNK_FOOTER_LENGTH);
+                pw.println("      store header");
+                pw.println("      " + new String(chunk.array(), chunk.position(), MVStore.CHUNK_FOOTER_LENGTH, "UTF-8").trim());
             }
         } catch (IOException e) {
             pw.println("ERROR: " + e);

h2/src/main/org/h2/mvstore/Page.java

@@ -841,7 +841,7 @@ public class Page {
         store.cachePage(pos, this, getMemory());
         long max = DataUtils.getPageMaxLength(pos);
         chunk.maxLength += max;
-        chunk.maxLengthLive += max;
+        chunk.maxLenLive += max;
         chunk.pageCount++;
         chunk.pageCountLive++;
     }

h2/src/test/org/h2/test/store/SequenceMap.java

@@ -33,7 +33,7 @@ public class SequenceMap extends MVMap<Long, Long> {
     }
 
     @Override
-    public void init(MVStore store, HashMap<String, String> config) {
+    public void init(MVStore store, HashMap<String, Object> config) {
         super.init(store, config);
     }
 

h2/src/test/org/h2/test/store/TestDataUtils.java

@@ -80,7 +80,7 @@ public class TestDataUtils extends TestBase {
         DataUtils.appendMap(buff,  "c", "1,2");
         DataUtils.appendMap(buff,  "d", "\"test\"");
         DataUtils.appendMap(buff,  "e", "}");
-        assertEquals(":,a:1,b:\",\",c:\"1,2\",d:\"\\\"test\\\"\",e:\"}\"", buff.toString());
+        assertEquals(":,a:1,b:\",\",c:\"1,2\",d:\"\\\"test\\\"\",e:}", buff.toString());
 
         HashMap<String, String> m = DataUtils.parseMap(buff.toString());
         assertEquals(6, m.size());

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -212,14 +212,21 @@ public class TestMVStore extends TestBase {
                 encryptionKey("007".toCharArray()).
                 fileName(fileName).
                 open();
-        Map<String, String> header = s.getStoreHeader();
-        assertEquals("1", header.get("format"));
+        Map<String, Object> header = s.getStoreHeader();
+        assertEquals("1", header.get("format").toString());
         header.put("formatRead", "1");
         header.put("format", "2");
         MVMap<Integer, String> m = s.openMap("data");
+        // this is to ensure the file header is overwritten
+        for (int i = 0; i < 10; i++) {
+            m.put(0, "Hello World " + i);
+            s.commit();
+            if (i > 5) {
+                s.setRetentionTime(0);
+            }
+        }
         m.put(0, "Hello World");
         s.close();
-
         try {
             s = new MVStore.Builder().
                     encryptionKey("007".toCharArray()).
@@ -523,10 +530,17 @@ public class TestMVStore extends TestBase {
         s = openStore(fileName);
         m = s.openMap("test");
         m.put(1, 1);
-        Map<String, String> header = s.getStoreHeader();
-        int format = Integer.parseInt(header.get("format"));
+        Map<String, Object> header = s.getStoreHeader();
+        int format = Integer.parseInt(header.get("format").toString());
         assertEquals(1, format);
         header.put("format", Integer.toString(format + 1));
+        // ensure the file header is overwritten
+        s.commit();
+        m.put(1, 10);
+        s.commit();
+        m.put(1, 20);
+        s.setRetentionTime(0);
+        s.commit();
         s.close();
         try {
             openStore(fileName).close();
@@ -649,16 +663,23 @@ public class TestMVStore extends TestBase {
         String fileName = getBaseDir() + "/testFileHeader.h3";
         MVStore s = openStore(fileName);
         long time = System.currentTimeMillis();
-        assertEquals("1", s.getStoreHeader().get("format"));
-        long creationTime = Long.parseLong(
-                s.getStoreHeader().get("created"), 16);
+        Map<String, Object> m = s.getStoreHeader();
+        assertEquals("1", m.get("format").toString());
+        long creationTime = (Long) m.get("created");
         assertTrue(Math.abs(time - creationTime) < 100);
-        s.getStoreHeader().put("test", "123");
+        m.put("test", "123");
         MVMap<Integer, Integer> map = s.openMap("test");
         map.put(10, 100);
+        // ensure the file header is overwritten
+        s.commit();
+        map.put(10, 110);
+        s.commit();
+        map.put(1, 120);
+        s.setRetentionTime(0);
+        s.commit();
         s.close();
         s = openStore(fileName);
-        assertEquals("123", s.getStoreHeader().get("test"));
+        assertEquals("123", s.getStoreHeader().get("test").toString());
         s.close();
     }