MVStore: new utility to compress a store. (Work in progress.)

h2/src/main/org/h2/mvstore/MVMap.java

@@ -1029,6 +1029,12 @@ public class MVMap<K, V> extends AbstractMap<K, V>
             throw DataUtils.newConcurrentModificationException(
                     getName());
         }
+        ;
+        // TODO work in progress
+        if(oldRoots.size() - list.size() > 1) {
+        //    System.out.println("reduced! from " + oldRoots.size() + " to " + list.size() +"  " + getClass());
+        }
+        
         oldRoots = list;
     }
 
@@ -1292,21 +1298,23 @@ public class MVMap<K, V> extends AbstractMap<K, V>
         this.writeVersion = writeVersion;
     }
     
-    public void copyFrom(MVMap<K, V> sourceMap) {
+    void copyFrom(MVMap<K, V> sourceMap) {
         ; // TODO work in progress
-        Page sourceRoot = sourceMap.root;
-        root = Page.create(this, writeVersion, sourceRoot);
-        root = copy(root, sourceRoot);
+        root = copy(sourceMap.root, null);
     }
     
-    private Page copy(Page target, Page source) {
-        target = copyOnWrite(target, writeVersion);
+    private Page copy(Page source, CursorPos parent) {
+        Page target = Page.create(this, writeVersion, source);
+        for (CursorPos p = parent; p != null; p = p.parent) {
+            p.page.setChild(p.index, target);
+        }
         if (!target.isLeaf()) {
+            CursorPos pos = new CursorPos(target, 0, parent);            
+            target = copyOnWrite(target, writeVersion);
             for (int i = 0; i < target.getChildPageCount(); i++) {
                 Page sourceChild = source.getChildPage(i);
-                Page targetChild = Page.create(this, writeVersion, sourceChild);
-                targetChild = copy(targetChild, sourceChild);
-                target.setChild(i, targetChild);
+                pos.index = i;
+                copy(sourceChild, pos);
             }
         }
         return target;

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -15,6 +15,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.h2.mvstore.type.DataType;
 import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
@@ -79,7 +80,8 @@ public class MVStoreTool {
             pw.println("File not found: " + fileName);
             return;
         }
-        pw.printf("File %s, length %d\n", fileName, FileUtils.size(fileName));
+        long size = FileUtils.size(fileName);
+        pw.printf("File %s, %d bytes, %d MB\n", fileName, size, size / 1024 / 1024);
         FileChannel file = null;
         int blockSize = MVStore.BLOCK_SIZE;
         try {
@@ -214,6 +216,7 @@ public class MVStoreTool {
                                 Chunk.FOOTER_LENGTH, DataUtils.LATIN).trim());
             }
             pw.printf("%n%0" + len + "x eof%n", fileSize);
+            pw.printf("\n");
         } catch (IOException e) {
             pw.println("ERROR: " + e);
             e.printStackTrace(pw);
@@ -278,6 +281,7 @@ public class MVStoreTool {
                         c.len
                         );
             }
+            pw.printf("\n");
         } catch (Exception e) {
             pw.println("ERROR: " + e);
             e.printStackTrace(pw);
@@ -292,4 +296,109 @@ public class MVStoreTool {
         return x.substring(0, 19);
     }
 
+    /**
+     * Compress the store by creating a new file and copying the live pages there.
+     * 
+     * @param fileName the file name
+     */
+    public static void compress(String fileName) {
+        compress(fileName, fileName + ".new");
+        FileUtils.moveTo(fileName, fileName + ".old");
+        FileUtils.moveTo(fileName, fileName);
+        FileUtils.delete(fileName + ".old");
+    }
+    
+    /**
+     * Copy all live pages from the source store to the target store.
+     * 
+     * @param sourceFileName the name of the source store
+     * @param targetFileName the name of the target store
+     */
+    public static void compress(String sourceFileName, String targetFileName) {
+        MVStore source = new MVStore.Builder().
+                fileName(sourceFileName).readOnly().open();
+        FileUtils.delete(targetFileName);
+        MVStore target = new MVStore.Builder().
+                fileName(targetFileName).open();
+        MVMap<String, String> sourceMeta = source.getMetaMap();
+        MVMap<String, String> targetMeta = target.getMetaMap();
+        for (Entry<String, String> m : sourceMeta.entrySet()) {
+            String key = m.getKey();
+            if (key.startsWith("chunk.")) {
+                // ignore
+            } else if (key.startsWith("map.")) {
+                // ignore
+            } else if (key.startsWith("name.")) {
+                // ignore
+            } else if (key.startsWith("root.")) {
+                // ignore
+            } else {
+                targetMeta.put(key, m.getValue());
+            }
+        }
+        for (String mapName : source.getMapNames()) {
+            MVMap.Builder<Object, Object> mp = 
+                    new MVMap.Builder<Object, Object>().
+                    keyType(new GenericDataType()).
+                    valueType(new GenericDataType());
+            MVMap<Object, Object> sourceMap = source.openMap(mapName, mp);
+            MVMap<Object, Object> targetMap = target.openMap(mapName, mp);
+            targetMap.copyFrom(sourceMap);
+            target.commit();
+        }
+        target.close();
+        source.close();
+    }
+
+    /**
+     * A data type that can read any data that is persisted, and converts it to
+     * a byte array.
+     */
+    static class GenericDataType implements DataType {
+
+        @Override
+        public int compare(Object a, Object b) {
+            throw DataUtils.newUnsupportedOperationException("Can not compare");
+        }
+
+        @Override
+        public int getMemory(Object obj) {
+            return obj == null ? 0 : ((byte[]) obj).length;
+        }
+
+        @Override
+        public void write(WriteBuffer buff, Object obj) {
+            if (obj != null) {
+                buff.put((byte[]) obj);
+            }
+        }
+
+        @Override
+        public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+            for (Object o : obj) {
+                write(buff, o);
+            }
+        }
+
+        @Override
+        public Object read(ByteBuffer buff) {
+            int len = buff.remaining();
+            if (len == 0) {
+                return null;
+            }
+            byte[] data = new byte[len];
+            buff.get(data);
+            return data;
+        }
+
+        @Override
+        public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+            for (int i = 0; i < obj.length; i++) {
+                obj[i] = read(buff);
+            }
+        }
+
+    }
+
+
 }

h2/src/test/org/h2/test/store/TestMVStoreTool.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.store;
+
+import java.util.Map.Entry;
+
+import org.h2.mvstore.MVMap;
+import org.h2.mvstore.MVStore;
+import org.h2.mvstore.MVStoreTool;
+import org.h2.store.fs.FileUtils;
+import org.h2.test.TestBase;
+
+/**
+ * Tests the MVStoreTool class.
+ */
+public class TestMVStoreTool extends TestBase {
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase test = TestBase.createCaller().init();
+        test.config.traceTest = true;
+        test.config.big = true;
+        test.test();
+    }
+
+    @Override
+    public void test() throws Exception {
+        testCompress();
+    }
+    
+    private void testCompress() {
+        String fileName = getBaseDir() + "/testCompress.h3";
+        FileUtils.delete(fileName);
+        // store with a very small page size, to make sure
+        // there are many leaf pages
+        MVStore s = new MVStore.Builder().
+                pageSplitSize(100).
+                fileName(fileName).autoCommitDisabled().open();
+        MVMap<Integer, Integer> map = s.openMap("data");
+        for (int i = 0; i < 10; i++) {
+            map.put(i, i * 10);
+            if (i % 3 == 0) {
+                s.commit();
+            }
+        }
+        s.close();
+        MVStoreTool.dump(fileName);
+        MVStoreTool.dump(fileName + ".new");
+        MVStoreTool.compress(fileName, fileName + ".new");
+        MVStore s1 = new MVStore.Builder().
+                fileName(fileName).readOnly().open();
+        MVStore s2 = new MVStore.Builder().
+                fileName(fileName + ".new").readOnly().open();
+        assertEquals(s1, s2);
+    }
+    
+    private void assertEquals(MVStore a, MVStore b) {
+        assertEquals(a.getMapNames().size(), b.getMapNames().size());
+        for (String mapName : a.getMapNames()) {
+            MVMap<?, ?> ma = a.openMap(mapName);
+            MVMap<?, ?> mb = a.openMap(mapName);
+            assertEquals(ma.sizeAsLong(), mb.sizeAsLong());
+            for (Entry<?, ?> e : ma.entrySet()) {
+                Object x = mb.get(e.getKey());
+                assertEquals(e.getValue().toString(), x.toString());
+            }
+        }
+    }
+    
+}