MVStore: tool to quickly compact a store.

h2/src/main/org/h2/mvstore/CursorPos.java

@@ -13,7 +13,7 @@ public class CursorPos {
     /**
      * The current page.
      */
-    public final Page page;
+    public Page page;
 
     /**
      * The current index.

h2/src/main/org/h2/mvstore/MVMap.java

@@ -995,7 +995,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
             return;
         }
         Page last = oldRoots.peekLast();
-        // TODO why is this?
+        // TODO why is this? (maybe not needed)
         ;
         oldest--;
         while (true) {
@@ -1257,23 +1257,32 @@ public class MVMap<K, V> extends AbstractMap<K, V>
      * @param sourceMap the source map
      */
     void copyFrom(MVMap<K, V> sourceMap) {
-        ; // TODO work in progress
-        root = copy(sourceMap.root, null);
+        beforeWrite();
+        newRoot(copy(sourceMap.root, null));
+        afterWrite();
     }
 
     private Page copy(Page source, CursorPos parent) {
         Page target = Page.create(this, writeVersion, source);
-        for (CursorPos p = parent; p != null; p = p.parent) {
-            p.page.setChild(p.index, target);
-        }
-        if (!target.isLeaf()) {
+        if (target.isLeaf()) {
+            Page child = target;
+            for (CursorPos p = parent; p != null; p = p.parent) {
+                p.page.setChild(p.index, child);
+                p.page = copyOnWrite(p.page, writeVersion);
+                child = p.page;
+                if (p.parent == null) {
+                    newRoot(p.page);
+                    afterWrite();
+                    beforeWrite();
+                }
+            }
+        } else {
             CursorPos pos = new CursorPos(target, 0, parent);
-            target = copyOnWrite(target, writeVersion);
             for (int i = 0; i < target.getChildPageCount(); i++) {
-                Page sourceChild = source.getChildPage(i);
                 pos.index = i;
-                copy(sourceChild, pos);
+                copy(source.getChildPage(i), pos);
             }
+            target = pos.page;
         }
         return target;
     }

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -45,6 +45,12 @@ public class MVStoreTool {
             } else if ("-info".equals(args[i])) {
                 String fileName = args[++i];
                 info(fileName, new PrintWriter(System.out));
+            } else if ("-compact".equals(args[i])) {
+                String fileName = args[++i];
+                compact(fileName, false);
+            } else if ("-compress".equals(args[i])) {
+                String fileName = args[++i];
+                compact(fileName, true);
             }
         }
     }
@@ -301,9 +307,10 @@ public class MVStoreTool {
      * there.
      *
      * @param fileName the file name
+     * @param compress whether to compress the data
      */
-    public static void compress(String fileName) {
-        compress(fileName, fileName + ".new");
+    public static void compact(String fileName, boolean compress) {
+        compact(fileName, fileName + ".new", compress);
         FileUtils.moveTo(fileName, fileName + ".old");
         FileUtils.moveTo(fileName, fileName);
         FileUtils.delete(fileName + ".old");
@@ -315,12 +322,18 @@ public class MVStoreTool {
      * @param sourceFileName the name of the source store
      * @param targetFileName the name of the target store
      */
-    public static void compress(String sourceFileName, String targetFileName) {
+    public static void compact(String sourceFileName, String targetFileName, boolean compress) {
         MVStore source = new MVStore.Builder().
-                fileName(sourceFileName).readOnly().open();
+                fileName(sourceFileName).
+                readOnly().
+                open();
         FileUtils.delete(targetFileName);
-        MVStore target = new MVStore.Builder().
-                fileName(targetFileName).open();
+        MVStore.Builder b = new MVStore.Builder().
+                fileName(targetFileName);
+        if (compress) {
+            b.compress();
+        }
+        MVStore target = b.open();
         MVMap<String, String> sourceMeta = source.getMetaMap();
         MVMap<String, String> targetMeta = target.getMetaMap();
         for (Entry<String, String> m : sourceMeta.entrySet()) {
@@ -345,7 +358,6 @@ public class MVStoreTool {
             MVMap<Object, Object> sourceMap = source.openMap(mapName, mp);
             MVMap<Object, Object> targetMap = target.openMap(mapName, mp);
             targetMap.copyFrom(sourceMap);
-            target.commit();
         }
         target.close();
         source.close();
@@ -364,7 +376,7 @@ public class MVStoreTool {
 
         @Override
         public int getMemory(Object obj) {
-            return obj == null ? 0 : ((byte[]) obj).length;
+            return obj == null ? 0 : ((byte[]) obj).length * 8;
         }
 
         @Override

h2/src/main/org/h2/mvstore/WriteBuffer.java

@@ -304,7 +304,11 @@ public class WriteBuffer {
         // grow at least 50% of the current size
         grow = Math.max(temp.capacity() / 2, grow);
         int newCapacity = temp.capacity() + grow;
-        buff = ByteBuffer.allocate(newCapacity);
+        try {
+            buff = ByteBuffer.allocate(newCapacity);
+        } catch (OutOfMemoryError e) {
+            throw new OutOfMemoryError("Capacity: " + newCapacity);
+        }
         temp.flip();
         buff.put(temp);
         if (newCapacity <= MAX_REUSE_CAPACITY) {

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -317,9 +317,9 @@ public class MVTableEngine implements TableEngine {
                 store.compactMoveChunks();
             } else {
                 long start = System.currentTimeMillis();
-                while (store.compact(99, 16 * 1024 * 1024)) {
+                while (store.compact(95, 16 * 1024 * 1024)) {
                     store.sync();
-                    store.compactMoveChunks(16 * 1024 * 1024);
+                    store.compactMoveChunks(95, 16 * 1024 * 1024);
                     long time = System.currentTimeMillis() - start;
                     if (time > maxCompactTime) {
                         break;
@@ -341,8 +341,8 @@ public class MVTableEngine implements TableEngine {
                     if (!store.getFileStore().isReadOnly()) {
                         transactionStore.close();
                         if (maxCompactTime > 0) {
-                            store.compact(99, 1 * 1024 * 1024);
-                            store.compactMoveChunks(1 * 1024 * 1024);
+                            store.compact(95, 1024 * 1024);
+                            store.compactMoveChunks(95, 1024 * 1024);
                         }
                     }
                     store.close();

h2/src/test/org/h2/test/store/TestMVStoreTool.java

@@ -32,12 +32,12 @@ public class TestMVStoreTool extends TestBase {
 
     @Override
     public void test() throws Exception {
-        ; // TODO work in progress
-        // testCompress();
+        testCompress();
     }
 
     private void testCompress() {
         String fileName = getBaseDir() + "/testCompress.h3";
+        FileUtils.createDirectory(getBaseDir());
         FileUtils.delete(fileName);
         // store with a very small page size, to make sure
         // there are many leaf pages
@@ -51,15 +51,26 @@ public class TestMVStoreTool extends TestBase {
                 s.commit();
             }
         }
+        for (int i = 0; i < 10; i++) {
+            map = s.openMap("data" + i);
+            for (int j = 0; j < i * i; j++) {
+                map.put(j, j * 10);
+            }
+            s.commit();
+        }
         s.close();
-//        MVStoreTool.dump(fileName);
-//        MVStoreTool.dump(fileName + ".new");
-        MVStoreTool.compress(fileName, fileName + ".new");
+        MVStoreTool.compact(fileName, fileName + ".new", false);
+        MVStoreTool.compact(fileName, fileName + ".new.compress", true);
         MVStore s1 = new MVStore.Builder().
                 fileName(fileName).readOnly().open();
         MVStore s2 = new MVStore.Builder().
                 fileName(fileName + ".new").readOnly().open();
+        MVStore s3 = new MVStore.Builder().
+                fileName(fileName + ".new.compress").readOnly().open();
         assertEquals(s1, s2);
+        assertEquals(s1, s3);
+        assertTrue(FileUtils.size(fileName + ".new") < FileUtils.size(fileName));
+        assertTrue(FileUtils.size(fileName + ".new.compress") < FileUtils.size(fileName + ".new"));
     }
 
     private void assertEquals(MVStore a, MVStore b) {