MVStore: improved compact operation

h2/src/main/org/h2/engine/Constants.java

@@ -105,7 +105,7 @@ public class Constants {
     /**
      * The minor version of this database.
      */
-;    public static final int VERSION_MINOR = 3;
+    public static final int VERSION_MINOR = 4;
 
     /**
      * The lock mode that means no locking is used at all.

h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java

@@ -14,6 +14,8 @@ import org.h2.util.MathUtils;
  * A free space bit set.
  */
 public class FreeSpaceBitSet {
+    
+    private static final boolean DETAILED_INFO = false;
 
     /**
      * The first usable block.
@@ -171,7 +173,27 @@ public class FreeSpaceBitSet {
 
     @Override
     public String toString() {
-        StringBuilder buff = new StringBuilder("[");
+        StringBuilder buff = new StringBuilder();
+        if (DETAILED_INFO) {
+            int onCount = 0, offCount = 0;
+            int on = 0;
+            for (int i = 0; i < set.length(); i++) {
+                if (set.get(i)) { 
+                    onCount++;
+                    on++;
+                } else {
+                    offCount++;
+                }
+                if ((i & 1023) == 1023) {
+                    buff.append(String.format("%3x", on)).append(' ');
+                    on = 0;
+                }
+            }
+            buff.append("\n");
+            buff.append(" on " + onCount + " off " + offCount);
+            buff.append(" " + 100 * onCount / (onCount+offCount) + "% used ");
+        }
+        buff.append('[');
         for (int i = 0;;) {
             if (i > 0) {
                 buff.append(", ");
@@ -185,7 +207,8 @@ public class FreeSpaceBitSet {
             buff.append(Integer.toHexString(end - 1));
             i = end + 1;
         }
-        return buff.append(']').toString();
+        buff.append(']');
+        return buff.toString();
     }
-
+    
 }
\ No newline at end of file

h2/src/main/org/h2/mvstore/MVMap.java

@@ -783,7 +783,6 @@ public class MVMap<K, V> extends AbstractMap<K, V>
     }
 
     private int rewrite(Page p, Set<Integer> set) {
-        ; // TODO write more tests
         if (p.isLeaf()) {
             long pos = p.getPos();
             if (pos == 0) {

h2/src/main/org/h2/mvstore/MVStore.java

@@ -1320,7 +1320,6 @@ public class MVStore {
             // nothing to do
             return false;
         }
-        ; // TODO write tests
         for (MVMap<?, ?> m : maps.values()) {
             @SuppressWarnings("unchecked")
             MVMap<Object, Object> map = (MVMap<Object, Object>) m;
@@ -1342,22 +1341,79 @@ public class MVStore {
         return true;
     }
     
+    public synchronized boolean compactMoveChunks() {
+        return compactMoveChunks(Long.MAX_VALUE);
+    }
+    
     /**
      * Compact the store by moving all chunks next to each other, if there is
      * free space between chunks. This might temporarily double the file size.
      * Chunks are overwritten irrespective of the current retention time. Before
      * overwriting chunks and before resizing the file, syncFile() is called.
      *
+     * @param moveSize the number of bytes to move
      * @return if anything was written
      */
-    public synchronized boolean compactMoveChunks() {
+    public synchronized boolean compactMoveChunks(long moveSize) {
         checkOpen();
         if (lastChunk == null) {
             // nothing to do
             return false;
         }
         int oldRetentionTime = retentionTime;
-        retentionTime = 0;
+        boolean oldReuse = reuseSpace;
+        try {
+            retentionTime = 0;
+            compactFreeUnusedChunks();
+            if (fileStore.getFillRate() == 100) {
+                return false;
+            }
+            ArrayList<Chunk> move;
+            long start = fileStore.getFirstFree() / BLOCK_SIZE;
+            move = compactGetMoveBlocks(start, moveSize);
+            compactMoveChunks(move);
+        } finally {
+            reuseSpace = oldReuse;
+            retentionTime = oldRetentionTime;
+        }
+        return true;
+    }
+    
+    private ArrayList<Chunk> compactGetMoveBlocks(long startBlock, long moveSize) {
+        ArrayList<Chunk> move = New.arrayList();
+        for (Chunk c : chunks.values()) {
+            if (c.block > startBlock) {
+                move.add(c);
+            }
+        }
+        // sort by block
+        Collections.sort(move, new Comparator<Chunk>() {
+            @Override
+            public int compare(Chunk o1, Chunk o2) {
+                return Long.signum(o1.block - o2.block);
+            }
+        });
+        // find which is the last block to keep
+        int count = 0;
+        long size = 0;
+        for (Chunk c : move) {
+            long chunkSize = c.len * (long) BLOCK_SIZE;
+            if (size + chunkSize > moveSize) {
+                break;
+            }
+            size += chunkSize;
+            count++;
+        }            
+        // move the first block (so the first gap is moved),
+        // and the one at the end (so the file shrinks)
+        while (move.size() > count && move.size() > 1) {
+            move.remove(1);
+        }
+        
+        return move;
+    }
+    
+    private void compactFreeUnusedChunks() {
         long time = getTime();
         ArrayList<Chunk> free = New.arrayList();
         for (Chunk c : chunks.values()) {
@@ -1375,16 +1431,9 @@ public class MVStore {
             int length = c.len * BLOCK_SIZE;
             fileStore.free(start, length);
         }
-        if (fileStore.getFillRate() == 100) {
-            return false;
-        }
-        long firstFree = fileStore.getFirstFree() / BLOCK_SIZE;
-        ArrayList<Chunk> move = New.arrayList();
-        for (Chunk c : chunks.values()) {
-            if (c.block > firstFree) {
-                move.add(c);
-            }
-        }
+    }
+
+    private void compactMoveChunks(ArrayList<Chunk> move) {
         for (Chunk c : move) {
             WriteBuffer buff = getWriteBuffer();
             long start = c.block * BLOCK_SIZE;
@@ -1410,7 +1459,6 @@ public class MVStore {
             markMetaChanged();
             meta.put(Chunk.getMetaKey(c.id), c.asString());
         }
-        boolean oldReuse = reuseSpace;
 
         // update the metadata (store at the end of the file)
         reuseSpace = false;
@@ -1453,11 +1501,6 @@ public class MVStore {
         commitAndSave();
         sync();
         shrinkFileIfPossible(0);
-
-        reuseSpace = oldReuse;
-        retentionTime = oldRetentionTime;
-
-        return true;
     }
 
     /**
@@ -1519,7 +1562,7 @@ public class MVStore {
         for (Chunk c : chunks.values()) {
             if (canOverwriteChunk(c, time)) {
                 long age = last.version - c.version + 1;
-                c.collectPriority = (int) (c.getFillRate() / age);
+                c.collectPriority = (int) (c.getFillRate() * 1000 / age);
                 old.add(c);
             }
         }
@@ -1547,7 +1590,7 @@ public class MVStore {
         for (Chunk c : old) {
             long size = c.maxLen - c.maxLenLive;
             if (move != null) {
-                if (saved > saving) {
+                if (c.collectPriority > 0 && saved > saving) {
                     break;
                 }
             }
@@ -1575,7 +1618,6 @@ public class MVStore {
         for (MVMap<?, ?> m : maps.values()) {
             @SuppressWarnings("unchecked")
             MVMap<Object, Object> map = (MVMap<Object, Object>) m;
-            ; // TODO write more tests
             map.rewrite(set);
         }
         commitAndSave();
@@ -1750,7 +1792,9 @@ public class MVStore {
             }
             buff.position(offset);
             Page page = new Page(map, 0);
+            int limit = buff.limit();
             page.read(buff, chunk.id, buff.position(), length);
+            buff.limit(limit);
             int type = page.isLeaf() ? 0 : 1;
             long pos = DataUtils.getPagePos(chunk.id, offset, pageLength, type);
             page.setPos(pos);

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -16,6 +16,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.h2.mvstore.type.DataType;
 import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
@@ -291,5 +292,55 @@ public class MVStoreTool {
         String x = new Timestamp(t).toString();
         return x.substring(0, 19);
     }
+    
+    /**
+     * A data type that can read any data that is persisted, and converts it to
+     * a byte array.
+     */
+    static class GenericDataType implements DataType {
+
+        @Override
+        public int compare(Object a, Object b) {
+            throw DataUtils.newUnsupportedOperationException("Can not compare");
+        }
+
+        @Override
+        public int getMemory(Object obj) {
+            return obj == null ? 0 : ((byte[]) obj).length;
+        }
+
+        @Override
+        public void write(WriteBuffer buff, Object obj) {
+            if (obj != null) {
+                buff.put((byte[]) obj);
+            }
+        }
 
+        @Override
+        public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
+            for (Object o : obj) {
+                write(buff, o);
+            }
+        }
+
+        @Override
+        public Object read(ByteBuffer buff) {
+            int len = buff.remaining();
+            if (len == 0) {
+                return null;
+            }
+            byte[] data = new byte[len];
+            buff.get(data);
+            return data;
+        }
+
+        @Override
+        public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
+            for (int i = 0; i < obj.length; i++) {
+                obj[i] = read(buff);
+            }
+        }
+        
+    }
+    
 }

h2/src/main/org/h2/mvstore/Page.java

@@ -763,6 +763,7 @@ public class Page {
                     "File corrupted in chunk {0}, expected page length =< {1}, got {2}",
                     chunkId, maxLength, pageLength);
         }
+        buff.limit(start + pageLength);
         short check = buff.getShort();
         int mapId = DataUtils.readVarInt(buff);
         if (mapId != map.getId()) {

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -310,18 +310,21 @@ public class MVTableEngine implements TableEngine {
         public void compactFile(long maxCompactTime) {
             store.setRetentionTime(0);
             if (maxCompactTime == Long.MAX_VALUE) {
+                store.setReuseSpace(false);
                 store.compactRewriteFully();
+                store.setReuseSpace(true);
+                store.compactMoveChunks();
             } else {
                 long start = System.currentTimeMillis();
-                while (store.compact(99, 4 * 1024 * 1024)) {
+                while (store.compact(99, 16 * 1024 * 1024)) {
                     store.sync();
+                    store.compactMoveChunks(16 * 1024 * 1024);
                     long time = System.currentTimeMillis() - start;
                     if (time > maxCompactTime) {
                         break;
                     }
                 }
             }
-            store.compactMoveChunks();
         }
 
         /**
@@ -336,12 +339,9 @@ public class MVTableEngine implements TableEngine {
                 if (!store.isClosed() && store.getFileStore() != null) {
                     if (!store.getFileStore().isReadOnly()) {
                         transactionStore.close();
-                        long start = System.currentTimeMillis();
-                        while (store.compact(90, 4 * 1024 * 1024)) {
-                            long time = System.currentTimeMillis() - start;
-                            if (time > maxCompactTime) {
-                                break;
-                            }
+                        if (maxCompactTime > 0) {
+                            store.compact(99, 1 * 1024 * 1024);
+                            store.compactMoveChunks(1 * 1024 * 1024);
                         }
                     }
                     store.close();