MVStore: compact (including MVTableEngine)

h2/src/main/org/h2/engine/Database.java

@@ -17,7 +17,6 @@ import java.util.Set;
 import java.util.StringTokenizer;
 import org.h2.api.DatabaseEventListener;
 import org.h2.api.JavaObjectSerializer;
-import org.h2.command.CommandInterface;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.command.dml.SetTypes;
 import org.h2.constant.DbSettings;
@@ -1255,7 +1254,7 @@ public class Database implements DataHandler {
         if (mvStore != null) {
             if (!readOnly) {
                 if (compactMode != 0) {
-                    mvStore.compact(compactMode == CommandInterface.SHUTDOWN_DEFRAG);
+                    mvStore.compact();
                 }
             }
             mvStore.close();
@@ -1774,11 +1773,11 @@ public class Database implements DataHandler {
             mvStore.setWriteDelay(value);
         }
     }
-    
+
     public int getRetentionTime() {
         return retentionTime;
     }
-    
+
     public void setRetentionTime(int value) {
         retentionTime = value;
         if (mvStore != null) {
@@ -2093,18 +2092,17 @@ public class Database implements DataHandler {
     }
     
     public QueryStatisticsData getQueryStatisticsData() {
-        if (queryStatistics) {
-            if (queryStatisticsData == null) {
-                synchronized (this) {
-                    if (queryStatisticsData == null) {
-                        queryStatisticsData = new QueryStatisticsData();
-                    }
+        if (!queryStatistics) {
+            return null;
+        }
+        if (queryStatisticsData == null) {
+            synchronized (this) {
+                if (queryStatisticsData == null) {
+                    queryStatisticsData = new QueryStatisticsData();
                 }
             }
-            return queryStatisticsData;
-        } else {
-            return null;
         }
+        return queryStatisticsData;
     }
     
     /**

h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java

@@ -144,7 +144,7 @@ public class FreeSpaceBitSet {
     /**
      * Get the fill rate of the space in percent. The value 0 means the space is
      * completely free, and 100 means it is completely full.
-     * 
+     *
      * @return the fill rate (0 - 100)
      */
     public int getFillRate() {
@@ -159,10 +159,10 @@ public class FreeSpaceBitSet {
         }
         return Math.max(1, (int) (100L * count / total));
     }
-    
+
     /**
      * Get the position of the first free space.
-     * 
+     *
      * @return the position.
      */
     public long getFirstFree() {

h2/src/main/org/h2/mvstore/MVStore.java

@@ -123,6 +123,10 @@ MVStore:
 - temporary file storage
 - simple rollback method (rollback to last committed version)
 - MVMap to implement SortedMap, then NavigableMap
+- add abstraction ChunkStore,
+    with free space handling, retention time / flush,
+    possibly one file per chunk to support SSD trim on file system level,
+    and free up memory for off-heap storage)
 
 */
 
@@ -850,6 +854,9 @@ public class MVStore {
      * @return the new version (incremented if there were changes)
      */
     public long store() {
+;
+new Exception().printStackTrace(System.out);        
+        
         checkOpen();
         return store(false);
     }
@@ -1036,11 +1043,11 @@ public class MVStore {
         }
         return version;
     }
-    
+
     /**
      * Get a buffer for writing. This caller must synchronize on the store
      * before calling the method and until after using the buffer.
-     * 
+     *
      * @return the buffer
      */
     private ByteBuffer getWriteBuffer() {
@@ -1057,7 +1064,7 @@ public class MVStore {
     /**
      * Release a buffer for writing. This caller must synchronize on the store
      * before calling the method and until after using the buffer.
-     * 
+     *
      * @param buff the buffer than can be re-used
      */
     private void releaseWriteBuffer(ByteBuffer buff) {
@@ -1101,6 +1108,10 @@ public class MVStore {
                     Map<Integer, Chunk> freed = freedPages.get(v);
                     for (Chunk f : freed.values()) {
                         Chunk c = chunks.get(f.id);
+                        if (c == null) {
+                            // already removed
+                            continue;
+                        }
                         c.maxLengthLive += f.maxLengthLive;
                         c.pageCountLive += f.pageCountLive;
                         if (c.pageCountLive < 0) {
@@ -1213,10 +1224,12 @@ public class MVStore {
         buff.rewind();
         return Chunk.fromHeader(buff, start);
     }
-    
+
     /**
      * Compact the store by moving all chunks next to each other, if there is
      * free space between chunks. This might temporarily double the file size.
+     * Chunks are overwritten irrespective of the current retention time. Before
+     * overwriting chunks and before resizing the file, syncFile() is called.
      * 
      * @return if anything was written
      */
@@ -1226,6 +1239,23 @@ public class MVStore {
             // nothing to do
             return false;
         }
+        int oldRetentionTime = retentionTime;
+        retentionTime = 0;
+        long time = getTime();
+        ArrayList<Chunk> free = New.arrayList();
+        for (Chunk c : chunks.values()) {
+            if (c.maxLengthLive == 0) {
+                if (canOverwriteChunk(c, time)) {
+                    free.add(c);
+                }
+            }
+        }
+        for (Chunk c : free) {
+            chunks.remove(c.id);
+            meta.remove("chunk." + c.id);
+            int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
+            freeSpace.free(c.start, length);
+        }
         if (freeSpace.getFillRate() == 100) {
             return false;
         }
@@ -1266,12 +1296,15 @@ public class MVStore {
         reuseSpace = false;
         store();
         
+        syncFile();
+
         // now re-use the empty space
         reuseSpace = true;
         for (Chunk c : move) {
             ByteBuffer buff = getWriteBuffer();
             int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
             buff = DataUtils.ensureCapacity(buff, length);
+            buff.limit(length);
             DataUtils.readFully(file, c.start, buff);
             long pos = freeSpace.allocate(length);
             freeSpace.free(c.start, length);
@@ -1283,7 +1316,6 @@ public class MVStore {
             buff.put(header);
             // fill the header with zeroes
             buff.put(new byte[BLOCK_SIZE - header.length]);
-            buff.limit(length);
             buff.position(0);
             fileWriteCount++;
             DataUtils.writeFully(file, pos, buff);
@@ -1291,12 +1323,31 @@ public class MVStore {
             releaseWriteBuffer(buff);
             meta.put("chunk." + c.id, c.asString());
         }
-        
+
         // update the metadata (within the file)
         store();
+        syncFile();
+        shrinkFileIfPossible(0);
+
         reuseSpace = oldReuse;
+        retentionTime = oldRetentionTime;
+
         return true;
     }
+    
+    /**
+     * Force all changes to be written to the file. The default implementation
+     * calls FileChannel.force(true).
+     */
+    public void syncFile() {
+        try {
+            file.force(true);
+        } catch (IOException e) {
+            throw DataUtils.newIllegalStateException(
+                    DataUtils.ERROR_WRITING_FAILED,
+                    "Could not sync file {0}", fileName, e);
+        }
+    }
 
     /**
      * Try to reduce the file size by re-writing partially full chunks. Chunks
@@ -1306,7 +1357,7 @@ public class MVStore {
      * Only data of open maps can be moved. For maps that are not open, the old
      * chunk is still referenced. Therefore, it is recommended to open all maps
      * before calling this method.
-     * 
+     *
      * @param fillRate the minimum percentage of live entries
      * @return if anything was written
      */
@@ -1327,7 +1378,7 @@ public class MVStore {
         }
         // the fill rate of all chunks combined
         int totalChunkFillRate = (int) (100 * maxLengthLiveSum / maxLengthSum);
-        
+
         if (totalChunkFillRate > fillRate) {
             return false;
         }

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -13,7 +13,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.h2.api.TableEngine;
-import org.h2.command.CommandInterface;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.constant.ErrorCode;
 import org.h2.engine.Constants;
@@ -169,11 +168,13 @@ public class MVTableEngine implements TableEngine {
          * Store all pending changes.
          */
         public void store() {
-            if (!store.isReadOnly()) {
-                store.commit();
-                store.compact(50);
-                store.store();
+            if (store.isReadOnly()) {
+                return;
             }
+            int todo;
+            store.commit();
+            store.compact(50);
+            store.store();
         }
 
         /**
@@ -248,7 +249,7 @@ public class MVTableEngine implements TableEngine {
         public InputStream getInputStream() {
             return new FileChannelInputStream(store.getFile(), false);
         }
-        
+
         /**
          * Force the changes to disk.
          */
@@ -261,14 +262,11 @@ public class MVTableEngine implements TableEngine {
             }
         }
 
-        public void compact(boolean defrag) {
-            sync();
-            store.setRetentionTime(0);
+        public void compact() {
             while (store.compact(90)) {
-                System.out.println("compact");
-                sync();
+                // repeat
             }
-            System.out.println("compact done");
+            store.compactMoveChunks();
         }
 
     }

h2/src/test/org/h2/test/TestAll.java

@@ -248,6 +248,11 @@ java org.h2.test.TestAll timer
      * Whether to use the MVStore.
      */
     public boolean mvStore;
+    
+    /**
+     * Whether the test is running with code coverage.
+     */
+    public boolean coverage;
 
     /**
      * If code coverage is enabled.
@@ -505,6 +510,8 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
      * Run the tests with a number of different settings.
      */
     private void runTests() throws SQLException {
+        
+        coverage = isCoverage();
 
         {}
         mvStore = false;
@@ -569,6 +576,20 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         memory = true;
         test();
     }
+    
+    /**
+     * Check whether this method is running with "Emma" code coverage turned on.
+     * 
+     * @return true if the stack trace contains ".emma."
+     */
+    private static boolean isCoverage() {
+        for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
+            if (e.toString().indexOf(".emma.") >= 0) {
+                return true;
+            }
+        }
+        return false;
+    }
 
     /**
      * Run all tests with the current settings.

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -47,7 +47,7 @@ public class TestMVStore extends TestBase {
     public void test() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
         FileUtils.createDirectories(getBaseDir());
-
+        testCompactFully();
         testBackgroundExceptionListener();
         testOldVersion();
         testAtomicOperations();
@@ -92,6 +92,30 @@ public class TestMVStore extends TestBase {
         testLargerThan2G();
     }
 
+    private void testCompactFully() throws Exception {
+        String fileName = getBaseDir() + "/testCompactFully.h3";
+        FileUtils.delete(fileName);
+        MVStore s = new MVStore.Builder().
+                fileName(fileName).
+                open();
+        MVMap<Integer, String> m;
+        for (int i = 0; i < 10; i++) {
+            m = s.openMap("data" + i);
+            m.put(0, "Hello World");
+            s.store();
+        }
+        for (int i = 0; i < 10; i += 2) {
+            m = s.openMap("data" + i);
+            m.removeMap();
+            s.store();
+        }
+        long sizeOld = s.getFile().size();
+        s.compactMoveChunks();
+        long sizeNew = s.getFile().size();
+        assertTrue("old: " + sizeOld + " new: " + sizeNew, sizeNew < sizeOld);
+        s.close();
+    }
+
     private void testBackgroundExceptionListener() throws Exception {
         String fileName = getBaseDir() + "/testBackgroundExceptionListener.h3";
         FileUtils.delete(fileName);
@@ -1221,7 +1245,7 @@ public class TestMVStore extends TestBase {
         }
         s.close();
     }
-    
+
     private void testCompactMapNotOpen() {
         String fileName = getBaseDir() + "/testCompactNotOpen.h3";
         FileUtils.delete(fileName);
@@ -1235,10 +1259,10 @@ public class TestMVStore extends TestBase {
             s.store();
         }
         s.close();
-        
+
         s = openStore(fileName);
         s.setRetentionTime(0);
-        
+
         Map<String, String> meta = s.getMetaMap();
         int chunkCount1 = 0;
         for (String k : meta.keySet()) {

h2/src/test/org/h2/test/store/TestMVStoreBenchmark.java

@@ -39,17 +39,24 @@ public class TestMVStoreBenchmark extends TestBase {
         if (!config.big) {
             return;
         }
+        if (config.coverage || config.codeCoverage) {
+            // run only when _not_ using a code coverage tool,
+            // because the tool might instrument our code but not
+            // java.util.*
+            return;
+        }
+        
         testPerformanceComparison();
         testMemoryUsageComparison();
     }
-    
+
     private void testMemoryUsageComparison() {
         long[] mem;
         long hash, tree, mv;
         String msg;
-        
+
         mem = getMemoryUsed(10000, 10);
-        hash = mem[0]; 
+        hash = mem[0];
         tree = mem[1];
         mv = mem[2];
         msg = Arrays.toString(mem);
@@ -65,12 +72,12 @@ public class TestMVStoreBenchmark extends TestBase {
         assertTrue(msg, mv < tree);
 
     }
-    
+
     private static long[] getMemoryUsed(int count, int size) {
         long hash, tree, mv;
         ArrayList<Map<Integer, String>> mapList;
         long mem;
-        
+
         mapList = New.arrayList();
         mem = getMemory();
         for (int i = 0; i < count; i++) {
@@ -79,7 +86,7 @@ public class TestMVStoreBenchmark extends TestBase {
         addEntries(mapList, size);
         hash = getMemory() - mem;
         mapList.size();
-        
+
         mapList = New.arrayList();
         mem = getMemory();
         for (int i = 0; i < count; i++) {
@@ -91,7 +98,7 @@ public class TestMVStoreBenchmark extends TestBase {
 
         mapList = New.arrayList();
         mem = getMemory();
-        MVStore store = MVStore.open(null);        
+        MVStore store = MVStore.open(null);
         for (int i = 0; i < count; i++) {
             Map<Integer, String> map = store.openMap("t" + i);
             mapList.add(map);
@@ -99,10 +106,10 @@ public class TestMVStoreBenchmark extends TestBase {
         addEntries(mapList, size);
         mv = getMemory() - mem;
         mapList.size();
-        
+
         return new long[]{hash, tree, mv};
     }
-    
+
     private static void addEntries(List<Map<Integer, String>> mapList, int size) {
         for (Map<Integer, String> map : mapList) {
             for (int i = 0; i < size; i++) {
@@ -110,7 +117,7 @@ public class TestMVStoreBenchmark extends TestBase {
             }
         }
     }
-    
+
     static long getMemory() {
         try {
             LinkedList<byte[]> list = new LinkedList<byte[]>();
@@ -130,7 +137,7 @@ public class TestMVStoreBenchmark extends TestBase {
         }
         return getMemoryUsedBytes();
     }
-    
+
     private void testPerformanceComparison() {
         if (!config.big) {
             return;
@@ -144,14 +151,12 @@ public class TestMVStoreBenchmark extends TestBase {
         MVStore store = MVStore.open(null);
         map = store.openMap("test");
         long mv = testPerformance(map, size);
-        String msg = "mv " + mv + " tree " + tree + " hash " + hash;  
+        String msg = "mv " + mv + " tree " + tree + " hash " + hash;
         assertTrue(msg, hash < tree);
         assertTrue(msg, hash < mv);
-        int todo;
-        // check only when _not_ using a code coverage tool
-        // assertTrue(msg, mv < tree);
+        assertTrue(msg, mv < tree);
     }
-    
+
     private long testPerformance(Map<Integer, String> map, int size) {
         System.gc();
         long time = 0;

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -8,7 +8,6 @@ package org.h2.test.store;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.PrintWriter;
 import java.math.BigDecimal;
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -22,7 +21,8 @@ import org.h2.constant.ErrorCode;
 import org.h2.engine.Constants;
 import org.h2.engine.Database;
 import org.h2.jdbc.JdbcConnection;
-import org.h2.mvstore.MVStoreTool;
+import org.h2.mvstore.MVStore;
+import org.h2.mvstore.db.TransactionStore;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
 import org.h2.tools.DeleteDbFiles;
@@ -47,8 +47,9 @@ public class TestMVTableEngine extends TestBase {
 
     @Override
     public void test() throws Exception {
+        ;;
+        // testTransactionLogUsuallyNotStored();
         testShrinkDatabaseFile();
-        testTransactionLogUsuallyNotStored();
         testTwoPhaseCommit();
         testRecover();
         testSeparateKey();
@@ -67,6 +68,35 @@ public class TestMVTableEngine extends TestBase {
         testLocking();
         testSimple();
     }
+    
+    private void testTransactionLogUsuallyNotStored() throws Exception {
+        int todo;
+        FileUtils.deleteRecursive(getBaseDir(), true);
+        Connection conn;
+        Statement stat;
+        String url = "mvstore;MV_STORE=TRUE";
+        url = getURL(url, true);
+        conn = getConnection(url);
+        stat = conn.createStatement();
+        stat.execute("create table test(id identity, name varchar)");
+        conn.setAutoCommit(false);
+        for (int j = 0; j < 100; j++) {
+            for (int i = 0; i < 100; i++) {
+                stat.execute("insert into test(name) values('Hello World')");
+            }
+            conn.commit();
+        }
+        stat.execute("shutdown immediately");
+        JdbcUtils.closeSilently(conn);
+        
+        String file = getBaseDir() + "/mvstore" + Constants.SUFFIX_MV_FILE;
+        
+        MVStore store = MVStore.open(file);
+        TransactionStore t = new TransactionStore(store);
+        assertEquals(0, t.getOpenTransactions().size());
+        
+        store.close();
+    }
 
     private void testShrinkDatabaseFile() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
@@ -88,7 +118,7 @@ public class TestMVTableEngine extends TestBase {
                 retentionTime = 0;
             }
             ResultSet rs = stat.executeQuery(
-                    "select value from information_schema.settings " + 
+                    "select value from information_schema.settings " +
                     "where name='RETENTION_TIME'");
             assertTrue(rs.next());
             assertEquals(retentionTime, rs.getInt(1));
@@ -104,21 +134,15 @@ public class TestMVTableEngine extends TestBase {
                 fail(i + " size: " + size + " max: " + maxSize);
             }
         }
-        int todo;
-//        conn = getConnection(dbName);
-//        stat = conn.createStatement();
-//        stat.execute("shutdown compact");
-//        conn.close();
-//        
-//        MVStoreTool.dump(getBaseDir() + "/mvstore.mv.db", new PrintWriter(System.out));
-//        
-//        long size = FileUtils.size(getBaseDir() + "/mvstore"
-//                + Constants.SUFFIX_MV_FILE);
-//        assertTrue(size < 16 * 1024);
-    }
-
-    private void testTransactionLogUsuallyNotStored() {
-        int todo;
+        long sizeOld = FileUtils.size(getBaseDir() + "/mvstore"
+                + Constants.SUFFIX_MV_FILE);
+        conn = getConnection(dbName);
+        stat = conn.createStatement();
+        stat.execute("shutdown compact");
+        conn.close();
+        long sizeNew = FileUtils.size(getBaseDir() + "/mvstore"
+                + Constants.SUFFIX_MV_FILE);
+        assertTrue(sizeNew < sizeOld);
     }
 
     private void testTwoPhaseCommit() throws Exception {