MVTableEngine: compact the database file; don't store the transaction log usually

h2/src/main/org/h2/engine/Database.java

@@ -607,7 +607,11 @@ public class Database implements DataHandler {
                 getPageStore();
             }
             starting = false;
-            writer = WriterThread.create(this, writeDelay);
+            if (mvStore == null) {
+                writer = WriterThread.create(this, writeDelay);
+            } else {
+                setWriteDelay(writeDelay);
+            }
         } else {
             if (autoServerMode) {
                 throw DbException.getUnsupportedException("autoServerMode && inMemory");
@@ -1252,12 +1256,11 @@ public class Database implements DataHandler {
         }
         reconnectModified(false);
         if (mvStore != null) {
-            if (!readOnly) {
-                if (compactMode != 0) {
-                    mvStore.compact();
-                }
+            if (!readOnly && compactMode != 0) {
+                mvStore.compactFile(dbSettings.maxCompactTime);
+            } else {
+                mvStore.close(dbSettings.maxCompactTime);
             }
-            mvStore.close();
         }
         closeFiles();
         if (persistent && lock == null && fileLockMethod != FileLock.LOCK_NO && fileLockMethod != FileLock.LOCK_FS) {
@@ -1770,7 +1773,8 @@ public class Database implements DataHandler {
             flushOnEachCommit = writeDelay < Constants.MIN_WRITE_DELAY;
         }
         if (mvStore != null) {
-            mvStore.setWriteDelay(value);
+            int millis = value < 0 ? 0 : value;
+            mvStore.getStore().setWriteDelay(millis);
         }
     }
 

h2/src/main/org/h2/mvstore/MVMap.java

@@ -988,6 +988,12 @@ public class MVMap<K, V> extends AbstractMap<K, V>
         return size > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) size;
     }
 
+    @Override
+    public boolean isEmpty() {
+        checkOpen();
+        return 0 == (root.isLeaf() ? root.getKeyCount() : root.getChildPageCount());
+    }
+
     public long getSize() {
         checkOpen();
         return root.getTotalCount();

h2/src/main/org/h2/mvstore/MVStore.java

@@ -225,7 +225,7 @@ public class MVStore {
     private int fileReadCount;
     private int fileWriteCount;
     private int unsavedPageCount;
-    private int maxUnsavedPages;
+    private int unsavedPageCountMax;
 
     /**
      * The time the store was created, in milliseconds since 1970.
@@ -255,7 +255,7 @@ public class MVStore {
     /**
      * The delay in milliseconds to automatically store changes.
      */
-    private int writeDelay = 1000;
+    private int writeDelay;
 
     MVStore(HashMap<String, Object> config) {
         String f = (String) config.get("fileName");
@@ -285,9 +285,8 @@ public class MVStore {
             o = config.get("writeBufferSize");
             mb = o == null ? 4 : (Integer) o;
             int writeBufferSize =  mb * 1024 * 1024;
-            maxUnsavedPages = writeBufferSize / pageSplitSize;
-            o = config.get("writeDelay");
-            writeDelay = o == null ? 1000 : (Integer) o;
+            int div = pageSplitSize;
+            unsavedPageCountMax = writeBufferSize / (div == 0 ? 1 : div);
         } else {
             cache = null;
             filePassword = null;
@@ -501,7 +500,10 @@ public class MVStore {
             rollbackTo(rollback);
         }
         this.lastCommittedVersion = currentVersion;
-        setWriteDelay(writeDelay);
+        
+        // setWriteDelay starts the thread, but only if
+        // the parameter is different than the current value
+        setWriteDelay(1000);
     }
 
     /**
@@ -854,9 +856,6 @@ public class MVStore {
      * @return the new version (incremented if there were changes)
      */
     public long store() {
-;
-new Exception().printStackTrace(System.out);        
-        
         checkOpen();
         return store(false);
     }
@@ -1704,6 +1703,17 @@ new Exception().printStackTrace(System.out);
         return unsavedPageCount;
     }
 
+    /**
+     * Get the maximum number of unsaved pages. If this number is exceeded,
+     * the unsaved changes are stored to disk, including uncommitted changes.
+     * Saved uncommitted changes are rolled back when opening the store.
+     *
+     * @return the number of maximum unsaved pages
+     */
+    public int getUnsavedPageCountMax() {
+        return unsavedPageCountMax;
+    }
+
     /**
      * Increment the number of unsaved pages.
      */
@@ -1719,7 +1729,7 @@ new Exception().printStackTrace(System.out);
             // store is possibly called within store, if the meta map changed
             return;
         }
-        if (unsavedPageCount > maxUnsavedPages && maxUnsavedPages > 0) {
+        if (unsavedPageCount > unsavedPageCountMax && unsavedPageCountMax > 0) {
             store(true);
         }
     }
@@ -2041,13 +2051,34 @@ new Exception().printStackTrace(System.out);
             // ignore
         }
     }
-
-    public void setWriteDelay(int value) {
-        writeDelay = value;
+    
+    /**
+     * Set the maximum delay in milliseconds to store committed changes (for
+     * file-based stores).
+     * <p>
+     * The default is 1000, meaning committed changes are stored after at
+     * most one second.
+     * <p>
+     * When the value is set to -1, committed changes are only written when
+     * calling the store method. When the value is set to 0, committed
+     * changes are immediately written on a commit, but please note this
+     * decreases performance and does still not guarantee the disk will
+     * actually write the data.
+     *
+     * @param millis the maximum delay
+     */
+    public void setWriteDelay(int millis) {
+        if (writeDelay == millis) {
+            return;
+        }
+        writeDelay = millis;
+        if (file == null) {
+            return;
+        }
         stopBackgroundThread();
         // start the background thread if needed
-        if (value > 0) {
-            int sleep = Math.max(1, value / 10);
+        if (millis > 0) {
+            int sleep = Math.max(1, millis / 10);
             Writer w = new Writer(this, sleep);
             Thread t = new Thread(w, "MVStore writer " + fileName);
             t.setDaemon(true);
@@ -2186,26 +2217,6 @@ new Exception().printStackTrace(System.out);
             return set("writeBufferSize", mb);
         }
 
-        /**
-         * Set the maximum delay in milliseconds to store committed changes (for
-         * file-based stores).
-         * <p>
-         * The default is 1000, meaning committed changes are stored after at
-         * most one second.
-         * <p>
-         * When the value is set to -1, committed changes are only written when
-         * calling the store method. When the value is set to 0, committed
-         * changes are immediately written on a commit, but please note this
-         * decreases performance and does still not guarantee the disk will
-         * actually write the data.
-         *
-         * @param millis the maximum delay
-         * @return this
-         */
-        public Builder writeDelay(int millis) {
-            return set("writeDelay", millis);
-        }
-
         /**
          * Set the amount of memory a page should contain at most, in bytes,
          * before it is split. The default is 6 KB. This is not a limit in the

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -171,7 +171,6 @@ public class MVTableEngine implements TableEngine {
             if (store.isReadOnly()) {
                 return;
             }
-            int todo;
             store.commit();
             store.compact(50);
             store.store();
@@ -187,22 +186,6 @@ public class MVTableEngine implements TableEngine {
             store.closeImmediately();
         }
 
-        /**
-         * Close the store. Pending changes are persisted.
-         */
-        public void close() {
-            if (!store.isClosed()) {
-                if (!store.isReadOnly()) {
-                    store.store();
-                }
-                store.close();
-            }
-        }
-
-        public void setWriteDelay(int value) {
-            store.setWriteDelay(value);
-        }
-
         /**
          * Commit all transactions that are in the committing state, and
          * rollback all open transactions.
@@ -262,12 +245,49 @@ public class MVTableEngine implements TableEngine {
             }
         }
 
-        public void compact() {
+        /**
+         * Compact the database file, that is, compact blocks that have a low
+         * fill rate, and move chunks next to each other. This will typically
+         * shrink the database file. Changes are flushed to the file, and old
+         * chunks are overwritten.
+         * 
+         * @param maxCompactTime the maximum time in milliseconds to compact
+         */
+        public void compactFile(long maxCompactTime) {
+            store.setRetentionTime(0);
+            long start = System.currentTimeMillis();
             while (store.compact(90)) {
-                // repeat
+                store.syncFile();
+                long time = System.currentTimeMillis() - start;
+                if (time > maxCompactTime) {
+                    break;
+                }
             }
             store.compactMoveChunks();
         }
+        
+        /**
+         * Close the store. Pending changes are persisted. Chunks with a low
+         * fill rate are compacted, but old chunks are kept for some time, so
+         * most likely the database file will not shrink.
+         * 
+         * @param maxCompactTime the maximum time in milliseconds to compact
+         */
+        public void close(long maxCompactTime) {
+            if (!store.isClosed()) {
+                if (!store.isReadOnly()) {
+                    store.store();
+                    long start = System.currentTimeMillis();
+                    while (store.compact(90)) {
+                        long time = System.currentTimeMillis() - start;
+                        if (time > maxCompactTime) {
+                            break;
+                        }
+                    }
+                }
+                store.close();
+            }
+        }
 
     }
 

h2/src/main/org/h2/mvstore/db/TransactionStore.java

@@ -356,6 +356,17 @@ public class TransactionStore {
         if (store.getWriteDelay() == 0) {
             store.commit();
         }
+        // to avoid having to store the transaction log,
+        // if there is no open transaction, 
+        // and if there have been many changes, store them now
+        if (undoLog.isEmpty()) {
+            int unsaved = store.getUnsavedPageCount();
+            int max = store.getUnsavedPageCountMax();
+            // save at 3/4 capacity
+            if (unsaved * 4 > max * 3) {
+                store.store();
+            }
+        }
     }
 
     /**

h2/src/main/org/h2/store/WriterThread.java

@@ -92,8 +92,6 @@ public class WriterThread implements Runnable {
                     traceSystem.getTrace(Trace.DATABASE).error(e, "flush");
                 }
             }
-            // TODO log writer: could also flush the dirty cache
-            // when there is low activity
 
             // wait 0 mean wait forever, which is not what we want
             wait = Math.max(wait, Constants.MIN_WRITE_DELAY);

h2/src/test/org/h2/test/jdbc/TestMetaData.java

@@ -1010,7 +1010,8 @@ public class TestMetaData extends TestBase {
         stat.execute("SET QUERY_STATISTICS TRUE");
         stat.execute("select * from test limit 10");
         stat.execute("select * from test limit 10");
-        // The "order by" makes the resultset more stable on windows, where the timer resolution is not that great 
+        // The "order by" makes the result set more stable on windows, where the
+        // timer resolution is not that great
         rs = stat.executeQuery("select * from INFORMATION_SCHEMA.QUERY_STATISTICS ORDER BY EXECUTION_COUNT");
         assertTrue(rs.next());
         assertEquals("select * from test limit 10", rs.getString("SQL_STATEMENT"));

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -123,7 +123,6 @@ public class TestMVStore extends TestBase {
         final AtomicReference<Exception> exRef = new AtomicReference<Exception>();
         s = new MVStore.Builder().
                 fileName(fileName).
-                writeDelay(2).
                 backgroundExceptionListener(new ExceptionListener() {
 
                     @Override
@@ -133,6 +132,7 @@ public class TestMVStore extends TestBase {
 
                 }).
                 open();
+        s.setWriteDelay(2);
         MVMap<Integer, String> m;
         m = s.openMap("data");
         s.getFile().close();
@@ -238,23 +238,25 @@ public class TestMVStore extends TestBase {
         MVMap<Integer, String> m;
 
         FileUtils.delete(fileName);
-        s = new MVStore.Builder().writeDelay(0).
+        s = new MVStore.Builder().
                 fileName(fileName).open();
+        s.setWriteDelay(0);
         m = s.openMap("data");
         m.put(1, "1");
         s.commit();
         s.close();
-        s = new MVStore.Builder().writeDelay(0).
+        s = new MVStore.Builder().
                 fileName(fileName).open();
+        s.setWriteDelay(0);
         m = s.openMap("data");
         assertEquals(1, m.size());
         s.close();
 
         FileUtils.delete(fileName);
         s = new MVStore.Builder().
-                writeDelay(1).
                 fileName(fileName).
                 open();
+        s.setWriteDelay(1);
         m = s.openMap("data");
         m.put(1, "Hello");
         s.store();
@@ -264,9 +266,9 @@ public class TestMVStore extends TestBase {
         // must not store, as nothing has been committed yet
         s.closeImmediately();
         s = new MVStore.Builder().
-                writeDelay(1).
                 fileName(fileName).
                 open();
+        s.setWriteDelay(1);
         m = s.openMap("data");
         assertEquals(null, m.get(2));
         m.put(2, "World");

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -47,8 +47,7 @@ public class TestMVTableEngine extends TestBase {
 
     @Override
     public void test() throws Exception {
-        ;;
-        // testTransactionLogUsuallyNotStored();
+        testTransactionLogUsuallyNotStored();
         testShrinkDatabaseFile();
         testTwoPhaseCommit();
         testRecover();
@@ -70,7 +69,6 @@ public class TestMVTableEngine extends TestBase {
     }
     
     private void testTransactionLogUsuallyNotStored() throws Exception {
-        int todo;
         FileUtils.deleteRecursive(getBaseDir(), true);
         Connection conn;
         Statement stat;
@@ -80,9 +78,11 @@ public class TestMVTableEngine extends TestBase {
         stat = conn.createStatement();
         stat.execute("create table test(id identity, name varchar)");
         conn.setAutoCommit(false);
+        PreparedStatement prep = conn.prepareStatement(
+                "insert into test(name) values(space(10000))");
         for (int j = 0; j < 100; j++) {
             for (int i = 0; i < 100; i++) {
-                stat.execute("insert into test(name) values('Hello World')");
+                prep.execute();
             }
             conn.commit();
         }
@@ -94,7 +94,6 @@ public class TestMVTableEngine extends TestBase {
         MVStore store = MVStore.open(file);
         TransactionStore t = new TransactionStore(store);
         assertEquals(0, t.getOpenTransactions().size());
-        
         store.close();
     }
 
@@ -129,7 +128,7 @@ public class TestMVTableEngine extends TestBase {
             long size = FileUtils.size(getBaseDir() + "/mvstore"
                     + Constants.SUFFIX_MV_FILE);
             if (i < 10) {
-                maxSize = (int) (Math.max(size, maxSize) * 1.1);
+                maxSize = (int) (Math.max(size, maxSize) * 1.2);
             } else if (size > maxSize) {
                 fail(i + " size: " + size + " max: " + maxSize);
             }