MVStore: the store is now auto-compacted automatically up to some point, to…

MVStore: the store is now auto-compacted automatically up to some point, to avoid very large file sizes. This area is still work in progress.

h2/src/docsrc/html/changelog.html

@@ -17,7 +17,8 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>MVStore: new utility to compress a store.
+<ul><li>MVStore: the store is now auto-compacted automatically up to some point,
+    to avoid very large file sizes. This area is still work in progress.
 </li><li>Sequences of temporary tables (auto-increment or identity columns)
     were persisted unnecessarily in the database file, and were not removed
     when re-opening the database.

h2/src/main/org/h2/mvstore/Chunk.java

@@ -184,8 +184,18 @@ public class Chunk {
         return c;
     }
 
+    /**
+     * Calculate the fill rate in %. 0 means empty, 100 means full.
+     * 
+     * @return the fill rate
+     */
     public int getFillRate() {
-        return (int) (maxLen == 0 ? 0 : 100 * maxLenLive / maxLen);
+        if (maxLenLive == 0) {
+            return 0;
+        } else if (maxLenLive == maxLen) {
+            return 100;
+        }
+        return 1 + (int) (98 * maxLenLive / maxLen);
     }
 
     @Override

h2/src/main/org/h2/mvstore/MVStore.java

@@ -237,6 +237,9 @@ public class MVStore {
      * The delay in milliseconds to automatically commit and write changes.
      */
     private int autoCommitDelay;
+    
+    private int autoCompactFillRate = 10;
+    private int autoCompactSize = 2 * 1024 * 1024;
 
     /**
      * Create and open the store.
@@ -1531,10 +1534,10 @@ public class MVStore {
      * before calling this method.
      *
      * @param targetFillRate the minimum percentage of live entries
-     * @param saving the amount of saved space
+     * @param write the number of bytes to write
      * @return if a chunk was re-written
      */
-    public synchronized boolean compact(int targetFillRate, int saving) {
+    public synchronized boolean compact(int targetFillRate, int write) {
         checkOpen();
         if (lastChunk == null) {
             // nothing to do
@@ -1588,21 +1591,20 @@ public class MVStore {
             }
         });
         // find out up to were in the old list we need to move
-        long saved = 0;
+        long written = 0;
         int chunkCount = 0;
         Chunk move = null;
         for (Chunk c : old) {
-            long size = c.maxLen - c.maxLenLive;
             if (move != null) {
-                if (c.collectPriority > 0 && saved > saving) {
+                if (c.collectPriority > 0 && written > write) {
                     break;
                 }
             }
-            saved += size;
+            written += c.maxLenLive;
             chunkCount++;
             move = c;
         }
-        if (chunkCount <= 1) {
+        if (chunkCount < 1) {
             return false;
         }
         // remove the chunks we want to keep from this list
@@ -2367,14 +2369,23 @@ public class MVStore {
         if (time <= lastCommitTime + autoCommitDelay) {
             return;
         }
-        if (!hasUnsavedChanges()) {
-            return;
+        if (hasUnsavedChanges()) {
+            try {
+                commitAndSave();
+            } catch (Exception e) {
+                if (backgroundExceptionHandler != null) {
+                    backgroundExceptionHandler.uncaughtException(null, e);
+                    return;
+                }
+            }
         }
-        try {
-            commitAndSave();
-        } catch (Exception e) {
-            if (backgroundExceptionHandler != null) {
-                backgroundExceptionHandler.uncaughtException(null, e);
+        if (autoCompactSize > 0) {
+            try {
+                compact(autoCompactFillRate, autoCompactSize);
+            } catch (Exception e) {
+                if (backgroundExceptionHandler != null) {
+                    backgroundExceptionHandler.uncaughtException(null, e);
+                }
             }
         }
     }

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -1299,7 +1299,6 @@ public class TestMVStore extends TestBase {
         }
         s.commit();
         assertTrue(s.compact(100, 50 * 1024));
-        assertTrue(s.compact(100, 1024));
         s.close();
         long len2 = FileUtils.size(fileName);
         assertTrue("len2: " + len2 + " len: " + len, len2 < len);

h2/src/tools/org/h2/dev/util/ThreadDumpCleaner.java

@@ -6,9 +6,14 @@
 package org.h2.dev.util;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.LineNumberReader;
+import java.io.PrintWriter;
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.regex.Pattern;
 
@@ -29,6 +34,10 @@ public class ThreadDumpCleaner {
                 "at java.io.FileInputStream.readBytes(?s).*?\n\n",
         "\".*?\".*?\n   java.lang.Thread.State:.*\n\t" + 
                 "at sun.nio.ch.ServerSocketChannelImpl.accept(?s).*?\n\n",
+        "\".*?\".*?\n   java.lang.Thread.State:.*\n\t" + 
+                "at sun.nio.ch.EPollArrayWrapper.epollWait(?s).*?\n\n",
+        "\".*?\".*?\n   java.lang.Thread.State:.*\n\t" + 
+                "at java.lang.Object.wait(?s).*?\n\n",
         "JNI global references:.*\n\n",
     };
     
@@ -46,25 +55,45 @@ public class ThreadDumpCleaner {
      * @param args the command line arguments
      */
     public static void main(String... args) throws IOException {
-        FileReader r = new FileReader(args[0]);
-        LineNumberReader in = new LineNumberReader(new BufferedReader(r));
-        new ThreadDumpCleaner().run(in);
+        String inFile = null, outFile = null;
+        for (int i = 0; i < args.length; i++) {
+            if (args[i].equals("-in")) {
+                inFile = args[++i];
+            } else if (args[i].equals("-out")) {
+                outFile = args[++i];
+            }
+        }
+        PrintWriter writer;
+        if (outFile != null) {
+            writer = new PrintWriter(new BufferedWriter(new FileWriter(outFile)));
+        } else {
+            writer = new PrintWriter(System.out);
+        }
+        Reader r;
+        if (inFile != null) {
+            r = new FileReader(inFile);
+        } else {
+            r = new InputStreamReader(System.in);
+        }
+        new ThreadDumpCleaner().run(
+                new LineNumberReader(new BufferedReader(r)), 
+                writer);
     }
     
-    private void run(LineNumberReader in) throws IOException {
+    private void run(LineNumberReader reader, PrintWriter writer) throws IOException {
         StringBuilder buff = new StringBuilder();
         while (true) {
-            String line = in.readLine();
+            String line = reader.readLine();
             if (line == null) {
                 break;
             }
             buff.append(line).append('\n');
             if (line.length() == 0) {
-                System.out.print(filter(buff.toString()));
+                writer.print(filter(buff.toString()));
                 buff = new StringBuilder();
             }
         }
-        System.out.print(filter(buff.toString()));
+        writer.println(filter(buff.toString()));
     }        
     
     private String filter(String s) {