The statement "shutdown defrag" now compresses the database (with the MVStore).…

The statement "shutdown defrag" now compresses the database (with the MVStore). The MVStore now automatically shrinks the file in the background if there is no read or write activity. This might be a bit too aggressive; feedback is welcome!

h2/src/docsrc/html/changelog.html

@@ -17,7 +17,12 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>Change default value of PAGE_SIZE from 2048 to 4096 to more closely match most filesystems block size 
+<ul><li>File system abstraction: support replacing existing files using move
+	(currently not for Windows).
+</li><li>The statement "shutdown defrag" now compresses the database (with the MVStore).
+</li><li>The MVStore now automatically shrinks the file in the background if there is no read or write activity.
+    This might be a bit too aggressive; feedback is welcome!
+</li><li>Change default value of PAGE_SIZE from 2048 to 4096 to more closely match most filesystems block size 
     (PageStore only; the MVStore already used 4096).
 </li><li>Auto-scale MAX_MEMORY_ROWS and CACHE_SIZE settings by the amount of available RAM. Gives a better
     out of box experience for people with more powerful machines.

h2/src/docsrc/html/roadmap.html

@@ -342,7 +342,7 @@ See also <a href="build.html#providing_patches">Providing Patches</a>.
 </li><li>Batch parameter for INSERT, UPDATE, and DELETE, and commit after each batch. See also MySQL DELETE.
 </li><li>MySQL compatibility: support ALTER TABLE .. MODIFY COLUMN.
 </li><li>Use a lazy and auto-close input stream (open resource when reading, close on eof).
-</li><li>PostgreSQL compatibility: generate_series.
+</li><li>PostgreSQL compatibility: generate_series (as an alias for system_range).
 </li><li>Connection pool: 'reset session' command (delete temp tables, rollback, auto-commit true).
 </li><li>Improve SQL documentation, see http://www.w3schools.com/sql/
 </li><li>MySQL compatibility: DatabaseMetaData.stores*() methods should return the same values. Test with SquirrelSQL.

h2/src/main/org/h2/engine/Database.java

@@ -1335,14 +1335,13 @@ public class Database implements DataHandler {
         }
         reconnectModified(false);
         if (mvStore != null) {
-            if (!readOnly) {
+            long maxCompactTime = dbSettings.maxCompactTime;
             if (compactMode == CommandInterface.SHUTDOWN_COMPACT) {
                 mvStore.compactFile(dbSettings.maxCompactTime);
             } else if (compactMode == CommandInterface.SHUTDOWN_DEFRAG) {
-                    mvStore.compactFile(Long.MAX_VALUE);
-                }
+                maxCompactTime = Long.MAX_VALUE;
             }
-            mvStore.close(dbSettings.maxCompactTime);
+            mvStore.close(maxCompactTime);
         }
         closeFiles();
         if (persistent && lock == null &&

h2/src/main/org/h2/mvstore/ConcurrentArrayList.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.mvstore;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+
+/**
+ * A very simple array list that supports concurrent access.
+ * Internally, it uses immutable objects.
+ *
+ * @param <K> the key type
+ */
+public class ConcurrentArrayList<K> {
+
+    /**
+     * The list.
+     */
+    volatile List<K> list = new List<K>(null, 0, 0);
+
+    /**
+     * Get the first element, or null if none.
+     *
+     * @return the first element
+     */
+    public K peekFirst() {
+        return list.peekFirst();
+    }
+
+    /**
+     * Get the last element, or null if none.
+     *
+     * @return the last element
+     */
+    public K peekLast() {
+        return list.peekLast();
+    }
+
+    /**
+     * Add an element at the end.
+     *
+     * @param obj the element
+     */
+    public synchronized void add(K obj) {
+        K[] array = new K[list.length + 1];
+        if (list.length > 0) {
+            System.arraycopy(list.list, list.offset, dest, destPos, length)
+        Arrays.copyOf(, newLength)
+        list = new List<K>(list.add(obj);
+    }
+
+    /**
+     * Remove the first element, if it matches.
+     *
+     * @param obj the element to remove
+     * @return true if the element matched and was removed
+     */
+    public synchronized boolean removeFirst(K obj) {
+        if (head.obj != obj) {
+            return false;
+        }
+        head = head.next;
+        return true;
+    }
+
+    /**
+     * Remove the last element, if it matches.
+     *
+     * @param obj the element to remove
+     * @return true if the element matched and was removed
+     */
+    public synchronized boolean removeLast(K obj) {
+        if (peekLast() != obj) {
+            return false;
+        }
+        head = Entry.removeLast(head);
+        return true;
+    }
+
+    /**
+     * Get an iterator over all entries.
+     *
+     * @return the iterator
+     */
+    public Iterator<K> iterator() {
+        return new Iterator<K>() {
+
+            List<K> list = head;
+
+            @Override
+            public boolean hasNext() {
+                return current != NULL;
+            }
+
+            @Override
+            public K next() {
+                K x = current.obj;
+                current = current.next;
+                return x;
+            }
+
+            @Override
+            public void remove() {
+                throw DataUtils.newUnsupportedOperationException("remove");
+            }
+
+        };
+    }
+
+    /**
+     * An entry in the linked list.
+     */
+    private static class List<K> {
+        final K[] list;
+        final int offset;
+        final int length;
+
+        List(K[] list, int offset, int length) {
+            this.list = list;
+            this.offset = offset;
+            this.length = length;
+        }
+
+        public K peekFirst() {
+            return length == 0 ? null : list[offset];
+        }
+
+        public K peekLast() {
+            return length == 0 ? null : list[offset + length - 1];
+        }
+
+    }
+
+}

h2/src/main/org/h2/mvstore/FileStore.java

@@ -125,6 +125,9 @@ public class FileStore {
      *            used
      */
     public void open(String fileName, boolean readOnly, char[] encryptionKey) {
+        if (file != null) {
+            return;
+        }
         if (fileName != null) {
             if (FilePath.get(fileName) instanceof FilePathDisk) {
                 // NIO is used, unless a different file system is specified the
@@ -356,4 +359,13 @@ public class FileStore {
         freeSpace.clear();
     }
 
+    /**
+     * Get the file name.
+     * 
+     * @return the file name
+     */
+    public String getFileName() {
+        return fileName;
+    }
+
 }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -42,6 +42,7 @@ TransactionStore:
     if there is only one connection
 
 MVStore:
+- FileStore: don't open and close when set using MVStore.Builder.fileStore
 - test and possibly improve compact operation (for large dbs)
 - is data kept in the stream store if the transaction is not committed?
 - automated 'kill process' and 'power failure' test
@@ -136,6 +137,7 @@ public class MVStore {
     private boolean closed;
 
     private FileStore fileStore;
+    private boolean fileStoreIsProvided;
 
     private final int pageSplitSize;
 
@@ -273,7 +275,10 @@ public class MVStore {
             return;
         }
         if (fileStore == null) {
+            fileStoreIsProvided = false;
             fileStore = new FileStore();
+        } else {
+            fileStoreIsProvided = true;
         }
         retentionTime = fileStore.getDefaultRetentionTime();
         boolean readOnly = config.containsKey("readOnly");
@@ -297,7 +302,9 @@ public class MVStore {
         
         char[] encryptionKey = (char[]) config.get("encryptionKey");
         try {
+            if (!fileStoreIsProvided) {
                 fileStore.open(fileName, readOnly, encryptionKey);
+            }
             if (fileStore.size() == 0) {
                 creationTime = getTime();
                 lastCommitTime = creationTime;
@@ -789,7 +796,9 @@ public class MVStore {
             chunks.clear();
             maps.clear();
             try {
+                if (!fileStoreIsProvided) {
                     fileStore.close();
+                }
             } finally {
                 fileStore = null;
             }
@@ -2656,10 +2665,14 @@ public class MVStore {
         }
 
         /**
-         * Use the provided file store instead of the default one. Please note
-         * that any kind of store (including an off-heap store) is considered a
-         * "persistence", while an "in-memory store" means objects are not
-         * persisted and fully kept in the JVM heap.
+         * Use the provided file store instead of the default one.
+         * <p>
+         * File stores passed in this way need to be open. They are not closed
+         * when closing the store.
+         * <p>
+         * Please note that any kind of store (including an off-heap store) is
+         * considered a "persistence", while an "in-memory store" means objects
+         * are not persisted and fully kept in the JVM heap.
          * 
          * @param store the file store
          * @return this

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -15,6 +15,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.h2.message.DbException;
 import org.h2.mvstore.type.DataType;
 import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
@@ -310,16 +311,51 @@ public class MVStoreTool {
 
     /**
      * Compress the store by creating a new file and copying the live pages
-     * there.
+     * there. Temporarily, a file with the suffix ".tempFile" is created. This
+     * file is then renamed, replacing the original file, if possible. If not,
+     * the new file is renamed to ".newFile", then the old file is removed, and
+     * the new file is renamed. This might be interrupted, so it's better to
+     * compactCleanUp before opening a store, in case this method was used.
      * 
      * @param fileName the file name
      * @param compress whether to compress the data
      */
     public static void compact(String fileName, boolean compress) {
-        compact(fileName, fileName + ".new", compress);
-        FileUtils.moveTo(fileName, fileName + ".old");
-        FileUtils.moveTo(fileName, fileName);
-        FileUtils.delete(fileName + ".old");
+        String tempName = fileName + ".tempFile";
+        FileUtils.delete(tempName);
+        compact(fileName, tempName, compress);
+        try {
+            FileUtils.moveAtomicReplace(tempName, fileName);
+        } catch (DbException e) {
+            String newName = fileName + ".newFile";
+            FileUtils.delete(newName);
+            FileUtils.move(tempName, newName);
+            FileUtils.delete(fileName);
+            FileUtils.move(newName, fileName);
+        }
+    }
+    
+    /**
+     * Clean up if needed, in a case a compact operation was interrupted due to
+     * killing the process or a power failure. This will delete temporary files
+     * (if any), and in case atomic file replacements were not used, rename the
+     * new file.
+     * 
+     * @param fileName the file name
+     */
+    public static void compactCleanUp(String fileName) {
+        String tempName = fileName + ".tempFile";
+        if (FileUtils.exists(tempName)) {
+            FileUtils.delete(tempName);
+        }
+        String newName = fileName + ".newFile";
+        if (FileUtils.exists(newName)) {
+            if (FileUtils.exists(fileName)) {
+                FileUtils.delete(newName);
+            } else {
+                FileUtils.move(newName, fileName);
+            }
+        }
     }
 
     /**
@@ -340,6 +376,18 @@ public class MVStoreTool {
             b.compress();
         }
         MVStore target = b.open();
+        compact(source, target);
+        target.close();
+        source.close();
+    }
+    
+    /**
+     * Copy all live pages from the source store to the target store.
+     *
+     * @param source the source store
+     * @param target the target store
+     */
+    public static void compact(MVStore source, MVStore target) {
         MVMap<String, String> sourceMeta = source.getMetaMap();
         MVMap<String, String> targetMeta = target.getMetaMap();
         for (Entry<String, String> m : sourceMeta.entrySet()) {
@@ -365,8 +413,6 @@ public class MVStoreTool {
             MVMap<Object, Object> targetMap = target.openMap(mapName, mp);
             targetMap.copyFrom(sourceMap);
         }
-        target.close();
-        source.close();
     }
 
     /**

h2/src/main/org/h2/mvstore/OffHeapStore.java

@@ -21,7 +21,7 @@ public class OffHeapStore extends FileStore {
 
     @Override
     public void open(String fileName, boolean readOnly, char[] encryptionKey) {
-        // nothing to do
+        memory.clear();
     }
 
     @Override
@@ -132,7 +132,7 @@ public class OffHeapStore extends FileStore {
 
     @Override
     public void close() {
-        // do nothing (keep the data until it is garbage collected)
+        memory.clear();
     }
 
     @Override

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -25,6 +25,7 @@ import org.h2.mvstore.DataUtils;
 import org.h2.mvstore.FileStore;
 import org.h2.mvstore.MVMap;
 import org.h2.mvstore.MVStore;
+import org.h2.mvstore.MVStoreTool;
 import org.h2.mvstore.db.TransactionStore.Transaction;
 import org.h2.mvstore.db.TransactionStore.TransactionMap;
 import org.h2.store.InDoubtTransaction;
@@ -57,6 +58,7 @@ public class MVTableEngine implements TableEngine {
             store = new Store(db, builder.open());
         } else {
             String fileName = dbPath + Constants.SUFFIX_MV_FILE;
+            MVStoreTool.compactCleanUp(fileName);
             builder.fileName(fileName);
             if (db.isReadOnly()) {
                 builder.readOnly();
@@ -310,12 +312,6 @@ public class MVTableEngine implements TableEngine {
          */
         public void compactFile(long maxCompactTime) {
             store.setRetentionTime(0);
-            if (maxCompactTime == Long.MAX_VALUE) {
-                store.setReuseSpace(false);
-                store.compactRewriteFully();
-                store.setReuseSpace(true);
-                store.compactMoveChunks();
-            } else {
             long start = System.currentTimeMillis();
             while (store.compact(95, 16 * 1024 * 1024)) {
                 store.sync();
@@ -326,7 +322,6 @@ public class MVTableEngine implements TableEngine {
                 }
             }
         }
-        }
 
         /**
          * Close the store. Pending changes are persisted. Chunks with a low
@@ -338,14 +333,18 @@ public class MVTableEngine implements TableEngine {
         public void close(long maxCompactTime) {
             try {
                 if (!store.isClosed() && store.getFileStore() != null) {
+                    boolean compactFully = false;
                     if (!store.getFileStore().isReadOnly()) {
                         transactionStore.close();
-                        if (maxCompactTime > 0) {
-                            store.compact(95, 1024 * 1024);
-                            store.compactMoveChunks(95, 1024 * 1024);
+                        if (maxCompactTime == Long.MAX_VALUE) {
+                            compactFully = true;
                         }
                     }
+                    String fileName = store.getFileStore().getFileName();
                     store.close();
+                    if (compactFully) {
+                        MVStoreTool.compact(fileName, true);
+                    }
                 }
             } catch (IllegalStateException e) {
                 throw DbException.get(ErrorCode.IO_EXCEPTION_1, e, "Closing");

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -13,6 +13,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
 import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.h2.mvstore.Chunk;
@@ -51,6 +52,7 @@ public class TestMVStore extends TestBase {
     public void test() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
         FileUtils.createDirectories(getBaseDir());
+        testProvidedFileStoreNotOpenedAndClosed();
         testVolatileMap();
         testEntrySet();
         testCompressEmptyPage();
@@ -110,6 +112,29 @@ public class TestMVStore extends TestBase {
         testLargerThan2G();
     }
     
+    private void testProvidedFileStoreNotOpenedAndClosed() {
+        final AtomicInteger openClose = new AtomicInteger();
+        FileStore fileStore = new OffHeapStore() {
+            
+            @Override
+            public void open(String fileName, boolean readOnly, char[] encryptionKey) {
+                openClose.incrementAndGet();
+                super.open(fileName, readOnly, encryptionKey);
+            }
+            
+            @Override
+            public void close() {
+                openClose.incrementAndGet();
+                super.close();
+            }
+        };
+        MVStore store = new MVStore.Builder().
+                fileStore(fileStore).
+                open();
+        store.close();
+        assertEquals(0, openClose.get());
+    }
+
     private void testVolatileMap() {
         String fileName = getBaseDir() + "/testVolatile.h3";
         MVStore store = new MVStore.Builder().

h2/src/test/org/h2/test/store/TestMVStoreTool.java

@@ -86,10 +86,17 @@ public class TestMVStoreTool extends TestBase {
                 fileName(fileName + ".new.compress").readOnly().open();
         assertEquals(s1, s2);
         assertEquals(s1, s3);
+        s1.close();
+        s2.close();
+        s3.close();
         long size1 = FileUtils.size(fileName);
         long size2 = FileUtils.size(fileName + ".new");
         long size3 = FileUtils.size(fileName + ".new.compress");
         assertTrue("size1: " + size1 + " size2: " + size2 + " size3: " + size3, size2 < size1 && size3 < size2);
+        MVStoreTool.compact(fileName, false);
+        assertEquals(size2, FileUtils.size(fileName));
+        MVStoreTool.compact(fileName, true);
+        assertEquals(size3, FileUtils.size(fileName));
     }
 
     private void assertEquals(MVStore a, MVStore b) {

h2/src/test/org/h2/test/synth/TestDiskFull.java

@@ -54,7 +54,7 @@ public class TestDiskFull extends TestBase {
         fs.setDiskFullCount(x, 0);
         String url = "jdbc:h2:unstable:memFS:diskFull" + x +
             ";FILE_LOCK=NO;TRACE_LEVEL_FILE=0;WRITE_DELAY=10;" +
-            "LOCK_TIMEOUT=100;CACHE_SIZE=4096";
+            "LOCK_TIMEOUT=100;CACHE_SIZE=4096;MAX_COMPACT_TIME=10";
         url = getURL(url, true);
         Connection conn = null;
         Statement stat = null;