Mainly MVStore improvements

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,11 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>The license has changed to MPL 2.0 + EPL 1.0.
+<ul><li>Recursive queries with many result rows (more than the setting "max_memory_rows") 
+    did not work correctly.
+</li><li>The license has changed to MPL 2.0 + EPL 1.0.
+</li><li>MVStore: temporary tables from result sets could survive re-opening a database,
+    which could result in a ClassCastException.
 </li><li>MVStore: unique indexes that were created later on did not work correctly
     if there were over 5000 rows in the table.
 </li><li>MVStore: creating secondary indexes on large tables 

h2/src/main/org/h2/command/dml/Select.java

@@ -603,7 +603,6 @@ public class Select extends Query {
         }
         if (randomAccessResult) {
             result = createLocalResult(result);
-            result.setRandomAccess();
         }
         if (isGroupQuery && !isGroupSortedQuery) {
             result = createLocalResult(result);

h2/src/main/org/h2/engine/Database.java

@@ -18,6 +18,7 @@ import java.util.StringTokenizer;
 import org.h2.api.DatabaseEventListener;
 import org.h2.api.ErrorCode;
 import org.h2.api.JavaObjectSerializer;
+import org.h2.command.CommandInterface;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.command.dml.SetTypes;
 import org.h2.constraint.Constraint;
@@ -725,7 +726,7 @@ public class Database implements DataHandler {
         }
         if (mvStore != null) {
             mvStore.initTransactions();
-            mvStore.removeTemporaryMaps();
+            mvStore.removeTemporaryMaps(objectIds);
         }
         recompileInvalidViews(systemSession);
         starting = false;
@@ -1325,8 +1326,12 @@ public class Database implements DataHandler {
         }
         reconnectModified(false);
         if (mvStore != null) {
-            if (!readOnly && compactMode != 0) {
+            if (!readOnly) {
+                if (compactMode == CommandInterface.SHUTDOWN_COMPACT) {
                     mvStore.compactFile(dbSettings.maxCompactTime);
+                } else if (compactMode == CommandInterface.SHUTDOWN_DEFRAG) {
+                    mvStore.compactFile(Long.MAX_VALUE);
+                }
             }
             mvStore.close(dbSettings.maxCompactTime);
         }

h2/src/main/org/h2/engine/Session.java

@@ -27,7 +27,7 @@ import org.h2.message.TraceSystem;
 import org.h2.mvstore.db.MVTable;
 import org.h2.mvstore.db.TransactionStore.Change;
 import org.h2.mvstore.db.TransactionStore.Transaction;
-import org.h2.result.ResultInterface;
+import org.h2.result.LocalResult;
 import org.h2.result.Row;
 import org.h2.schema.Schema;
 import org.h2.store.DataHandler;
@@ -99,7 +99,7 @@ public class Session extends SessionWithState {
     private long transactionStart;
     private long currentCommandStart;
     private HashMap<String, Value> variables;
-    private HashSet<ResultInterface> temporaryResults;
+    private HashSet<LocalResult> temporaryResults;
     private int queryTimeout;
     private boolean commitOrRollbackDisabled;
     private Table waitForLock;
@@ -1268,7 +1268,7 @@ public class Session extends SessionWithState {
      *
      * @param result the temporary result set
      */
-    public void addTemporaryResult(ResultInterface result) {
+    public void addTemporaryResult(LocalResult result) {
         if (!result.needToClose()) {
             return;
         }
@@ -1283,7 +1283,7 @@ public class Session extends SessionWithState {
 
     private void closeTemporaryResults() {
         if (temporaryResults != null) {
-            for (ResultInterface result : temporaryResults) {
+            for (LocalResult result : temporaryResults) {
                 result.close();
             }
             temporaryResults = null;

h2/src/main/org/h2/mvstore/Cursor.java

@@ -21,6 +21,7 @@ public class Cursor<K, V> implements Iterator<K> {
     private CursorPos pos;
     private K current, last;
     private V currentValue, lastValue;
+    private Page lastPage;
     private final Page root;
     private boolean initialized;
 
@@ -46,6 +47,7 @@ public class Cursor<K, V> implements Iterator<K> {
         K c = current;
         last = current;
         lastValue = currentValue;
+        lastPage = pos == null ? null : pos.page;
         fetchNext();
         return c;
     }
@@ -68,6 +70,10 @@ public class Cursor<K, V> implements Iterator<K> {
         return lastValue;
     }
     
+    Page getPage() {
+        return lastPage;
+    }
+
     /**
      * Skip over that many entries. This method is relatively fast (for this map
      * implementation) even if many entries need to be skipped.

h2/src/main/org/h2/mvstore/MVMap.java

@@ -773,6 +773,72 @@ public class MVMap<K, V> extends AbstractMap<K, V>
         return new Cursor<K, V>(this, root, from);
     }
     
+    /**
+     * Re-write any pages that belong to one of the chunks in the given set.
+     * 
+     * @param set the set of chunk ids
+     */
+    public void rewrite(Set<Integer> set) {
+        rewrite(root, set);
+    }
+
+    public int rewrite(Page p, Set<Integer> set) {
+        ; // TODO write more tests
+        if (p.isLeaf()) {
+            long pos = p.getPos();
+            if (pos == 0) {
+                return 0;
+            }
+            int chunkId = DataUtils.getPageChunkId(pos);
+            if (!set.contains(chunkId)) {
+                return 0;
+            }
+            @SuppressWarnings("unchecked")
+            K key = (K) p.getKey(0);
+            @SuppressWarnings("unchecked")
+            V value = (V) p.getValue(0);
+            put(key, value);
+            return 1;
+        }
+        int writtenPageCount = 0;
+        for (int i = 0; i < p.getChildPageCount(); i++) {
+            long pos = p.getChildPagePos(i);
+            if (pos == 0) {
+                continue;
+            }
+            if (DataUtils.getPageType(pos) == DataUtils.PAGE_TYPE_LEAF) {
+                int chunkId = DataUtils.getPageChunkId(pos);
+                if (!set.contains(chunkId)) {
+                    continue;
+                }
+            }
+            writtenPageCount += rewrite(p.getChildPage(i), set);
+        }
+        if (writtenPageCount == 0) {
+            long pos = p.getPos();
+            if (pos != 0) {
+                int chunkId = DataUtils.getPageChunkId(pos);
+                if (set.contains(chunkId)) {
+                    // an inner node page that is in one of the chunks,
+                    // but only points to chunks that are not in the set:
+                    // if no child was changed, we need to do that now
+                    Page p2 = p;
+                    while (!p2.isLeaf()) {
+                        p2 = p2.getChildPage(0);
+                    }
+                    @SuppressWarnings("unchecked")
+                    K key = (K) p2.getKey(0);
+                    @SuppressWarnings("unchecked")
+                    V value = (V) p2.getValue(0);
+                    put(key, value);
+                    writtenPageCount++;
+                }
+            }
+        }
+        return writtenPageCount;
+    }
+
+
     /**
      * Get a cursor to iterate over a number of keys and values.
      *

h2/src/main/org/h2/mvstore/MVStore.java

@@ -203,12 +203,12 @@ public class MVStore {
     private long lastStoredVersion;
 
     /**
-     * The estimated number of average-sized unsaved pages. This number may not
-     * be completely accurate, because it may be changed concurrently, and
-     * because temporary pages are counted.
+     * The estimated memory used by unsaved pages. This number is not accurate,
+     * also because it may be changed concurrently, and because temporary pages
+     * are counted.
      */
-    private int unsavedPageCount;
-    private int autoCommitPageCount;
+    private int unsavedMemory;
+    private int autoCommitMemory;
     private boolean saveNeeded;
 
     /**
@@ -287,9 +287,7 @@ public class MVStore {
         o = config.get("autoCommitBufferSize");
         int kb = o == null ? 512 : (Integer) o;
         // 19 KB memory is about 1 KB storage
-        int autoCommitBufferSize = kb * 1024 * 19;
-        int div = pageSplitSize;
-        autoCommitPageCount = autoCommitBufferSize / (div == 0 ? 1 : div);
+        autoCommitMemory = kb * 1024 * 19;
         char[] encryptionKey = (char[]) config.get("encryptionKey");
         try {
             fileStore.open(fileName, readOnly, encryptionKey);
@@ -916,7 +914,7 @@ public class MVStore {
     }
 
     private long storeNow() {
-        int currentUnsavedPageCount = unsavedPageCount;
+        int currentUnsavedPageCount = unsavedMemory;
         long storeVersion = currentStoreVersion;
         long version = ++currentVersion;
         setWriteVersion(version);
@@ -1099,18 +1097,6 @@ public class MVStore {
                         chunkId++;
                     }
                 }
-
-//                    }
-//                    while (chunkId <= lastChunk.id) {
-//                        if (chunks.get(chunkId) == null) {
-//                            // one of the chunks in between
-//                            // was removed
-//                            needHeader = true;
-//                            break;
-//                        }
-//                        chunkId++;
-//                    }
-//                }
             }
         }
 
@@ -1133,7 +1119,7 @@ public class MVStore {
 
         // some pages might have been changed in the meantime (in the newest
         // version)
-        unsavedPageCount = Math.max(0, unsavedPageCount
+        unsavedMemory = Math.max(0, unsavedMemory
                 - currentUnsavedPageCount);
 
         metaChanged = false;
@@ -1322,6 +1308,39 @@ public class MVStore {
         return Chunk.readChunkHeader(buff, p);
     }
     
+    /**
+     * Compact the store by moving all live pages to new chunks.
+     *
+     * @return if anything was written
+     */
+    public synchronized boolean compactRewriteFully() {
+        checkOpen();
+        if (lastChunk == null) {
+            // nothing to do
+            return false;
+        }
+        ; // TODO write tests
+        for (MVMap<?, ?> m : maps.values()) {
+            @SuppressWarnings("unchecked")
+            MVMap<Object, Object> map = (MVMap<Object, Object>) m;
+            Cursor<Object, Object> cursor = map.cursor(null);
+            Page lastPage = null;
+            while (cursor.hasNext()) {
+                cursor.next();
+                Page p = cursor.getPage();
+                if (p == lastPage) {
+                    continue;
+                }
+                Object k = p.getKey(0);
+                Object v = p.getValue(0);
+                map.put(k, v);
+                lastPage = p;
+            }
+        }
+        commitAndSave();
+        return true;
+    }
+    
     /**
      * Compact the store by moving all chunks next to each other, if there is
      * free space between chunks. This might temporarily double the file size.
@@ -1448,6 +1467,132 @@ public class MVStore {
         fileStore.sync();
     }
     
+    /**
+     * Try to increase the fill rate by re-writing partially full chunks. Chunks
+     * with a low number of live items are re-written.
+     * <p>
+     * If the current fill rate is higher than the target fill rate, nothing is
+     * done. If not at least a minimum amount of space can be saved, nothing is
+     * done.
+     * <p>
+     * Please note this method will not necessarily reduce the file size, as
+     * empty chunks are not overwritten.
+     * <p>
+     * Only data of open maps can be moved. For maps that are not open, the old
+     * chunk is still referenced. Therefore, it is recommended to open all maps
+     * before calling this method.
+     *
+     * @param targetFillRate the minimum percentage of live entries
+     * @param saving the amount of saved space
+     * @return if a chunk was re-written
+     */
+    public synchronized boolean compact(int targetFillRate, int saving) {
+        checkOpen();
+        if (lastChunk == null) {
+            // nothing to do
+            return false;
+        }
+
+        // calculate the fill rate
+        long maxLengthSum = 0;
+        long maxLengthLiveSum = 0;
+        for (Chunk c : chunks.values()) {
+            maxLengthSum += c.maxLen;
+            maxLengthLiveSum += c.maxLenLive;
+        }
+        // the fill rate of all chunks combined
+        if (maxLengthSum <= 0) {
+            // avoid division by 0
+            maxLengthSum = 1;
+        }
+        int fillRate = (int) (100 * maxLengthLiveSum / maxLengthSum);
+        if (fillRate >= targetFillRate) {
+            return false;
+        }
+
+        long time = getTime();
+
+        // the 'old' list contains the chunks we want to free up
+        ArrayList<Chunk> old = New.arrayList();
+        Chunk last = chunks.get(lastChunk.id);
+        for (Chunk c : chunks.values()) {
+            if (canOverwriteChunk(c, time)) {
+                long age = last.version - c.version + 1;
+                c.collectPriority = (int) (c.getFillRate() / age);
+                old.add(c);
+            }
+        }
+        if (old.size() == 0) {
+            return false;
+        }
+
+        // sort the list, so the first entry should be collected first
+        Collections.sort(old, new Comparator<Chunk>() {
+            @Override
+            public int compare(Chunk o1, Chunk o2) {
+                int comp = new Integer(o1.collectPriority).
+                        compareTo(o2.collectPriority);
+                if (comp == 0) {
+                    comp = new Long(o1.maxLenLive).
+                        compareTo(o2.maxLenLive);
+                }
+                return comp;
+            }
+        });
+        // find out up to were in the old list we need to move
+        long saved = 0;
+        int chunkCount = 0;
+        Chunk move = null;
+        for (Chunk c : old) {
+            long size = c.maxLen - c.maxLenLive;
+            if (move != null) {
+                if (saved > saving) {
+                    break;
+                }
+            }
+            saved += size;
+            chunkCount++;
+            move = c;
+        }
+        if (chunkCount <= 1) {
+            return false;
+        }
+        // remove the chunks we want to keep from this list
+        boolean remove = false;
+        for (Iterator<Chunk> it = old.iterator(); it.hasNext();) {
+            Chunk c = it.next();
+            if (move == c) {
+                remove = true;
+            } else if (remove) {
+                it.remove();
+            }
+        }
+        HashSet<Integer> set = New.hashSet();
+        for (Chunk c : old) {
+            set.add(c.id);
+        }
+        for (MVMap<?, ?> m : maps.values()) {
+            @SuppressWarnings("unchecked")
+            MVMap<Object, Object> map = (MVMap<Object, Object>) m;
+            ; // TODO write more tests
+            map.rewrite(set);
+        }
+        commitAndSave();
+        boolean again = false;
+        for (Chunk c : old) {
+            if (c.maxLenLive > 0) {
+                // not cleared - that means bookkeeping of live pages
+                // is broken; copyLive will fix this
+                again = true;
+                copyLive(c);
+            }
+        }
+        if (again) {
+            commitAndSave();
+        }
+        return true;
+    }
+
     /**
      * Try to increase the fill rate by re-writing partially full chunks. Chunks
      * with a low number of live items are re-written.
@@ -1468,7 +1613,7 @@ public class MVStore {
      *      which is also the size of the new chunk
      * @return if a chunk was re-written
      */
-    public synchronized boolean compact(int targetFillRate, int minSaving) {
+    public synchronized boolean compactOld(int targetFillRate, int minSaving) {
         checkOpen();
         if (lastChunk == null) {
             // nothing to do
@@ -1689,8 +1834,7 @@ public class MVStore {
             // the value could be smaller than 0 because
             // in some cases a page is allocated,
             // but never stored
-            int count = 1 + memory / pageSplitSize;
-            unsavedPageCount = Math.max(0, unsavedPageCount - count);
+            unsavedMemory = Math.max(0, unsavedMemory - memory);
             return;
         }
 
@@ -1891,10 +2035,9 @@ public class MVStore {
      * @param memory the memory usage of the page
      */
     void registerUnsavedPage(int memory) {
-        int count = 1 + memory / pageSplitSize;
-        unsavedPageCount += count;
-        int newValue = unsavedPageCount;
-        if (newValue > autoCommitPageCount && autoCommitPageCount > 0) {
+        unsavedMemory += memory;
+        int newValue = unsavedMemory;
+        if (newValue > autoCommitMemory && autoCommitMemory > 0) {
             saveNeeded = true;
         }
     }
@@ -1905,9 +2048,12 @@ public class MVStore {
     void beforeWrite() {
         if (saveNeeded) {
             saveNeeded = false;
+            // check again, because it could have been written by now
+            if (unsavedMemory > autoCommitMemory && autoCommitMemory > 0) {
                 commitAndSave();
             }
         }
+    }
 
     /**
      * Get the store version. The store version is usually used to upgrade the
@@ -2162,7 +2308,7 @@ public class MVStore {
      * Commit and save all changes, if there are any.
      */
     void commitInBackground() {
-        if (unsavedPageCount == 0 || closed) {
+        if (unsavedMemory == 0 || closed) {
             return;
         }
 
@@ -2266,26 +2412,25 @@ public class MVStore {
     }
 
     /**
-     * Get the maximum number of unsaved pages. If this number is exceeded,
-     * unsaved changes are stored to disk.
+     * Get the maximum memory (in bytes) used for unsaved pages. If this number
+     * is exceeded, unsaved changes are stored to disk.
      * 
-     * @return the number of maximum unsaved pages
+     * @return the memory in bytes
      */
-    public int getAutoCommitPageCount() {
-        return autoCommitPageCount;
+    public int getAutoCommitMemory() {
+        return autoCommitMemory;
     }
 
     /**
-     * Get the estimated number of unsaved pages. If the value exceeds the
-     * auto-commit page count, the changes are committed.
+     * Get the estimated memory (in bytes) of unsaved data. If the value exceeds the
+     * auto-commit memory, the changes are committed.
      * <p>
-     * The returned value may not be completely accurate, but can be used to
-     * estimate the memory usage for unsaved data.
+     * The returned value is an estimation only.
      *
-     * @return the number of unsaved pages
+     * @return the memory in bytes
      */
-    public int getUnsavedPageCount() {
-        return unsavedPageCount;
+    public int getUnsavedMemory() {
+        return unsavedMemory;
     }
 
     /**
@@ -2399,9 +2544,9 @@ public class MVStore {
         }
 
         /**
-         * Set the size of the write buffer, in KB (for file-based stores).
-         * Unless auto-commit is disabled, changes are automatically saved if
-         * there are more than this amount of changes.
+         * Set the size of the write buffer, in KB disk space (for file-based
+         * stores). Unless auto-commit is disabled, changes are automatically
+         * saved if there are more than this amount of changes.
          * <p>
          * The default is 512 KB.
          * <p>

h2/src/main/org/h2/mvstore/Page.java

@@ -222,6 +222,10 @@ public class Page {
         return p != null ? p : map.readPage(children[index]);
     }
     
+    public long getChildPagePos(int index) {
+        return children[index];
+    }
+
     /**
      * Get the child page at the given index, if it is live.
      *

h2/src/main/org/h2/store/Data.java

@@ -14,11 +14,9 @@ import java.io.OutputStream;
 import java.io.Reader;
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.sql.Date;
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
-import java.sql.Time;
 import java.sql.Timestamp;
 
 import org.h2.api.ErrorCode;

h2/src/main/org/h2/store/fs/FilePath.java

@@ -321,4 +321,35 @@ public abstract class FilePath {
         return this;
     }
     
+    /**
+     * The options to open a file.
+     */
+    public enum FileOpenOption {
+        
+        /**
+         * Append at the end of the file.
+         */
+        APPEND,
+        
+        /**
+         * Written data is not buffered.
+         */
+        DSYNC,
+        
+        /**
+         * Open the file for read access.
+         */
+        READ,
+        
+        /**
+         * Written data and metadata is not buffered.
+         */
+        SYNC,
+        
+        /**
+         * Open the file for write access.
+         */
+        WRITE,
+    }
+
 }

h2/src/main/org/h2/table/Column.java

@@ -6,9 +6,7 @@
  */
 package org.h2.table;
 
-import java.sql.Date;
 import java.sql.ResultSetMetaData;
-import java.sql.Timestamp;
 
 import org.h2.api.ErrorCode;
 import org.h2.command.Parser;

h2/src/main/org/h2/table/TableView.java

@@ -26,7 +26,6 @@ import org.h2.index.IndexType;
 import org.h2.index.ViewIndex;
 import org.h2.message.DbException;
 import org.h2.result.LocalResult;
-import org.h2.result.ResultInterface;
 import org.h2.result.Row;
 import org.h2.result.SortOrder;
 import org.h2.schema.Schema;
@@ -46,41 +45,6 @@ public class TableView extends Table {
 
     private static final long ROW_COUNT_APPROXIMATION = 100;
     
-    private static final class CacheKey {
-        private final int[] masks;
-        private final Session session;
-        
-        public CacheKey(int[] masks, Session session) {
-            this.masks = masks;
-            this.session = session;
-        }
-
-        @Override
-        public int hashCode() {
-            final int prime = 31;
-            int result = 1;
-            result = prime * result + Arrays.hashCode(masks);
-            result = prime * result + session.hashCode();
-            return result;
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (this == obj)
-                return true;
-            if (obj == null)
-                return false;
-            if (getClass() != obj.getClass())
-                return false;
-            CacheKey other = (CacheKey) obj;
-            if (!Arrays.equals(masks, other.masks))
-                return false;
-            if (session != other.session)
-                return false;
-            return true;
-        }
-    }
-
     private String querySQL;
     private ArrayList<Table> tables;
     private String[] columnNames;
@@ -572,7 +536,7 @@ public class TableView extends Table {
         this.recursiveResult = value;
     }
 
-    public ResultInterface getRecursiveResult() {
+    public LocalResult getRecursiveResult() {
         return recursiveResult;
     }
 
@@ -596,4 +560,48 @@ public class TableView extends Table {
         }
     }
     
+    /**
+     * The key of the index cache for views.
+     */
+    private static final class CacheKey {
+        
+        private final int[] masks;
+        private final Session session;
+        
+        public CacheKey(int[] masks, Session session) {
+            this.masks = masks;
+            this.session = session;
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + Arrays.hashCode(masks);
+            result = prime * result + session.hashCode();
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            CacheKey other = (CacheKey) obj;
+            if (session != other.session) {
+                return false;
+            }
+            if (!Arrays.equals(masks, other.masks)) {
+                return false;
+            }
+            return true;
+        }
+    }
+
 }

h2/src/main/org/h2/tools/Console.java

@@ -4,6 +4,11 @@
  * (http://h2database.com/html/license.html).
  * Initial Developer: H2 Group
  */
+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0, 
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
 package org.h2.tools;
 
 //## AWT ##

h2/src/test/org/h2/test/TestAll.java

@@ -362,10 +362,10 @@ java org.h2.test.TestAll timer
      */
     String cacheType;
 
-    private Server server;
-
     AbbaLockingDetector abbaLockingDetector;
     
+    private Server server;
+
     /**
      * Run all tests.
      *
@@ -385,7 +385,15 @@ java org.h2.test.TestAll timer
         
         // use lower values, to better test those cases,
         // and to speed up the tests (for delays)
-        System.setProperty("h2.maxMemoryRows", "128");
+  
+        ; // TEST
+//        System.setProperty("h2.maxMemoryRows", "2");
+        
+        System.setProperty("h2.maxMemoryRows", "100");
+//        System.setProperty("h2.maxMemoryRows", "1000");
+//        System.setProperty("h2.maxMemoryRows", "2");
+        
+        
         System.setProperty("h2.check2", "true");
         System.setProperty("h2.delayWrongPasswordMin", "0");
         System.setProperty("h2.delayWrongPasswordMax", "0");
@@ -610,7 +618,6 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestAlterSchemaRename().runTest(this);
         new TestAutoRecompile().runTest(this);
         new TestBitField().runTest(this);
-        new TestBnf().runTest(this);
         new TestBackup().runTest(this);
         new TestBigDb().runTest(this);
         new TestBigResult().runTest(this);
@@ -752,6 +759,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
 
         // unit
         new TestAutoReconnect().runTest(this);
+        new TestBnf().runTest(this);
         new TestCache().runTest(this);
         new TestClearReferences().runTest(this);
         new TestCollation().runTest(this);