MVStore

h2/src/docsrc/html/mvstore.html

@@ -466,11 +466,11 @@ The plan is to use the MVStore as the default storage engine for the H2 database
 in the future (supporting SQL, JDBC, transactions, MVCC, and so on).
 This is work in progress. To try it out, append
 <code>;MV_STORE=TRUE</code>
-to the database URL. In general, functionality and performance should be
+to the database URL. In general, performance should be
 similar than the current default storage engine (the page store).
-There are a few features that have not been implemented yet or are not complete,
-for example the <code>.lock.db</code> file is still used to lock a database
-(the plan is to no longer use this file by default).
+Even thought it can be used with the default table level locking,
+it is recommended to use it together with the MVCC mode 
+(to do that, append <code>;MVCC=TRUE</code> to the database URL).
 </p>
 
 <h2 id="differences">Similar Projects and Differences to Other Storage Engines</h2>

h2/src/main/org/h2/engine/Database.java

@@ -208,7 +208,15 @@ public class Database implements DataHandler {
         if ("r".equals(accessModeData)) {
             readOnly = true;
         }
-        this.fileLockMethod = FileLock.getFileLockMethod(lockMethodName);
+        if (dbSettings.mvStore && lockMethodName == null) {
+            if (autoServerMode) {
+                fileLockMethod = FileLock.LOCK_FILE;
+            } else {
+                fileLockMethod = FileLock.LOCK_FS;
+            }
+        } else {
+            fileLockMethod = FileLock.getFileLockMethod(lockMethodName);
+        }
         this.databaseURL = ci.getURL();
         String listener = ci.removeProperty("DATABASE_EVENT_LISTENER", null);
         if (listener != null) {
@@ -806,7 +814,7 @@ public class Database implements DataHandler {
      * @param session the session
      */
     public void verifyMetaLocked(Session session) {
-        if (!lockMeta(session) && lockMode != 0) {
+        if (!lockMeta(session) && lockMode != Constants.LOCK_MODE_OFF) {
             throw DbException.throwInternalError();
         }
     }
@@ -840,7 +848,7 @@ public class Database implements DataHandler {
             Cursor cursor = metaIdIndex.find(session, r, r);
             if (cursor.next()) {
                 if (SysProperties.CHECK) {
-                    if (lockMode != 0 && !wasLocked) {
+                    if (lockMode != Constants.LOCK_MODE_OFF && !wasLocked) {
                         throw DbException.throwInternalError();
                     }
                 }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -13,6 +13,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -48,11 +49,10 @@ Documentation
 TestMVStoreDataLoss
 
 MVTableEngine:
+- verify tests don't use the PageStore
+- test and possibly allow MVCC & MULTI_THREADED
+- maybe enable MVCC by default (but allow to disable it) 
 - use StreamStore to avoid deadlocks
-- when the MVStore was enabled before, use it again
-    (probably by checking existence of the mvstore file)
-- not use the .h2.db file
-- not use the .lock.db file
 
 TransactionStore:
 
@@ -456,6 +456,24 @@ public class MVStore {
         maps.put(id, map);
         return map;
     }
+    
+    /**
+     * Get the set of all map names.
+     * 
+     * @return the set of names
+     */
+    public synchronized Set<String> getMapNames() {
+        HashSet<String> set = New.hashSet();
+        checkOpen();
+        for (Iterator<String> it = meta.keyIterator("name."); it.hasNext();) {
+            String x = it.next();
+            if (!x.startsWith("name.")) {
+                break;
+            }
+            set.add(x.substring("name.".length()));
+        }
+        return set;
+    }
 
     /**
      * Get the metadata map. This data is for informational purposes only. The
@@ -1885,11 +1903,11 @@ public class MVStore {
      * Get the name of the given map.
      *
      * @param id the map id
-     * @return the name
+     * @return the name, or null if not found
      */
-    synchronized String getMapName(int id) {
+    public synchronized String getMapName(int id) {
         String m = meta.get("map." + id);
-        return DataUtils.parseMap(m).get("name");
+        return m == null ? null : DataUtils.parseMap(m).get("name");
     }
 
     /**

h2/src/main/org/h2/mvstore/db/MVPrimaryIndex.java

@@ -214,7 +214,7 @@ public class MVPrimaryIndex extends BaseIndex {
     @Override
     public double getCost(Session session, int[] masks, TableFilter filter, SortOrder sortOrder) {
         try {
-            long cost = 10 * (dataMap.sizeAsLongEstimated() + Constants.COST_ROW_OFFSET);
+            long cost = 10 * (dataMap.sizeAsLongMax() + Constants.COST_ROW_OFFSET);
             return cost;
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
@@ -278,15 +278,24 @@ public class MVPrimaryIndex extends BaseIndex {
         return map.sizeAsLong();
     }
 
-    @Override
-    public long getRowCountApproximation() {
+    /**
+     * The maximum number of rows, including uncommitted rows of any session.
+     * 
+     * @return the maximum number of rows
+     */
+    public long getRowCountMax() {
         try {
-            return dataMap.sizeAsLongEstimated();
+            return dataMap.sizeAsLongMax();
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
         }
     }
 
+    @Override
+    public long getRowCountApproximation() {
+        return getRowCountMax();
+    }
+
     @Override
     public long getDiskSpaceUsed() {
         // TODO estimate disk space usage

h2/src/main/org/h2/mvstore/db/MVSecondaryIndex.java

@@ -195,7 +195,7 @@ public class MVSecondaryIndex extends BaseIndex {
     @Override
     public double getCost(Session session, int[] masks, TableFilter filter, SortOrder sortOrder) {
         try {
-            return 10 * getCostRangeIndex(masks, dataMap.sizeAsLongEstimated(), filter, sortOrder);
+            return 10 * getCostRangeIndex(masks, dataMap.sizeAsLongMax(), filter, sortOrder);
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
         }
@@ -244,7 +244,7 @@ public class MVSecondaryIndex extends BaseIndex {
     @Override
     public boolean needRebuild() {
         try {
-            return dataMap.sizeAsLongEstimated() == 0;
+            return dataMap.sizeAsLongMax() == 0;
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
         }
@@ -259,7 +259,7 @@ public class MVSecondaryIndex extends BaseIndex {
     @Override
     public long getRowCountApproximation() {
         try {
-            return dataMap.sizeAsLongEstimated();
+            return dataMap.sizeAsLongMax();
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
         }

h2/src/main/org/h2/mvstore/db/MVSpatialIndex.java

@@ -282,7 +282,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
     @Override
     public boolean needRebuild() {
         try {
-            return dataMap.sizeAsLongEstimated() == 0;
+            return dataMap.sizeAsLongMax() == 0;
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
         }
@@ -297,7 +297,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
     @Override
     public long getRowCountApproximation() {
         try {
-            return dataMap.sizeAsLongEstimated();
+            return dataMap.sizeAsLongMax();
         } catch (IllegalStateException e) {
             throw DbException.get(ErrorCode.OBJECT_CLOSED);
         }

h2/src/main/org/h2/mvstore/db/MVTable.java

@@ -388,7 +388,7 @@ public class MVTable extends TableBase {
             if (store.store.hasMap("index." + indexId)) {
                 mainIndexColumn = -1;
             }
-        } else if (primaryIndex.getRowCount(session) != 0) {
+        } else if (primaryIndex.getRowCountMax() != 0) {
             mainIndexColumn = -1;
         }
         if (mainIndexColumn != -1) {

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -305,6 +305,11 @@ public class MVTableEngine implements TableEngine {
             statisticsStart = fs == null ? 0 : fs.getReadCount();
         }
 
+        /**
+         * Stop collecting statistics.
+         *
+         * @return the statistics
+         */
         public Map<String, Integer> statisticsEnd() {
             HashMap<String, Integer> map = New.hashMap();
             FileStore fs = store.getFileStore();

h2/src/main/org/h2/mvstore/db/TransactionStore.java

@@ -11,7 +11,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Map.Entry;
 
 import org.h2.mvstore.Cursor;
@@ -67,6 +66,11 @@ public class TransactionStore {
     private int lastTransactionId;
 
     private int maxTransactionId = 0xffff;
+    
+    /**
+     * The next id of a temporary map.
+     */
+    private int nextTempMapId;
 
     /**
      * Create a new transaction store.
@@ -96,6 +100,13 @@ public class TransactionStore {
                 new MVMap.Builder<Long, Object[]>().
                 valueType(undoLogValueType);
         undoLog = store.openMap("undoLog", builder);
+        // remove all temporary maps
+        for (String mapName : store.getMapNames()) {
+            if (mapName.startsWith("temp.")) {
+                MVMap<Object, Integer> temp = openTempMap(mapName);
+                store.removeMap(temp);
+            }
+        }
         init();
     }
 
@@ -369,14 +380,11 @@ public class TransactionStore {
         if (map != null) {
             return map;
         }
-        // TODO open map by id if possible
-        Map<String, String> meta = store.getMetaMap();
-        String m = meta.get("map." + mapId);
-        if (m == null) {
+        String mapName = store.getMapName(mapId);
+        if (mapName == null) {
             // the map was removed later on
             return null;
         }
-        String mapName = DataUtils.parseMap(m).get("name");
         VersionedValueType vt = new VersionedValueType(dataType);
         MVMap.Builder<Object, VersionedValue> mapBuilder =
                 new MVMap.Builder<Object, VersionedValue>().
@@ -385,6 +393,24 @@ public class TransactionStore {
         maps.put(mapId, map);
         return map;
     }
+    
+    synchronized MVMap<Object, Integer> createTempMap() {
+        String mapName = "temp." + nextTempMapId++;
+        return openTempMap(mapName);
+    }
+    
+    /**
+     * Open a temporary map.
+     * 
+     * @param mapName the map name
+     * @return the map
+     */
+    MVMap<Object, Integer> openTempMap(String mapName) {
+        MVMap.Builder<Object, Integer> mapBuilder =
+                new MVMap.Builder<Object, Integer>().
+                keyType(dataType);
+        return store.openMap(mapName, mapBuilder);
+    }
 
     /**
      * End this transaction
@@ -836,11 +862,12 @@ public class TransactionStore {
         }
 
         /**
-         * Get the size of the raw map.
-         *
-         * @return the size
+         * Get the size of the raw map. This includes uncommitted entries, and
+         * transiently removed entries, so it is the maximum number of entries.
+         * 
+         * @return the maximum size
          */
-        public long sizeAsLongEstimated() {
+        public long sizeAsLongMax() {
             return map.sizeAsLong();
         }
 
@@ -850,18 +877,60 @@ public class TransactionStore {
          * @return the size
          */
         public long sizeAsLong() {
-            // TODO this method is very slow
-            long size = 0;
-            Cursor<K, VersionedValue> cursor = map.cursor(null);
-            while (cursor.hasNext()) {
-                K key = cursor.next();
-                VersionedValue data = cursor.getValue();
-                data = getValue(key, readLogId, data);
-                if (data != null && data.value != null) {
-                    size++;
+            long sizeRaw = map.sizeAsLong();
+            MVMap<Long, Object[]> undo = transaction.store.undoLog;
+            long undoLogSize;
+            synchronized (undo) {
+                undoLogSize = undo.sizeAsLong();
+            }
+            if (undoLogSize == 0) {
+                return sizeRaw;
+            }
+            if (undoLogSize > sizeRaw) {
+                // the undo log is larger than the map -
+                // count the entries of the map
+                long size = 0;
+                Cursor<K, VersionedValue> cursor = map.cursor(null);
+                while (cursor.hasNext()) {
+                    K key = cursor.next();
+                    VersionedValue data = cursor.getValue();
+                    data = getValue(key, readLogId, data);
+                    if (data != null && data.value != null) {
+                        size++;
+                    }
                 }
+                return size;
+            }
+            // the undo log is smaller than the map -
+            // scan the undo log and subtract invisible entries
+            synchronized (undo) {
+                // re-fetch in case any transaction was committed now
+                long size = map.sizeAsLong();
+                MVMap<Object, Integer> temp = transaction.store.createTempMap();
+                try {
+                    for (Entry<Long, Object[]> e : undo.entrySet()) {
+                        Object[] op = e.getValue();
+                        int m = (Integer) op[0];
+                        if (m != mapId) {
+                            // a different map - ignore
+                            continue;
+                        }
+                        @SuppressWarnings("unchecked")
+                        K key = (K) op[1];
+                        if (get(key) == null) {
+                            Integer old = temp.put(key, 1);
+                            // count each key only once
+                            // (there might be multiple changes for the same key)
+                            if (old == null) {
+                                size--;
+                            }
+                        }
+                    }
+                } finally {
+                    transaction.store.store.removeMap(temp);
+                }
+                return size;
             }
-            return size;
         }
 
         /**

h2/src/test/org/h2/test/TestBase.java

@@ -249,6 +249,7 @@ public abstract class TestBase {
         if (name.startsWith("jdbc:")) {
             if (config.mvStore) {
                 name = addOption(name, "MV_STORE", "true");
+                // name = addOption(name, "MVCC", "true");
             }
             return name;
         }
@@ -274,6 +275,7 @@ public abstract class TestBase {
         }
         if (config.mvStore) {
             url = addOption(url, "MV_STORE", "true");
+            // url = addOption(url, "MVCC", "true");
         }
         if (!config.memory) {
             if (config.smallLog && admin) {

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -1289,6 +1289,7 @@ public class TestMVStore extends TestBase {
         FileUtils.delete(fileName);
         MVStore s = openStore(fileName);
         MVMap<String, String> m = s.getMetaMap();
+        assertEquals("[]", s.getMapNames().toString());
         MVMap<String, String> data = s.openMap("data");
         data.put("1", "Hello");
         data.put("2", "World");
@@ -1296,6 +1297,11 @@ public class TestMVStore extends TestBase {
         assertEquals(1, s.getCurrentVersion());
         assertTrue(m.containsKey("chunk.1"));
         assertFalse(m.containsKey("chunk.2"));
+        
+        assertEquals("[data]", s.getMapNames().toString());
+        assertEquals("data", s.getMapName(data.getId()));
+        assertNull(s.getMapName(s.getMetaMap().getId()));
+        assertNull(s.getMapName(data.getId() + 1));
 
         String id = s.getMetaMap().get("name.data");
         assertEquals("name:data", m.get("map." + id));

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -73,8 +73,64 @@ public class TestMVTableEngine extends TestBase {
     }
 
     private void testCount() throws Exception {
-        ;
-        // Test that count(*) is fast even if the map is very large
+        if (config.memory) {
+            return;
+        }
+        
+        FileUtils.deleteRecursive(getBaseDir(), true);
+        Connection conn;
+        Connection conn2;
+        Statement stat;
+        Statement stat2;
+        String url = "mvstore;MV_STORE=TRUE;MVCC=TRUE";
+        url = getURL(url, true);
+        conn = getConnection(url);
+        stat = conn.createStatement();
+        stat.execute("create table test(id int)");
+        stat.execute("create table test2(id int)");
+        stat.execute("insert into test select x from system_range(1, 10000)");
+        conn.close();
+        
+        ResultSet rs;
+        String plan;
+        
+        conn2 = getConnection(url);
+        stat2 = conn2.createStatement();
+        rs = stat2.executeQuery("explain analyze select count(*) from test");
+        rs.next();
+        plan = rs.getString(1);
+        assertTrue(plan, plan.indexOf("reads:") < 0);
+
+        conn = getConnection(url);
+        stat = conn.createStatement();
+        conn.setAutoCommit(false);
+        stat.execute("insert into test select x from system_range(1, 1000)");
+        rs = stat.executeQuery("select count(*) from test");
+        rs.next();
+        assertEquals(11000, rs.getInt(1));
+        
+        // not yet committed
+        rs = stat2.executeQuery("explain analyze select count(*) from test");
+        rs.next();
+        plan = rs.getString(1);
+        // transaction log is small, so no need to read the table
+        assertTrue(plan, plan.indexOf("reads:") < 0);
+        rs = stat2.executeQuery("select count(*) from test");
+        rs.next();
+        assertEquals(10000, rs.getInt(1));
+        
+        stat.execute("insert into test2 select x from system_range(1, 11000)");
+        rs = stat2.executeQuery("explain analyze select count(*) from test");
+        rs.next();
+        plan = rs.getString(1);
+        // transaction log is larger than the table, so read the table
+        assertTrue(plan, plan.indexOf("reads:") >= 0);
+        rs = stat2.executeQuery("select count(*) from test");
+        rs.next();
+        assertEquals(10000, rs.getInt(1));
+        
+        conn2.close();
+        conn.close();
     }
     
     private void testMinMaxWithNull() throws Exception {

h2/src/test/org/h2/test/store/TestTransactionStore.java

@@ -46,6 +46,7 @@ public class TestTransactionStore extends TestBase {
     @Override
     public void test() throws Exception {
         FileUtils.createDirectories(getBaseDir());
+        testCountWithOpenTransactions();
         testConcurrentUpdate();
         testRepeatedChange();
         testTransactionAge();
@@ -60,6 +61,39 @@ public class TestTransactionStore extends TestBase {
         testCompareWithPostgreSQL();
     }
     
+    private void testCountWithOpenTransactions() {
+        MVStore s;
+        TransactionStore ts;
+        s = MVStore.open(null);
+        ts = new TransactionStore(s);
+
+        Transaction tx1 = ts.begin();
+        TransactionMap<Integer, Integer> map1 = tx1.openMap("data");
+        int size = 150;
+        for (int i = 0; i < size; i++) {
+            map1.put(i, i * 10);
+        }
+        tx1.commit();
+        tx1 = ts.begin();
+        map1 = tx1.openMap("data");
+
+        Transaction tx2 = ts.begin();
+        TransactionMap<Integer, Integer> map2 = tx2.openMap("data");
+        
+        Random r = new Random(1);
+        for (int i = 0; i < size * 3; i++) {
+            assertEquals("op: " + i, size, (int) map1.sizeAsLong());
+            // keep the first 10%, and add 10%
+            int k = size / 10 + r.nextInt(size);
+            if (r.nextBoolean()) {
+                map2.remove(k);
+            } else {
+                map2.put(k, i);
+            }
+        }
+        s.close();
+    }
+    
     private void testConcurrentUpdate() {
         MVStore s;
         TransactionStore ts;