MVStore table engine (WIP)

h2/src/main/org/h2/engine/Session.java

@@ -23,8 +23,8 @@ import org.h2.jdbc.JdbcConnection;
 import org.h2.message.DbException;
 import org.h2.message.Trace;
 import org.h2.message.TraceSystem;
-import org.h2.mvstore.db.TransactionStore2;
-import org.h2.mvstore.db.TransactionStore2.Transaction;
+import org.h2.mvstore.db.TransactionStore;
+import org.h2.mvstore.db.TransactionStore.Transaction;
 import org.h2.result.ResultInterface;
 import org.h2.result.Row;
 import org.h2.schema.Schema;
@@ -1257,7 +1257,7 @@ public class Session extends SessionWithState {
      * @param store the store
      * @return the transaction
      */
-    public Transaction getTransaction(TransactionStore2 store) {
+    public Transaction getTransaction(TransactionStore store) {
         if (transaction == null) {
             transaction = store.begin();
             startStatement = -1;

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -76,6 +76,11 @@ public class DataUtils {
      */
     private static final byte[] EMPTY_BYTES = {};
 
+    /**
+     * The maximum byte to grow a buffer at a time.
+     */
+    private static final int MAX_GROW = 16 * 1024 * 1024;
+
     /**
      * Get the length of the variable size int.
      *
@@ -739,8 +744,7 @@ public class DataUtils {
     private static ByteBuffer grow(ByteBuffer buff, int len) {
         len = buff.remaining() + len;
         int capacity = buff.capacity();
-        // grow at most 1 MB at a time
-        len = Math.max(len, Math.min(capacity + 1024 * 1024, capacity * 2));
+        len = Math.max(len, Math.min(capacity + MAX_GROW, capacity * 2));
         ByteBuffer temp = ByteBuffer.allocate(len);
         buff.flip();
         temp.put(buff);

h2/src/main/org/h2/mvstore/db/MVDelegateIndex.java

@@ -35,9 +35,7 @@ public class MVDelegateIndex extends BaseIndex {
         }
     }
 
-    static int count;
     public void add(Session session, Row row) {
-        count++;
         // nothing to do
     }
 

h2/src/main/org/h2/mvstore/db/MVPrimaryIndex.java

@@ -18,8 +18,8 @@ import org.h2.index.Cursor;
 import org.h2.index.IndexType;
 import org.h2.message.DbException;
 import org.h2.mvstore.MVMap;
-import org.h2.mvstore.db.TransactionStore2.Transaction;
-import org.h2.mvstore.db.TransactionStore2.TransactionMap;
+import org.h2.mvstore.db.TransactionStore.Transaction;
+import org.h2.mvstore.db.TransactionStore.TransactionMap;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
 import org.h2.result.SortOrder;
@@ -59,7 +59,6 @@ public class MVPrimaryIndex extends BaseIndex {
                 valueType(valueType);
         dataMap = mvTable.getTransaction(null).openMap(mapName, mapBuilder);
         Value k = dataMap.lastKey();
-        dataMap.getTransaction().commit();
         lastKey = k == null ? 0 : k.getLong();
     }
 
@@ -73,7 +72,6 @@ public class MVPrimaryIndex extends BaseIndex {
         rename(newName + "_DATA");
         String newMapName = newName + "_DATA_" + getId();
         map.renameMap(newMapName);
-        map.getTransaction().commit();
         mapName = newMapName;
     }
 
@@ -233,10 +231,7 @@ public class MVPrimaryIndex extends BaseIndex {
 
     @Override
     public long getRowCountApproximation() {
-        TransactionMap<Value, Value> map = getMap(null);
-        long size = map.getSize();
-        map.getTransaction().commit();
-        return size;
+        return getMap(null).getSize();
     }
 
     public long getDiskSpaceUsed() {

h2/src/main/org/h2/mvstore/db/MVSecondaryIndex.java

@@ -17,8 +17,8 @@ import org.h2.index.Cursor;
 import org.h2.index.IndexType;
 import org.h2.message.DbException;
 import org.h2.mvstore.MVMap;
-import org.h2.mvstore.db.TransactionStore2.Transaction;
-import org.h2.mvstore.db.TransactionStore2.TransactionMap;
+import org.h2.mvstore.db.TransactionStore.Transaction;
+import org.h2.mvstore.db.TransactionStore.TransactionMap;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
 import org.h2.result.SortOrder;
@@ -87,7 +87,6 @@ public class MVSecondaryIndex extends BaseIndex {
         TransactionMap<Value, Value> map = getMap(null);
         String newMapName = newName + "_" + getId();
         map.renameMap(newMapName);
-        map.getTransaction().commit();
         mapName = newMapName;
         super.rename(newName);
     }
@@ -218,10 +217,7 @@ public class MVSecondaryIndex extends BaseIndex {
 
     @Override
     public boolean needRebuild() {
-        TransactionMap<Value, Value> map = getMap(null);
-        boolean result = map.getSize() == 0;
-        map.getTransaction().commit();
-        return result;
+        return getMap(null).getSize() == 0;
     }
 
     @Override
@@ -232,10 +228,7 @@ public class MVSecondaryIndex extends BaseIndex {
 
     @Override
     public long getRowCountApproximation() {
-        TransactionMap<Value, Value> map = getMap(null);
-        long size = map.getSize();
-        map.getTransaction().commit();
-        return size;
+        return getMap(null).getSize();
     }
 
     public long getDiskSpaceUsed() {

h2/src/main/org/h2/mvstore/db/MVTable.java

@@ -28,7 +28,7 @@ import org.h2.index.IndexType;
 import org.h2.index.MultiVersionIndex;
 import org.h2.message.DbException;
 import org.h2.message.Trace;
-import org.h2.mvstore.db.TransactionStore2.Transaction;
+import org.h2.mvstore.db.TransactionStore.Transaction;
 import org.h2.result.Row;
 import org.h2.result.SortOrder;
 import org.h2.schema.SchemaObject;
@@ -48,7 +48,7 @@ import org.h2.value.Value;
 public class MVTable extends TableBase {
 
     private final String storeName;
-    private final TransactionStore2 store;
+    private final TransactionStore store;
     private MVPrimaryIndex primaryIndex;
     private ArrayList<Index> indexes = New.arrayList();
     private long lastModificationId;
@@ -68,7 +68,7 @@ public class MVTable extends TableBase {
      */
     private boolean waitForLock;
 
-    public MVTable(CreateTableData data, String storeName, TransactionStore2 store) {
+    public MVTable(CreateTableData data, String storeName, TransactionStore store) {
         super(data);
         nextAnalyze = database.getSettings().analyzeAuto;
         this.storeName = storeName;
@@ -485,26 +485,16 @@ public class MVTable extends TableBase {
     @Override
     public void removeRow(Session session, Row row) {
         lastModificationId = database.getNextModificationDataId();
-        int i = indexes.size() - 1;
+        Transaction t = getTransaction(session);
+        long savepoint = t.setSavepoint();
         try {
-            for (; i >= 0; i--) {
+            for (int i = indexes.size() - 1; i >= 0; i--) {
                 Index index = indexes.get(i);
                 index.remove(session, row);
             }
             rowCount--;
         } catch (Throwable e) {
-            try {
-                while (++i < indexes.size()) {
-                    Index index = indexes.get(i);
-                    index.add(session, row);
-                }
-            } catch (DbException e2) {
-                // this could happen, for example on failure in the storage
-                // but if that is not the case it means there is something wrong
-                // with the database
-                trace.error(e2, "could not undo operation");
-                throw e2;
-            }
+            t.rollbackToSavepoint(savepoint);
             throw DbException.convert(e);
         }
         analyzeIfRequired(session);
@@ -524,26 +514,16 @@ public class MVTable extends TableBase {
     @Override
     public void addRow(Session session, Row row) {
         lastModificationId = database.getNextModificationDataId();
-        int i = 0;
+        Transaction t = getTransaction(session);
+        long savepoint = t.setSavepoint();
         try {
-            for (int size = indexes.size(); i < size; i++) {
+            for (int i = 0, size = indexes.size(); i < size; i++) {
                 Index index = indexes.get(i);
                 index.add(session, row);
             }
             rowCount++;
         } catch (Throwable e) {
-            try {
-                while (--i >= 0) {
-                    Index index = indexes.get(i);
-                    index.remove(session, row);
-                }
-            } catch (DbException e2) {
-                // this could happen, for example on failure in the storage
-                // but if that is not the case it means there is something wrong
-                // with the database
-                trace.error(e2, "could not undo operation");
-                throw e2;
-            }
+            t.rollbackToSavepoint(savepoint);
             DbException de = DbException.convert(e);
             if (de.getErrorCode() == ErrorCode.DUPLICATE_KEY_1) {
                 for (int j = 0; j < indexes.size(); j++) {
@@ -572,7 +552,7 @@ public class MVTable extends TableBase {
         }
         int rows = session.getDatabase().getSettings().analyzeSample;
 int test;        
-       // Analyze.analyzeTable(session, this, rows, false);
+//        Analyze.analyzeTable(session, this, rows, false);
     }
 
     @Override
@@ -687,7 +667,7 @@ int test;
         return session.getTransaction(store);
     }
 
-    public TransactionStore2 getStore() {
+    public TransactionStore getStore() {
         return store;
     }
 

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -42,7 +42,6 @@ public class MVTableEngine implements TableEngine {
             if (store == null) {
                 return;
             }
-            // TODO this stores uncommitted transactions as well
             store(store.getStore());
         }
     }
@@ -115,8 +114,11 @@ public class MVTableEngine implements TableEngine {
      * @param store the store
      */
     static void store(MVStore store) {
-        store.compact(50);
-        store.store();
+        if (!store.isReadOnly()) {
+            store.commit();
+            store.compact(50);
+            store.store();
+        }
     }
 
     /**
@@ -142,12 +144,12 @@ public class MVTableEngine implements TableEngine {
         /**
          * The transaction store.
          */
-        private final TransactionStore2 transactionStore;
+        private final TransactionStore transactionStore;
 
         public Store(Database db, MVStore store) {
             this.db = db;
             this.store = store;
-            this.transactionStore = new TransactionStore2(store,
+            this.transactionStore = new TransactionStore(store,
                     new ValueDataType(null, null, null));
         }
 
@@ -155,7 +157,7 @@ public class MVTableEngine implements TableEngine {
             return store;
         }
 
-        public TransactionStore2 getTransactionStore() {
+        public TransactionStore getTransactionStore() {
             return transactionStore;
         }
 

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -39,22 +39,24 @@ public class TestMVTableEngine extends TestBase {
         testEncryption();
         testReadOnly();
         testReuseDiskSpace();
-        // testDataTypes();
+        testDataTypes();
         testLocking();
-        // testSimple();
+        testSimple();
     }
 
     private void testSpeed() throws Exception {
         String dbName;
         for (int i = 0; i < 10; i++) {
             dbName = "mvstore";
-//            dbName += ";LOCK_MODE=0";
+            dbName += ";LOCK_MODE=0";
+//            dbName += ";LOG=0";
             testSpeed(dbName);
 int tes;            
 //Profiler prof = new Profiler().startCollecting();
             dbName = "mvstore" +
                     ";DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
-//            dbName += ";LOCK_MODE=0";
+            dbName += ";LOCK_MODE=0";
+//            dbName += ";LOG=0";
             testSpeed(dbName);
 //System.out.println(prof.getTop(10));
         }
@@ -70,17 +72,51 @@ int tes;
         String password = getPassword();
         conn = DriverManager.getConnection(url, user, password);
         stat = conn.createStatement();
-        stat.execute("create table test(id int primary key, name varchar(255))");
+        long time = System.currentTimeMillis();
+//        stat.execute(
+//                "create table test(id int primary key, name varchar(255))" +
+//                "as select x, 'Hello World' from system_range(1, 200000)");
+        stat.execute("create table test(id int primary key, name varchar)");
         PreparedStatement prep = conn
                 .prepareStatement("insert into test values(?, ?)");
-        prep.setString(2, "Hello World xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
-        long time = System.currentTimeMillis();
-        for (int i = 0; i < 200000; i++) {
+        
+        // -mx4g
+        
+        // 10000 / 8000
+        // 1229 mvstore;LOCK_MODE=0
+        // 1455 mvstore;LOCK_MODE=0
+        // 19 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+        // 659 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+
+        // 1000 / 80000
+        // 1383 mvstore;LOCK_MODE=0
+        // 1618 mvstore;LOCK_MODE=0
+        // 244 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+        // 965 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+        
+        // 100 / 800000
+        // 2061 mvstore;LOCK_MODE=0
+        // 2281 mvstore;LOCK_MODE=0
+        // 1414 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+        // 2647 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+
+        // 10 / 8000000
+        // 11453 mvstore;LOCK_MODE=0
+        // 11720 mvstore;LOCK_MODE=0
+        // 13605 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+        // 25172 mvstore;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine;LOCK_MODE=0
+        
+        prep.setString(2, new String(new char[10]).replace((char) 0, 'x'));
+        for (int i = 0; i < 8000000; i++) {
+            
             prep.setInt(1, i);
             prep.execute();
         }
         System.out.println((System.currentTimeMillis() - time) + " " + dbName);
+//Profiler prof = new Profiler().startCollecting();
         conn.close();
+//System.out.println(prof.getTop(10));        
+        System.out.println((System.currentTimeMillis() - time) + " " + dbName);
     }
 
     private void testEncryption() throws Exception {

h2/src/test/org/h2/test/store/TestTransactionStore.java

@@ -16,9 +16,9 @@ import java.util.List;
 import java.util.Random;
 
 import org.h2.mvstore.MVStore;
-import org.h2.mvstore.db.TransactionStore2;
-import org.h2.mvstore.db.TransactionStore2.Transaction;
-import org.h2.mvstore.db.TransactionStore2.TransactionMap;
+import org.h2.mvstore.db.TransactionStore;
+import org.h2.mvstore.db.TransactionStore.Transaction;
+import org.h2.mvstore.db.TransactionStore.TransactionMap;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
 import org.h2.util.New;
@@ -60,7 +60,7 @@ public class TestTransactionStore extends TestBase {
      */
     private void testMultiStatement() {
         MVStore s = MVStore.open(null);
-        TransactionStore2 ts = new TransactionStore2(s);
+        TransactionStore ts = new TransactionStore(s);
         Transaction tx;
         TransactionMap<String, String> m;
         long startUpdate;
@@ -141,14 +141,14 @@ public class TestTransactionStore extends TestBase {
         FileUtils.delete(fileName);
 
         MVStore s;
-        TransactionStore2 ts;
+        TransactionStore ts;
         Transaction tx;
         Transaction txOld;
         TransactionMap<String, String> m;
         List<Transaction> list;
 
         s = MVStore.open(fileName);
-        ts = new TransactionStore2(s);
+        ts = new TransactionStore(s);
         tx = ts.begin();
         assertEquals(null, tx.getName());
         tx.setName("first transaction");
@@ -166,7 +166,7 @@ public class TestTransactionStore extends TestBase {
         s.close();
 
         s = MVStore.open(fileName);
-        ts = new TransactionStore2(s);
+        ts = new TransactionStore(s);
         tx = ts.begin();
         assertEquals(1, tx.getId());
         m = tx.openMap("test");
@@ -184,7 +184,7 @@ public class TestTransactionStore extends TestBase {
         s.close();
 
         s = MVStore.open(fileName);
-        ts = new TransactionStore2(s);
+        ts = new TransactionStore(s);
         tx = ts.begin();
         m = tx.openMap("test");
         // TransactionStore was not closed, so we lost some ids
@@ -209,7 +209,7 @@ public class TestTransactionStore extends TestBase {
 
     private void testSavepoint() throws Exception {
         MVStore s = MVStore.open(null);
-        TransactionStore2 ts = new TransactionStore2(s);
+        TransactionStore ts = new TransactionStore(s);
         Transaction tx;
         TransactionMap<String, String> m;
 
@@ -262,7 +262,7 @@ public class TestTransactionStore extends TestBase {
                 "create table test(id int primary key, name varchar(255))");
 
         MVStore s = MVStore.open(null);
-        TransactionStore2 ts = new TransactionStore2(s);
+        TransactionStore ts = new TransactionStore(s);
         for (int i = 0; i < connectionCount; i++) {
             Statement stat = statements.get(i);
             // 100 ms to avoid blocking (the test is single threaded)
@@ -395,7 +395,7 @@ public class TestTransactionStore extends TestBase {
     private void testConcurrentTransactionsReadCommitted() {
         MVStore s = MVStore.open(null);
 
-        TransactionStore2 ts = new TransactionStore2(s);
+        TransactionStore ts = new TransactionStore(s);
 
         Transaction tx1, tx2;
         TransactionMap<String, String> m1, m2;
@@ -467,7 +467,7 @@ public class TestTransactionStore extends TestBase {
     private void testSingleConnection() {
         MVStore s = MVStore.open(null);
 
-        TransactionStore2 ts = new TransactionStore2(s);
+        TransactionStore ts = new TransactionStore(s);
 
         Transaction tx;
         TransactionMap<String, String> m;