MVStore: table engine

h2/src/main/org/h2/command/ddl/Analyze.java

@@ -13,7 +13,6 @@ import org.h2.engine.Right;
 import org.h2.engine.Session;
 import org.h2.result.ResultInterface;
 import org.h2.table.Column;
-import org.h2.table.RegularTable;
 import org.h2.table.Table;
 import org.h2.util.StatementBuilder;
 import org.h2.value.Value;
@@ -53,7 +52,7 @@ public class Analyze extends DefineCommand {
      * @param manual whether the command was called by the user
      */
     public static void analyzeTable(Session session, Table table, int sample, boolean manual) {
-        if (!(table instanceof RegularTable) || table.isHidden() || session == null) {
+        if (!(table.getTableType().equals(Table.TABLE)) || table.isHidden() || session == null) {
             return;
         }
         if (!manual) {

h2/src/main/org/h2/command/dml/BackupCommand.java

@@ -21,6 +21,7 @@ import org.h2.engine.Database;
 import org.h2.engine.Session;
 import org.h2.expression.Expression;
 import org.h2.message.DbException;
+import org.h2.mvstore.db.MVTableEngine;
 import org.h2.result.ResultInterface;
 import org.h2.store.FileLister;
 import org.h2.store.PageStore;
@@ -56,6 +57,7 @@ public class BackupCommand extends Prepared {
             throw DbException.get(ErrorCode.DATABASE_IS_NOT_PERSISTENT);
         }
         try {
+            MVTableEngine.flush(db);
             String name = db.getName();
             name = FileUtils.getName(name);
             OutputStream zip = FileUtils.newOutputStream(fileName, false);
@@ -75,6 +77,9 @@ public class BackupCommand extends Prepared {
                     if (n.endsWith(Constants.SUFFIX_LOB_FILE)) {
                         backupFile(out, base, n);
                     }
+                    if (n.endsWith(Constants.SUFFIX_MV_FILE)) {
+                        backupFile(out, base, n);
+                    }
                 }
             }
             out.close();

h2/src/main/org/h2/mvstore/MVStore.java

@@ -603,8 +603,8 @@ public class MVStore {
 
     /**
      * Commit all changes and persist them to disk. This method does nothing if
-     * there are no unsaved changes, otherwise it stores the data and increments
-     * the current version.
+     * there are no unsaved changes, otherwise it increments the current version
+     * and stores the data (for file based stores).
      *
      * @return the new version (incremented if there were changes)
      */
@@ -619,6 +619,10 @@ public class MVStore {
         long version = incrementVersion();
         long time = getTime();
 
+        if (file == null) {
+            return version;
+        }
+
         // the last chunk was not completely correct in the last store()
         // this needs to be updated now (it's better not to update right after
         // storing, because that would modify the meta map again)

h2/src/main/org/h2/mvstore/db/MVDelegateIndex.java

@@ -29,7 +29,7 @@ public class MVDelegateIndex extends BaseIndex {
         IndexColumn[] cols = IndexColumn.wrap(new Column[] { table.getColumn(mainIndex.getMainIndexColumn())});
         this.initBaseIndex(table, id, name, cols, indexType);
         this.mainIndex = mainIndex;
-        if (!database.isPersistent() || id < 0) {
+        if (id < 0) {
             throw DbException.throwInternalError("" + name);
         }
     }

h2/src/main/org/h2/mvstore/db/MVPrimaryIndex.java

@@ -103,6 +103,7 @@ public class MVPrimaryIndex extends BaseIndex {
             throw e;
         }
         map.put(row.getKey(), array);
+        lastKey = Math.max(lastKey, row.getKey());
     }
 
     @Override
@@ -172,6 +173,9 @@ public class MVPrimaryIndex extends BaseIndex {
 
     @Override
     public void truncate(Session session) {
+        if (mvTable.getContainsLargeObject()) {
+            database.getLobStorage().removeAllForTable(table.getId());
+        }
         map.clear();
     }
 

h2/src/main/org/h2/mvstore/db/MVSecondaryIndex.java

@@ -26,6 +26,7 @@ import org.h2.table.IndexColumn;
 import org.h2.util.New;
 import org.h2.value.Value;
 import org.h2.value.ValueLong;
+import org.h2.value.ValueNull;
 
 /**
  * A table stored in a MVStore.
@@ -40,6 +41,9 @@ public class MVSecondaryIndex extends BaseIndex {
                 IndexColumn[] columns, IndexType indexType) {
         this.mvTable = table;
         initBaseIndex(table, id, indexName, columns, indexType);
+        if (!database.isStarting()) {
+            checkIndexColumnTypes(columns);
+        }
         // always store the row key in the map key,
         // even for unique indexes, as some of the index columns could be null
         keyColumns = columns.length + 1;
@@ -54,6 +58,15 @@ public class MVSecondaryIndex extends BaseIndex {
         map = table.getStore().openMap(getName() + "_" + getId(), map);
     }
 
+    private static void checkIndexColumnTypes(IndexColumn[] columns) {
+        for (IndexColumn c : columns) {
+            int type = c.column.getType();
+            if (type == Value.CLOB || type == Value.BLOB) {
+                throw DbException.get(ErrorCode.FEATURE_NOT_SUPPORTED_1, "Index on BLOB or CLOB column: " + c.column.getCreateSQL());
+            }
+        }
+    }
+
     @Override
     public void close(Session session) {
         // ok
@@ -158,10 +171,16 @@ public class MVSecondaryIndex extends BaseIndex {
 
     @Override
     public Cursor findFirstOrLast(Session session, boolean first) {
-        if (map.getSize() == 0) {
-            return new MVStoreCursor(session, Collections.<Value[]>emptyList().iterator(), null);
-        }
         Value[] key = first ? map.firstKey() : map.lastKey();
+        while (true) {
+            if (key == null) {
+                return new MVStoreCursor(session, Collections.<Value[]>emptyList().iterator(), null);
+            }
+            if (key[0] != ValueNull.INSTANCE) {
+                break;
+            }
+            key = first ? map.higherKey(key) : map.lowerKey(key);
+        }
         ArrayList<Value[]> list = New.arrayList();
         list.add(key);
         MVStoreCursor cursor = new MVStoreCursor(session, list.iterator(), null);

h2/src/main/org/h2/mvstore/db/MVTable.java

@@ -12,6 +12,7 @@ import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Set;
 import org.h2.api.DatabaseEventListener;
+import org.h2.command.ddl.Analyze;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
@@ -37,6 +38,7 @@ import org.h2.table.Table;
 import org.h2.table.TableBase;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
+import org.h2.value.DataType;
 import org.h2.value.Value;
 
 /**
@@ -53,6 +55,10 @@ public class MVTable extends TableBase {
     private volatile Session lockExclusive;
     private HashSet<Session> lockShared = New.hashSet();
     private final Trace traceLock;
+    private int changesSinceAnalyze;
+    private int nextAnalyze;
+    private boolean containsLargeObject;
+    private Column rowIdColumn;
 
     /**
      * True if one thread ever was waiting to lock this table. This is to avoid
@@ -63,9 +69,15 @@ public class MVTable extends TableBase {
 
     public MVTable(CreateTableData data, String storeName, MVStore store) {
         super(data);
+        nextAnalyze = database.getSettings().analyzeAuto;
         this.storeName = storeName;
         this.store = store;
         this.isHidden = data.isHidden;
+        for (Column col : getColumns()) {
+            if (DataType.isLargeObject(col.getType())) {
+                containsLargeObject = true;
+            }
+        }
         traceLock = database.getTrace(Trace.LOCK);
     }
 
@@ -475,7 +487,6 @@ public class MVTable extends TableBase {
             for (; i >= 0; i--) {
                 Index index = indexes.get(i);
                 index.remove(session, row);
-                checkRowCount(session, index, -1);
             }
             rowCount--;
         } catch (Throwable e) {
@@ -483,7 +494,6 @@ public class MVTable extends TableBase {
                 while (++i < indexes.size()) {
                     Index index = indexes.get(i);
                     index.add(session, row);
-                    checkRowCount(session, index, 0);
                 }
             } catch (DbException e2) {
                 // this could happen, for example on failure in the storage
@@ -506,6 +516,7 @@ public class MVTable extends TableBase {
             index.truncate(session);
         }
         rowCount = 0;
+        changesSinceAnalyze = 0;
         storeIfRequired();
     }
 
@@ -517,7 +528,6 @@ public class MVTable extends TableBase {
             for (int size = indexes.size(); i < size; i++) {
                 Index index = indexes.get(i);
                 index.add(session, row);
-                checkRowCount(session, index, 1);
             }
             rowCount++;
         } catch (Throwable e) {
@@ -525,7 +535,6 @@ public class MVTable extends TableBase {
                 while (--i >= 0) {
                     Index index = indexes.get(i);
                     index.remove(session, row);
-                    checkRowCount(session, index, 0);
                 }
             } catch (DbException e2) {
                 // this could happen, for example on failure in the storage
@@ -552,12 +561,17 @@ public class MVTable extends TableBase {
         storeIfRequired();
     }
 
-    private void checkRowCount(Session session, Index index, int offset) {
-        // TODO verify
-    }
-
     private void analyzeIfRequired(Session session) {
-        // TODO analyze
+        if (nextAnalyze == 0 || nextAnalyze > changesSinceAnalyze++) {
+            return;
+        }
+        changesSinceAnalyze = 0;
+        int n = 2 * nextAnalyze;
+        if (n > 0) {
+            nextAnalyze = n;
+        }
+        int rows = session.getDatabase().getSettings().analyzeSample;
+        Analyze.analyzeTable(session, this, rows, false);
     }
 
     @Override
@@ -590,6 +604,10 @@ public class MVTable extends TableBase {
         return lastModificationId;
     }
 
+    public boolean getContainsLargeObject() {
+        return containsLargeObject;
+    }
+
     @Override
     public boolean isDeterministic() {
         return true;
@@ -606,6 +624,12 @@ public class MVTable extends TableBase {
     }
 
     public void removeChildrenAndResources(Session session) {
+        if (containsLargeObject) {
+            // unfortunately, the data is gone on rollback
+            truncate(session);
+            database.getLobStorage().removeAllForTable(getId());
+            database.lockMeta(session);
+        }
         super.removeChildrenAndResources(session);
         // go backwards because database.removeIndex will call table.removeIndex
         while (indexes.size() > 1) {
@@ -658,4 +682,12 @@ public class MVTable extends TableBase {
         return store;
     }
 
+    public Column getRowIdColumn() {
+        if (rowIdColumn == null) {
+            rowIdColumn = new Column(Column.ROWID, Value.LONG);
+            rowIdColumn.setTable(this, -1);
+        }
+        return rowIdColumn;
+    }
+
 }

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -28,6 +28,21 @@ public class MVTableEngine implements TableEngine {
 
     static final Map<String, Store> STORES = new WeakHashMap<String, Store>();
 
+    public static void flush(Database db) {
+        String storeName = db.getDatabasePath();
+        if (storeName == null) {
+            return;
+        }
+        synchronized (STORES) {
+            Store store = STORES.get(storeName);
+            if (store == null) {
+                return;
+            }
+            // TODO this stores uncommitted transactions as well
+            store.store.store();
+        }
+    }
+
     @Override
     public TableBase createTable(CreateTableData data) {
         Database db = data.session.getDatabase();
@@ -61,11 +76,13 @@ public class MVTableEngine implements TableEngine {
     static void closeTable(String storeName, MVTable table) {
         synchronized (STORES) {
             Store store = STORES.get(storeName);
-            store.openTables.remove(table);
-            if (store.openTables.size() == 0) {
-                store.store.store();
-                store.store.close();
-                STORES.remove(storeName);
+            if (store != null) {
+                store.openTables.remove(table);
+                if (store.openTables.size() == 0) {
+                    store.store.store();
+                    store.store.close();
+                    STORES.remove(storeName);
+                }
             }
         }
     }

h2/src/test/org/h2/test/TestAll.java

@@ -582,6 +582,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestLargeBlob().runTest(this);
         new TestLinkedTable().runTest(this);
         new TestListener().runTest(this);
+        // verify
         new TestLob().runTest(this);
         new TestMemoryUsage().runTest(this);
         new TestMultiConn().runTest(this);
@@ -596,8 +597,10 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestReadOnly().runTest(this);
         new TestRecursiveQueries().runTest(this);
         new TestRights().runTest(this);
+        // verify
         new TestRunscript().runTest(this);
         new TestSQLInjection().runTest(this);
+        // verify
         new TestSessionsLocks().runTest(this);
         new TestSelectCountNonNullColumn().runTest(this);
         new TestSequence().runTest(this);
@@ -706,6 +709,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         new TestFileSystem().runTest(this);
         new TestIntArray().runTest(this);
         new TestIntIntHashMap().runTest(this);
+        // verify
         new TestJmx().runTest(this);
         new TestMathUtils().runTest(this);
         new TestModifyOnWrite().runTest(this);

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -8,6 +8,7 @@ package org.h2.test.store;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.Statement;
+import java.util.Random;
 import org.h2.mvstore.db.MVTableEngine;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
@@ -37,18 +38,53 @@ public class TestMVTableEngine extends TestBase {
         deleteDb("cases");
         Connection conn = getConnection("cases");
         Statement stat = conn.createStatement();
-
-        conn = getConnection("cases");
-        stat = conn.createStatement();
-        stat.execute("set max_operation_memory 1");
-        stat.execute("create table test(id int)");
-        stat.execute("insert into test values(1), (2)");
-        stat.execute("create index idx on test(id)");
-        conn.setAutoCommit(false);
-        stat.execute("update test set id = id where id=2");
-        stat.execute("update test set id = id");
-        conn.rollback();
-
+        ResultSet rs;
+
+        Random random = new Random(1);
+        int len = getSize(50, 500);
+        for (int i = 0; i < len; i++) {
+            stat.execute("drop table if exists test");
+            stat.execute("create table test(x int)");
+            if (random.nextBoolean()) {
+                int count = random.nextBoolean() ? 1 : 1 + random.nextInt(len);
+                if (count > 0) {
+                    stat.execute("insert into test select null from system_range(1, " + count + ")");
+                }
+            }
+            int maxExpected = -1;
+            int minExpected = -1;
+            if (random.nextInt(10) != 1) {
+                minExpected = 1;
+                maxExpected = 1 + random.nextInt(len);
+                stat.execute("insert into test select x from system_range(1, " + maxExpected + ")");
+            }
+            String sql = "create index idx on test(x";
+            if (random.nextBoolean()) {
+                sql += " desc";
+            }
+            if (random.nextBoolean()) {
+                if (random.nextBoolean()) {
+                    sql += " nulls first";
+                } else {
+                    sql += " nulls last";
+                }
+            }
+            sql += ")";
+            stat.execute(sql);
+
+            rs = stat.executeQuery("select min(x), max(x) from test");
+            rs.next();
+            int min = rs.getInt(1);
+            if (rs.wasNull()) {
+                min = -1;
+            }
+            int max = rs.getInt(2);
+            if (rs.wasNull()) {
+                max = -1;
+            }
+            assertEquals(minExpected, min);
+            assertEquals(maxExpected, max);
+        }
         conn.close();
 
     }