Page store: opening a large database was slow if it was not closed before.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,8 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>Page store: new write and read counters in the meta data table. Use 
+<ul><li>Page store: opening a large database was slow if it was not closed before.
+</li><li>Page store: new write and read counters in the meta data table. Use 
     SELECT * FROM INFORMATION_SCHEMA.SETTINGS WHERE NAME IN(
     'info.FILE_WRITE_TOTAL', 'info.FILE_WRITE', 'info.FILE_READ', 
     'info.CACHE_MAX_SIZE', 'info.CACHE_SIZE')

h2/src/main/org/h2/api/DatabaseEventListener.java

@@ -18,7 +18,7 @@ import java.util.EventListener;
 public interface DatabaseEventListener extends EventListener {
 
     /**
-     * This state is used when scanning the data or index file.
+     * This state is used when scanning the database file.
      */
     int STATE_SCAN_FILE = 0;
 

h2/src/main/org/h2/index/PageBtree.java

@@ -89,7 +89,6 @@ public abstract class PageBtree extends Page {
      *
      * @param rowCount the stored row count
      */
-    // TODO remove
     abstract void setRowCountStored(int rowCount) throws SQLException;
 
     /**

h2/src/main/org/h2/index/PageBtreeIndex.java

@@ -18,6 +18,7 @@ import org.h2.store.PageStore;
 import org.h2.table.Column;
 import org.h2.table.IndexColumn;
 import org.h2.table.TableData;
+import org.h2.util.MathUtils;
 import org.h2.value.Value;
 import org.h2.value.ValueLob;
 import org.h2.value.ValueNull;
@@ -100,6 +101,7 @@ public class PageBtreeIndex extends PageIndex {
             store.update(newRoot);
             root = newRoot;
         }
+        invalidateRowCount();
         rowCount++;
     }
 
@@ -217,6 +219,7 @@ public class PageBtreeIndex extends PageIndex {
         } else {
             PageBtree root = getPage(rootPageId);
             root.remove(row);
+            invalidateRowCount();
             rowCount--;
         }
     }
@@ -275,14 +278,16 @@ public class PageBtreeIndex extends PageIndex {
     }
 
     public long getRowCount(Session session) {
-        return tableData.getRowCount(session);
+        return rowCount;
     }
 
-    public void close(Session session) {
+    public void close(Session session) throws SQLException {
         if (trace.isDebugEnabled()) {
             trace.debug("close");
         }
-        // TODO write the row count
+        // can not close the index because it might get used afterwards,
+        // for example after running recovery
+        writeRowCount();
     }
 
     /**
@@ -363,4 +368,14 @@ public class PageBtreeIndex extends PageIndex {
         store.addIndex(this);
     }
 
+    private void invalidateRowCount() throws SQLException {
+        PageBtree root = getPage(rootPageId);
+        root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
+    }
+
+    public void writeRowCount() throws SQLException {
+        PageBtree root = getPage(rootPageId);
+        root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -7,6 +7,7 @@
 package org.h2.index;
 
 import java.sql.SQLException;
+import org.h2.api.DatabaseEventListener;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Session;
@@ -41,7 +42,7 @@ public class PageBtreeNode extends PageBtree {
      */
     private int[] childPageIds;
 
-    // private int rowCountStored = UNKNOWN_ROWCOUNT;
+    private int rowCountStored = UNKNOWN_ROWCOUNT;
 
     private int rowCount = UNKNOWN_ROWCOUNT;
 
@@ -93,11 +94,8 @@ public class PageBtreeNode extends PageBtree {
                     "page:" + getPos() + " expected index:" + index.getId() +
                     "got:" + indexId);
         }
-        rowCount = data.readInt();
+        rowCount = rowCountStored = data.readInt();
         entryCount = data.readShortInt();
-        if (!PageStore.STORE_BTREE_ROWCOUNT) {
-            rowCount = UNKNOWN_ROWCOUNT;
-        }
         childPageIds = new int[entryCount + 1];
         childPageIds[entryCount] = data.readInt();
         rows = PageStore.newSearchRows(entryCount);
@@ -213,10 +211,18 @@ public class PageBtreeNode extends PageBtree {
         return -1;
     }
 
-    private void updateRowCount(int offset) {
-        if (PageStore.STORE_BTREE_ROWCOUNT) {
+    private void updateRowCount(int offset) throws SQLException {
+        if (rowCount != UNKNOWN_ROWCOUNT) {
             rowCount += offset;
         }
+        if (rowCountStored != UNKNOWN_ROWCOUNT) {
+            rowCountStored = UNKNOWN_ROWCOUNT;
+            index.getPageStore().logUndo(this, data);
+            if (written) {
+                writeHead();
+            }
+            index.getPageStore().update(this);
+        }
     }
 
     PageBtree split(int splitPoint) throws SQLException {
@@ -350,14 +356,23 @@ public class PageBtreeNode extends PageBtree {
             for (int child : childPageIds) {
                 PageBtree page = index.getPage(child);
                 count += page.getRowCount();
+                index.getDatabase().setProgress(DatabaseEventListener.STATE_SCAN_FILE, index.getName(), count, Integer.MAX_VALUE);
             }
             rowCount = count;
         }
         return rowCount;
     }
 
-    void setRowCountStored(int rowCount) {
+    void setRowCountStored(int rowCount) throws SQLException {
         this.rowCount = rowCount;
+        if (rowCountStored != rowCount) {
+            rowCountStored = rowCount;
+            index.getPageStore().logUndo(this, data);
+            if (written) {
+                writeHead();
+            }
+            index.getPageStore().update(this);
+        }
     }
 
     private void check() {
@@ -384,7 +399,7 @@ public class PageBtreeNode extends PageBtree {
         data.writeShortInt(0);
         data.writeInt(parentPageId);
         data.writeVarInt(index.getId());
-        data.writeInt(rowCount);
+        data.writeInt(rowCountStored);
         data.writeShortInt(entryCount);
     }
 
@@ -510,6 +525,8 @@ public class PageBtreeNode extends PageBtree {
         store.logUndo(this, data);
         PageBtreeNode p2 = PageBtreeNode.create(index, newPos, parentPageId);
         readAllRows();
+        p2.rowCountStored = rowCountStored;
+        p2.rowCount = rowCount;
         p2.childPageIds = childPageIds;
         p2.rows = rows;
         p2.entryCount = entryCount;

h2/src/main/org/h2/index/PageDataIndex.java

@@ -307,11 +307,6 @@ public class PageDataIndex extends PageIndex implements RowIndex {
         store.logAddOrRemoveRow(session, tableData.getId(), row, false);
     }
 
-    private void invalidateRowCount() throws SQLException {
-        PageData root = getPage(rootPageId, 0);
-        root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
-    }
-
     public void remove(Session session) throws SQLException {
         if (trace.isDebugEnabled()) {
             trace.debug(this + " remove");
@@ -424,8 +419,7 @@ public class PageDataIndex extends PageIndex implements RowIndex {
         }
         // can not close the index because it might get used afterwards,
         // for example after running recovery
-        PageData root = getPage(rootPageId, 0);
-        root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
+        writeRowCount();
     }
 
     Iterator<Row> getDelta() {
@@ -484,4 +478,14 @@ public class PageDataIndex extends PageIndex implements RowIndex {
         return getName();
     }
 
+    private void invalidateRowCount() throws SQLException {
+        PageData root = getPage(rootPageId, 0);
+        root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
+    }
+
+    public void writeRowCount() throws SQLException {
+        PageData root = getPage(rootPageId, 0);
+        root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
+    }
+
 }

h2/src/main/org/h2/index/PageDataNode.java

@@ -8,6 +8,7 @@ package org.h2.index;
 
 import java.sql.SQLException;
 import java.util.Arrays;
+import org.h2.api.DatabaseEventListener;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Session;
@@ -292,6 +293,7 @@ public class PageDataNode extends PageData {
                     throw Message.throwInternalError("Page it its own child: " + getPos());
                 }
                 count += page.getRowCount();
+                index.getDatabase().setProgress(DatabaseEventListener.STATE_SCAN_FILE, index.getTable() + "." + index.getName(), count, Integer.MAX_VALUE);
             }
             rowCount = count;
         }

h2/src/main/org/h2/index/PageDelegateIndex.java

@@ -112,4 +112,8 @@ public class PageDelegateIndex extends PageIndex {
         return mainIndex.getRowCountApproximation();
     }
 
+    public void writeRowCount() {
+        // ignore
+    }
+
 }

h2/src/main/org/h2/index/PageIndex.java

@@ -6,6 +6,8 @@
  */
 package org.h2.index;
 
+import java.sql.SQLException;
+
 /**
  * A page store index.
  */
@@ -24,4 +26,9 @@ public abstract class PageIndex extends BaseIndex {
         return 0;
     }
 
+    /**
+     * Write back the row count if it has changed.
+     */
+    public abstract void writeRowCount() throws SQLException;
+
 }

h2/src/main/org/h2/store/PageStore.java

@@ -90,21 +90,18 @@ public class PageStore implements CacheWriter {
     // TODO optimization: check if calling Data.getValueLen slows things down
     // TODO order pages so that searching for a key only seeks forward
     // TODO optimization: update: only log the key and changed values
-    // TODO maybe remove some parent pointers
     // TODO index creation: use less space (ordered, split at insertion point)
     // TODO detect circles in linked lists
     // (input stream, free list, extend pages...)
     // at runtime and recovery
     // synchronized correctly (on the index?)
     // TODO remove trace or use isDebugEnabled
-    // TODO recover tool: don't re-do uncommitted operations
     // TODO recover tool: support syntax to delete a row with a key
     // TODO don't store default values (store a special value)
     // TODO split files (1 GB max size)
     // TODO add a setting (that can be changed at runtime) to call fsync
     // and delay on each commit
     // TODO check for file size (exception if not exact size expected)
-    // TODO implement missing code for STORE_BTREE_ROWCOUNT (maybe enable)
     // TODO online backup using bsdiff
 
     // TODO when removing DiskFile:
@@ -132,11 +129,6 @@ public class PageStore implements CacheWriter {
      */
     public static final int PAGE_SIZE_DEFAULT = 2 * 1024;
 
-    /**
-     * Store the rowcount in b-tree indexes.
-     */
-    public static final boolean STORE_BTREE_ROWCOUNT = false;
-
     private static final int PAGE_ID_FREE_LIST_ROOT = 3;
     private static final int PAGE_ID_META_ROOT = 4;
 
@@ -188,7 +180,7 @@ public class PageStore implements CacheWriter {
     private TableData metaTable;
     private PageDataIndex metaIndex;
     private IntIntHashMap metaRootPageId = new IntIntHashMap();
-    private HashMap<Integer, Index> metaObjects = New.hashMap();
+    private HashMap<Integer, PageIndex> metaObjects = New.hashMap();
 
     /**
      * The map of reserved pages, to ensure index head pages
@@ -313,6 +305,12 @@ public class PageStore implements CacheWriter {
         }
     }
 
+    private void writeIndexRowCounts() throws SQLException {
+        for (PageIndex index: metaObjects.values()) {
+            index.writeRowCount();
+        }
+    }
+
     private void writeBack() throws SQLException {
         ObjectArray<CacheObject> list = cache.getAllChanged();
         CacheObject.sort(list);
@@ -332,6 +330,7 @@ public class PageStore implements CacheWriter {
         }
         synchronized (database) {
             database.checkPowerOff();
+            writeIndexRowCounts();
             writeBack();
             log.checkpoint();
             switchLog();
@@ -1034,10 +1033,13 @@ public class PageStore implements CacheWriter {
         }
         recoveryRunning = false;
         reservedPages = null;
+        writeIndexRowCounts();
         writeBack();
         // clear the cache because it contains pages with closed indexes
         cache.clear();
         freeLists.clear();
+        metaObjects.clear();
+        metaObjects.put(-1, metaIndex);
         if (setReadOnly) {
             database.setReadOnly(true);
         }
@@ -1201,12 +1203,9 @@ public class PageStore implements CacheWriter {
 
     private void removeMeta(int logPos, Row row) throws SQLException {
         int id = row.getValue(0).getInt();
-        Index index = metaObjects.get(id);
+        PageIndex index = metaObjects.get(id);
         int rootPageId = index.getRootPageId();
         index.getTable().removeIndex(index);
-        if (index instanceof MultiVersionIndex) {
-            index = ((MultiVersionIndex) index).getBaseIndex();
-        }
         if (index instanceof PageBtreeIndex) {
             if (index.isTemporary()) {
                 systemSession.removeLocalTempTableIndex(index);
@@ -1297,7 +1296,13 @@ public class PageStore implements CacheWriter {
             }
             meta = table.addIndex(session, "I" + id, id, cols, indexType, id, null);
         }
-        metaObjects.put(id, meta);
+        PageIndex index;
+        if (meta instanceof MultiVersionIndex) {
+            index = (PageIndex) ((MultiVersionIndex) meta).getBaseIndex();
+        } else {
+            index = (PageIndex) meta;
+        }
+        metaObjects.put(id, index);
     }
 
     /**

h2/src/main/org/h2/table/TableData.java

@@ -10,7 +10,6 @@ import java.sql.SQLException;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Set;
-
 import org.h2.api.DatabaseEventListener;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.constant.ErrorCode;
@@ -28,8 +27,8 @@ import org.h2.index.IndexType;
 import org.h2.index.MultiVersionIndex;
 import org.h2.index.NonUniqueHashIndex;
 import org.h2.index.PageBtreeIndex;
-import org.h2.index.PageDelegateIndex;
 import org.h2.index.PageDataIndex;
+import org.h2.index.PageDelegateIndex;
 import org.h2.index.RowIndex;
 import org.h2.index.ScanIndex;
 import org.h2.index.TreeIndex;
@@ -39,7 +38,6 @@ import org.h2.result.Row;
 import org.h2.result.SortOrder;
 import org.h2.schema.SchemaObject;
 import org.h2.store.DataPage;
-import org.h2.store.PageStore;
 import org.h2.store.Record;
 import org.h2.store.RecordReader;
 import org.h2.util.MathUtils;
@@ -156,12 +154,11 @@ public class TableData extends Table implements RecordReader {
 
     private void checkRowCount(Session session, Index index, int offset) {
         if (SysProperties.CHECK && !database.isMultiVersion()) {
-            if (database.isPageStoreEnabled() && !PageStore.STORE_BTREE_ROWCOUNT) {
-                return;
-            }
-            long rc = index.getRowCount(session);
-            if (rc != rowCount + offset) {
-                Message.throwInternalError("rowCount expected " + (rowCount + offset) + " got " + rc + " " + getName() + "." + index.getName());
+            if (!(index instanceof PageDelegateIndex)) {
+                long rc = index.getRowCount(session);
+                if (rc != rowCount + offset) {
+                    Message.throwInternalError("rowCount expected " + (rowCount + offset) + " got " + rc + " " + getName() + "." + index.getName());
+                }
             }
         }
     }

h2/src/test/org/h2/test/unit/TestPageStore.java

@@ -15,12 +15,15 @@ import java.sql.Statement;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeSet;
+import org.h2.api.DatabaseEventListener;
 import org.h2.test.TestBase;
 
 /**
  * Test the page store.
  */
-public class TestPageStore extends TestBase {
+public class TestPageStore extends TestBase implements DatabaseEventListener {
+
+    static StringBuilder eventBuffer = new StringBuilder();
 
     /**
      * Run just this test.
@@ -33,6 +36,8 @@ public class TestPageStore extends TestBase {
     }
 
     public void test() throws Exception {
+        testLargeDatabaseFastOpen();
+        testUniqueIndexReopen();
         testExistingOld();
         testLargeRows();
         testRecoverDropIndex();
@@ -45,6 +50,56 @@ public class TestPageStore extends TestBase {
         testFuzzOperations();
     }
 
+    private void testLargeDatabaseFastOpen() throws SQLException {
+        if (config.memory) {
+            return;
+        }
+        deleteDb("pageStore");
+        Connection conn;
+        String url = "pageStore";
+        conn = getConnection(url);
+        conn.createStatement().execute("CREATE TABLE TEST(ID INT PRIMARY KEY, NAME VARCHAR)");
+        conn.createStatement().execute("create unique index idx_test_name on test(name)");
+        conn.createStatement().execute("INSERT INTO TEST SELECT X, X || space(10) FROM SYSTEM_RANGE(1, 1000)");
+        conn.close();
+        conn = getConnection(url);
+        conn.createStatement().execute("DELETE FROM TEST WHERE ID=1");
+        conn.createStatement().execute("CHECKPOINT");
+        conn.createStatement().execute("SHUTDOWN IMMEDIATELY");
+        try {
+            conn.close();
+        } catch (SQLException e) {
+            // ignore
+        }
+        eventBuffer.setLength(0);
+        conn = getConnection(url + ";DATABASE_EVENT_LISTENER='" + getClass().getName() + "'");
+        assertEquals("init;opened;", eventBuffer.toString());
+        conn.close();
+    }
+
+    private void testUniqueIndexReopen() throws SQLException {
+        if (config.memory) {
+            return;
+        }
+        deleteDb("pageStore");
+        Connection conn;
+        String url = "pageStore";
+        conn = getConnection(url);
+        conn.createStatement().execute("CREATE TABLE test(ID INT PRIMARY KEY, NAME VARCHAR(255))");
+        conn.createStatement().execute("create unique index idx_test_name on test(name)");
+        conn.createStatement().execute("INSERT INTO TEST VALUES(1, 'Hello')");
+        conn.close();
+        conn = getConnection(url);
+        try {
+            conn.createStatement().execute("INSERT INTO TEST VALUES(2, 'Hello')");
+            fail();
+        } catch (SQLException e) {
+            assertKnownException(e);
+        }
+        conn.close();
+    }
+
+
     private void testExistingOld() throws SQLException {
         if (config.memory) {
             return;
@@ -359,4 +414,36 @@ public class TestPageStore extends TestBase {
         trace("   " + m);
     }
 
+    public void closingDatabase() {
+        event("closing");
+    }
+
+    public void diskSpaceIsLow(long stillAvailable) {
+        event("diskSpaceIsLow " + stillAvailable);
+    }
+
+    public void exceptionThrown(SQLException e, String sql) {
+        event("exceptionThrown " + e + " " + sql);
+    }
+
+    public void init(String url) {
+        event("init");
+    }
+
+    public void opened() {
+        event("opened");
+    }
+
+    public void setProgress(int state, String name, int x, int max) {
+        if (name.startsWith("SYS:SYS_ID")) {
+            // ignore
+            return;
+        }
+        event("setProgress " + state + " " + name + " " + x + " " + max);
+    }
+
+    private void event(String s) {
+        eventBuffer.append(s).append(';');
+    }
+
 }