Page store: automatically compact the database (work in progress)

h2/src/main/org/h2/engine/Database.java

@@ -1238,6 +1238,8 @@ public class Database implements DataHandler {
                     pageStore.checkpoint();
                     pageStore.trim();
                 } catch (Throwable e) {
+                    int test;
+                    // e.printStackTrace(System.out);
                     traceSystem.getTrace(Trace.DATABASE).error("close", e);
                 }
             }

h2/src/main/org/h2/index/PageBtreeIndex.java

@@ -27,11 +27,10 @@ import org.h2.value.ValueNull;
  * This is the most common type of index, a b tree index.
  * Only the data of the indexed columns are stored in the index.
  */
-public class PageBtreeIndex extends BaseIndex {
+public class PageBtreeIndex extends PageIndex {
 
     private PageStore store;
     private TableData tableData;
-    private final int headPos;
     private boolean needRebuild;
     private long rowCount;
 
@@ -41,25 +40,24 @@ public class PageBtreeIndex extends BaseIndex {
         // trace.setLevel(TraceSystem.DEBUG);
         tableData = table;
         if (!database.isPersistent() || id < 0) {
-            this.headPos = 0;
             throw Message.throwInternalError("" + indexName);
         }
         this.store = database.getPageStore();
         store.addIndex(this);
         if (headPos == Index.EMPTY_HEAD) {
             // new index
+            rootPageId = store.allocatePage();
             needRebuild = true;
-            this.headPos = headPos = store.allocatePage();
             // TODO currently the head position is stored in the log
             // it should not for new tables, otherwise redo of other operations
             // must ensure this page is not used for other things
-            store.addMeta(this, session, headPos);
-            PageBtreeLeaf root = new PageBtreeLeaf(this, headPos, store.createData());
+            store.addMeta(this, session);
+            PageBtreeLeaf root = new PageBtreeLeaf(this, rootPageId, store.createData());
             root.parentPageId = PageBtree.ROOT;
             store.updateRecord(root, true, root.data);
         } else {
-            this.headPos = headPos;
-            PageBtree root = getPage(headPos);
+            rootPageId = store.getRootPageId(this);
+            PageBtree root = getPage(rootPageId);
             rowCount = root.getRowCount();
             if (rowCount == 0 && store.isRecoveryRunning()) {
                 needRebuild = true;
@@ -75,10 +73,6 @@ public class PageBtreeIndex extends BaseIndex {
         }
     }
 
-    public int getHeadPos() {
-        return headPos;
-    }
-
     public void add(Session session, Row row) throws SQLException {
         if (trace.isDebugEnabled()) {
             trace.debug("add " + row.getPos());
@@ -86,7 +80,7 @@ public class PageBtreeIndex extends BaseIndex {
         // safe memory
         SearchRow newRow = getSearchRow(row);
         while (true) {
-            PageBtree root = getPage(headPos);
+            PageBtree root = getPage(rootPageId);
             int splitPoint = root.addRowTry(newRow);
             if (splitPoint == -1) {
                 break;
@@ -100,8 +94,8 @@ public class PageBtreeIndex extends BaseIndex {
             int rootPageId = root.getPos();
             int id = store.allocatePage();
             page1.setPageId(id);
-            page1.setParentPageId(headPos);
-            page2.setParentPageId(headPos);
+            page1.setParentPageId(rootPageId);
+            page2.setParentPageId(rootPageId);
             PageBtreeNode newRoot = new PageBtreeNode(this, rootPageId, store.createData());
             newRoot.parentPageId = PageBtree.ROOT;
             newRoot.init(page1, pivot, page2);
@@ -161,7 +155,7 @@ public class PageBtreeIndex extends BaseIndex {
         if (SysProperties.CHECK && store == null) {
             throw Message.getSQLException(ErrorCode.OBJECT_CLOSED);
         }
-        PageBtree root = getPage(headPos);
+        PageBtree root = getPage(rootPageId);
         PageBtreeCursor cursor = new PageBtreeCursor(session, this, last);
         root.find(cursor, first, bigger);
         return cursor;
@@ -180,7 +174,7 @@ public class PageBtreeIndex extends BaseIndex {
             }
             return cursor;
         }
-        PageBtree root = getPage(headPos);
+        PageBtree root = getPage(rootPageId);
         PageBtreeCursor cursor = new PageBtreeCursor(session, this, null);
         root.last(cursor);
         cursor.previous();
@@ -223,7 +217,7 @@ public class PageBtreeIndex extends BaseIndex {
         if (rowCount == 1) {
             removeAllRows();
         } else {
-            PageBtree root = getPage(headPos);
+            PageBtree root = getPage(rootPageId);
             root.remove(row);
             rowCount--;
         }
@@ -234,7 +228,7 @@ public class PageBtreeIndex extends BaseIndex {
             trace.debug("remove");
         }
         removeAllRows();
-        store.freePage(headPos, false, null);
+        store.freePage(rootPageId, false, null);
         store.removeMeta(this, session);
     }
 
@@ -250,11 +244,11 @@ public class PageBtreeIndex extends BaseIndex {
     }
 
     private void removeAllRows() throws SQLException {
-        PageBtree root = getPage(headPos);
+        PageBtree root = getPage(rootPageId);
         root.freeChildren();
-        root = new PageBtreeLeaf(this, headPos, store.createData());
+        root = new PageBtreeLeaf(this, rootPageId, store.createData());
         root.parentPageId = PageBtree.ROOT;
-        store.removeRecord(headPos);
+        store.removeRecord(rootPageId);
         store.updateRecord(root, true, null);
         rowCount = 0;
     }
@@ -358,4 +352,11 @@ public class PageBtreeIndex extends BaseIndex {
         return true;
     }
 
+    public void setRootPageId(Session session, int newPos) throws SQLException {
+        store.removeMeta(this, session);
+        this.rootPageId = newPos;
+        store.addMeta(this, session);
+        store.addIndex(this);
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -9,11 +9,13 @@ package org.h2.index;
 import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
+import org.h2.engine.Session;
 import org.h2.message.Message;
 import org.h2.result.SearchRow;
 import org.h2.store.Data;
 import org.h2.store.DataPage;
 import org.h2.store.Page;
+import org.h2.store.PageStore;
 
 /**
  * A b-tree leaf page that contains index data.
@@ -283,4 +285,24 @@ public class PageBtreeLeaf extends PageBtree {
         return "page[" + getPos() + "] b-tree leaf table:" + index.getId() + " entries:" + entryCount;
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+        PageStore store = index.getPageStore();
+        PageBtreeLeaf p2 = new PageBtreeLeaf(index, newPos, store.createData());
+        readAllRows();
+        p2.rows = rows;
+        p2.entryCount = entryCount;
+        p2.offsets = offsets;
+        p2.onlyPosition = onlyPosition;
+        p2.parentPageId = parentPageId;
+        p2.start = start;
+        store.updateRecord(p2, false, null);
+        if (parentPageId == ROOT) {
+            index.setRootPageId(session, newPos);
+        } else {
+            PageBtreeNode p = (PageBtreeNode) store.getPage(parentPageId);
+            p.moveChild(getPos(), newPos);
+        }
+        store.freePage(getPos(), true, data);
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -9,6 +9,7 @@ package org.h2.index;
 import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
+import org.h2.engine.Session;
 import org.h2.message.Message;
 import org.h2.result.SearchRow;
 import org.h2.store.Data;
@@ -477,4 +478,42 @@ public class PageBtreeNode extends PageBtree {
         return "page[" + getPos() + "] b-tree node table:" + index.getId() + " entries:" + entryCount;
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+        PageStore store = index.getPageStore();
+        PageBtreeNode p2 = new PageBtreeNode(index, newPos, store.createData());
+        readAllRows();
+        p2.childPageIds = childPageIds;
+        p2.rows = rows;
+        p2.entryCount = entryCount;
+        p2.offsets = offsets;
+        p2.onlyPosition = onlyPosition;
+        p2.parentPageId = parentPageId;
+        p2.start = start;
+        store.updateRecord(p2, false, null);
+        if (parentPageId == ROOT) {
+            index.setRootPageId(session, newPos);
+        } else {
+            PageBtreeNode p = (PageBtreeNode) store.getPage(parentPageId);
+            p.moveChild(getPos(), newPos);
+        }
+        for (int i = 0; i < childPageIds.length; i++) {
+            PageBtree p = (PageBtree) store.getPage(childPageIds[i]);
+            p.setParentPageId(newPos);
+            store.updateRecord(p, true, p.data);
+        }
+        store.freePage(getPos(), true, data);
+    }
+
+    public void moveChild(int oldPos, int newPos) throws SQLException {
+        for (int i = 0; i < childPageIds.length; i++) {
+            if (childPageIds[i] == oldPos) {
+                written = false;
+                childPageIds[i] = newPos;
+                index.getPageStore().updateRecord(this, true, data);
+                return;
+            }
+        }
+        throw Message.throwInternalError();
+    }
+
 }
\ No newline at end of file

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -399,4 +399,24 @@ public class PageDataLeaf extends PageData {
         return "page[" + getPos() + "] data leaf table:" + index.getId() + " entries:" + entryCount;
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+//        PageStore store = index.getPageStore();
+//        PageBtreeLeaf p2 = new PageBtreeLeaf(index, newPos, store.createData());
+//        readAllRows();
+//        p2.rows = rows;
+//        p2.entryCount = entryCount;
+//        p2.offsets = offsets;
+//        p2.onlyPosition = onlyPosition;
+//        p2.parentPageId = parentPageId;
+//        p2.start = start;
+//        store.updateRecord(p2, false, null);
+//        if (firstOverflowPageId != 0) {
+//        }
+//        if (parentPageId == ROOT) {
+//        } else {
+//            PageBtreeNode p = (PageBtreeNode) store.getPage(parentPageId);
+//            p.moveChild(getPos(), newPos);
+//        }
+    }
+
 }

h2/src/main/org/h2/index/PageDataNode.java

@@ -329,4 +329,9 @@ public class PageDataNode extends PageData {
         return "page[" + getPos() + "] data node table:" + index.getId() + " entries:" + entryCount;
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+        // TODO Auto-generated method stub
+
+    }
+
 }

h2/src/main/org/h2/index/PageDataOverflow.java

@@ -8,6 +8,7 @@ package org.h2.index;
 
 import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
+import org.h2.engine.Session;
 import org.h2.message.Message;
 import org.h2.store.Data;
 import org.h2.store.DataPage;
@@ -180,4 +181,9 @@ public class PageDataOverflow extends Page {
         this.parentPage = parent;
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+        // TODO Auto-generated method stub
+
+    }
+
 }

h2/src/main/org/h2/index/PageIndex.java

+/*
+ * Copyright 2004-2009 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.index;
+
+/**
+ * A page store index.
+ */
+public abstract class PageIndex extends BaseIndex {
+
+    protected int rootPageId;
+
+    public int getRootPageId() {
+        return rootPageId;
+    }
+
+    public int getHeadPos() {
+        return 0;
+    }
+
+}

h2/src/main/org/h2/index/PageScanIndex.java

@@ -33,11 +33,10 @@ import org.h2.value.ValueLob;
  * all rows of a table. Each regular table has one such object, even if no
  * primary key or indexes are defined.
  */
-public class PageScanIndex extends BaseIndex implements RowIndex {
+public class PageScanIndex extends PageIndex implements RowIndex {
 
     private PageStore store;
     private TableData tableData;
-    private final int headPos;
     private int lastKey;
     private long rowCount;
     private HashSet<Row> delta;
@@ -55,22 +54,21 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
         this.store = database.getPageStore();
         store.addIndex(this);
         if (!database.isPersistent()) {
-            this.headPos = 0;
             throw Message.throwInternalError(table.getName());
         }
         if (headPos == Index.EMPTY_HEAD) {
             // new table
-            this.headPos = headPos = store.allocatePage();
+            rootPageId = store.allocatePage();
             // TODO currently the head position is stored in the log
             // it should not for new tables, otherwise redo of other operations
             // must ensure this page is not used for other things
-            store.addMeta(this, session, headPos);
-            PageDataLeaf root = new PageDataLeaf(this, headPos, store.createData());
+            store.addMeta(this, session);
+            PageDataLeaf root = new PageDataLeaf(this, rootPageId, store.createData());
             root.parentPageId = PageData.ROOT;
             store.updateRecord(root, true, root.data);
         } else {
-            this.headPos = headPos;
-            PageData root = getPage(headPos, 0);
+            rootPageId = store.getRootPageId(this);
+            PageData root = getPage(rootPageId, 0);
             lastKey = root.getLastKey();
             rowCount = root.getRowCount();
             // could have been created before, but never committed
@@ -86,10 +84,6 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
         table.setRowCount(rowCount);
     }
 
-    public int getHeadPos() {
-        return headPos;
-    }
-
     public void add(Session session, Row row) throws SQLException {
         if (row.getPos() == 0) {
             row.setPos(++lastKey);
@@ -112,7 +106,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
             }
         }
         while (true) {
-            PageData root = getPage(headPos, 0);
+            PageData root = getPage(rootPageId, 0);
             int splitPoint = root.addRowTry(row);
             if (splitPoint == -1) {
                 break;
@@ -126,8 +120,8 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
             int rootPageId = root.getPos();
             int id = store.allocatePage();
             page1.setPageId(id);
-            page1.setParentPageId(headPos);
-            page2.setParentPageId(headPos);
+            page1.setParentPageId(rootPageId);
+            page2.setParentPageId(rootPageId);
             PageDataNode newRoot = new PageDataNode(this, rootPageId, store.createData());
             newRoot.parentPageId = PageData.ROOT;
             newRoot.init(page1, pivot, page2);
@@ -177,8 +171,8 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
             empty.parentPageId = parent;
             return empty;
         }
-        if (p.index.headPos != headPos) {
-            throw Message.throwInternalError("Wrong index: " + p.index.getName() + ":" + p.index.headPos + " " + getName() + ":" + headPos);
+        if (p.index.rootPageId != rootPageId) {
+            throw Message.throwInternalError("Wrong index: " + p.index.getName() + ":" + p.index.rootPageId + " " + getName() + ":" + rootPageId);
         }
         if (parent != -1) {
             if (p.getParentPageId() != parent) {
@@ -193,7 +187,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     }
 
     public Cursor find(Session session, SearchRow first, SearchRow last) throws SQLException {
-        PageData root = getPage(headPos, 0);
+        PageData root = getPage(rootPageId, 0);
         return root.find(session);
     }
 
@@ -226,7 +220,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
             removeAllRows();
         } else {
             int key = row.getPos();
-            PageData root = getPage(headPos, 0);
+            PageData root = getPage(rootPageId, 0);
             root.remove(key);
             invalidateRowCount();
             rowCount--;
@@ -248,7 +242,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     }
 
     private void invalidateRowCount() throws SQLException {
-        PageData root = getPage(headPos, 0);
+        PageData root = getPage(rootPageId, 0);
         root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
     }
 
@@ -257,7 +251,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
             trace.debug("remove");
         }
         removeAllRows();
-        store.freePage(headPos, false, null);
+        store.freePage(rootPageId, false, null);
         store.removeMeta(this, session);
     }
 
@@ -277,11 +271,11 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     }
 
     private void removeAllRows() throws SQLException {
-        PageData root = getPage(headPos, 0);
+        PageData root = getPage(rootPageId, 0);
         root.freeChildren();
-        root = new PageDataLeaf(this, headPos, store.createData());
+        root = new PageDataLeaf(this, rootPageId, store.createData());
         root.parentPageId = PageData.ROOT;
-        store.removeRecord(headPos);
+        store.removeRecord(rootPageId);
         store.updateRecord(root, true, null);
         rowCount = 0;
         lastKey = 0;
@@ -292,7 +286,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     }
 
     public Row getRow(Session session, int key) throws SQLException {
-        PageData root = getPage(headPos, 0);
+        PageData root = getPage(rootPageId, 0);
         return root.getRow(key);
     }
 
@@ -348,7 +342,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
         }
         // can not close the index because it might get used afterwards,
         // for example after running recovery
-        PageData root = getPage(headPos, 0);
+        PageData root = getPage(rootPageId, 0);
         root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
     }
 

h2/src/main/org/h2/store/Page.java

@@ -6,6 +6,9 @@
  */
 package org.h2.store;
 
+import java.sql.SQLException;
+import org.h2.engine.Session;
+
 
 /**
  * A page. Format:
@@ -66,4 +69,13 @@ public abstract class Page extends Record {
      */
     public static final int TYPE_STREAM_DATA = 8;
 
+    /**
+     * Copy the data to a new location, change the parent to point to the new
+     * location, and free up the current page.
+     *
+     * @param session the session
+     * @param newPos the new position
+     */
+    public abstract void moveTo(Session session, int newPos) throws SQLException;
+
 }

h2/src/main/org/h2/store/PageFreeList.java

@@ -7,6 +7,7 @@ package org.h2.store;
 
 import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
+import org.h2.engine.Session;
 import org.h2.message.Message;
 import org.h2.util.BitField;
 
@@ -102,7 +103,7 @@ public class PageFreeList extends Page {
         if (free >= pageCount) {
             return -1;
         }
-        return free;
+        return free + getPos();
     }
 
     int getLastUsed() {
@@ -197,4 +198,10 @@ public class PageFreeList extends Page {
         return used.get(pageId - getPos());
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+        // the old data does not need to be copied, as free-list pages
+        // at the end of the file are not required
+        store.freePage(getPos(), true, data);
+    }
+
 }

h2/src/main/org/h2/store/PageLog.java

@@ -258,7 +258,7 @@ public class PageLog {
                     int tableId = in.readInt();
                     Row row = readRow(in, data);
                     if (stage == RECOVERY_STAGE_UNDO && x == ADD) {
-                        store.allocateIfHead(pos, tableId, row);
+                        store.allocateIfIndexRoot(pos, tableId, row);
                     } else if (stage == RECOVERY_STAGE_REDO) {
                         if (isSessionCommitted(sessionId, logId, pos)) {
                             if (trace.isDebugEnabled()) {
@@ -517,6 +517,7 @@ public class PageLog {
             }
             session.addLogPos(logSectionId, logPos);
             row.setLastLog(logSectionId, logPos);
+            logPos++;
 
             data.reset();
             data.checkCapacity(row.getByteCount(data));
@@ -545,6 +546,8 @@ public class PageLog {
                 trace.debug("log truncate s:" + session.getId() + " table:" + tableId);
             }
             session.addLogPos(logSectionId, logPos);
+            logPos++;
+
             data.reset();
             out.write(TRUNCATE);
             out.writeInt(session.getId());
@@ -579,6 +582,7 @@ public class PageLog {
         }
         undo = new BitField();
         logSectionId++;
+        logPos = 0;
         pageOut.fillPage();
         int currentDataPage = pageOut.getCurrentDataPageId();
         logSectionPageMap.put(logSectionId, currentDataPage);
@@ -588,6 +592,10 @@ public class PageLog {
         return logSectionId;
     }
 
+    long getLogPos() {
+        return logPos;
+    }
+
     /**
      * Remove all pages until the given log (excluding).
      *

h2/src/main/org/h2/store/PageStreamData.java

@@ -7,6 +7,7 @@
 package org.h2.store;
 
 import java.sql.SQLException;
+import org.h2.engine.Session;
 
 /**
  * A data page of a stream. The format is:
@@ -160,4 +161,23 @@ public class PageStreamData extends Page {
         remaining = length;
     }
 
+    public void moveTo(Session session, int newPos) throws SQLException {
+//        PageStreamData d2 = new PageStreamData(store, newPos, trunk);
+//        d2.initWrite();
+//        initRead();
+//        byte[] buff = new byte[remaining];
+//        read(buff, 0, remaining);
+//        d2.write(buff, 0, remaining);
+//        store.updateRecord(d2, false, null);
+//        PageStreamTrunk t = (PageStreamTrunk) store.getPage(trunk);
+//        t.moveChild(getPos(), newPos);
+//        store.freePage(getPos(), true, data);
+    }
+
+    void setTrunkPage(int newTrunk) throws SQLException {
+        this.trunk = newTrunk;
+        data.setInt(0, trunk);
+        store.updateRecord(this, true, data);
+    }
+
 }
\ No newline at end of file

h2/src/main/org/h2/store/PageStreamTrunk.java

@@ -7,6 +7,7 @@
 package org.h2.store;
 
 import java.sql.SQLException;
+import org.h2.engine.Session;
 
 /**
  * A trunk page of a stream. It contains the page numbers of the stream, and
@@ -180,4 +181,37 @@ public class PageStreamTrunk extends Page {
         return store.getPageSize() >> 2;
     }
 
+    /**
+     * One of the children has moved to another place.
+     *
+     * @param oldPos the old position
+     * @param newPos the new position
+     */
+    void moveChild(int oldPos, int newPos) throws SQLException {
+        for (int i = 0; i < pageIds.length; i++) {
+            if (pageIds[i] == oldPos) {
+                pageIds[i] = newPos;
+                break;
+            }
+        }
+        store.updateRecord(this, true, data);
+    }
+
+    public void moveTo(Session session, int newPos) throws SQLException {
+//        PageStreamTrunk p2 = new PageStreamTrunk(store, parent, newPos, nextTrunk, pageIds);
+//        store.updateRecord(p2, false, null);
+//        for (int i = 0; i < pageCount; i++) {
+//            int p = pageIds[i];
+//            PageStreamData d = (PageStreamData) store.getPage(p);
+//            if (d != null) {
+//                d.setTrunkPage(newPos);
+//            }
+//        }
+//        if (store.getLogFirstTrunkPage() == getPos()) {
+//            int dataPageId = store.getLogFirstDataPage();
+//            store.setLogFirstPage(newPos, dataPageId);
+//        }
+//        store.freePage(getPos(), true, data);
+    }
+
 }