Page store: various bugfixes.

h2/src/main/org/h2/index/PageBtree.java

@@ -166,7 +166,10 @@ public abstract class PageBtree extends Page {
     SearchRow getRow(int at) throws SQLException {
         SearchRow row = rows[at];
         if (row == null) {
-            row = index.readRow(data, offsets[at], onlyPosition);
+            row = index.readRow(data, offsets[at], onlyPosition, true);
+            rows[at] = row;
+        } else if (!index.hasData(row)) {
+            row = index.readRow(row.getKey());
             rows[at] = row;
         }
         return row;
@@ -186,6 +189,7 @@ public abstract class PageBtree extends Page {
      * @param id the new page id
      */
     void setPageId(int id) throws SQLException {
+        changeCount = index.getPageStore().getChangeCount();
         written = false;
         index.getPageStore().removeRecord(getPos());
         setPos(id);
@@ -214,6 +218,7 @@ public abstract class PageBtree extends Page {
      */
     void setParentPageId(int id) throws SQLException {
         index.getPageStore().logUndo(this, data);
+        changeCount = index.getPageStore().getChangeCount();
         written = false;
         parentPageId = id;
     }
@@ -243,7 +248,11 @@ public abstract class PageBtree extends Page {
      */
     protected void readAllRows() throws SQLException {
         for (int i = 0; i < entryCount; i++) {
-            getRow(i);
+            SearchRow row = rows[i];
+            if (row == null) {
+                row = index.readRow(data, offsets[i], onlyPosition, false);
+                rows[i] = row;
+            }
         }
     }
 
@@ -257,4 +266,11 @@ public abstract class PageBtree extends Page {
         return index.getPageStore().getPageSize();
     }
 
+    public boolean canRemove() {
+        if (changeCount >= index.getPageStore().getChangeCount()) {
+            return false;
+        }
+        return super.canRemove();
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeIndex.java

@@ -74,6 +74,14 @@ public class PageBtreeIndex extends PageIndex {
         }
         // safe memory
         SearchRow newRow = getSearchRow(row);
+        try {
+            addRow(newRow);
+        } finally {
+            store.incrementChangeCount();
+        }
+    }
+
+    private void addRow(SearchRow newRow) throws SQLException {
         while (true) {
             PageBtree root = getPage(rootPageId);
             int splitPoint = root.addRowTry(newRow);
@@ -217,10 +225,14 @@ public class PageBtreeIndex extends PageIndex {
         if (rowCount == 1) {
             removeAllRows();
         } else {
+            try {
                 PageBtree root = getPage(rootPageId);
                 root.remove(row);
                 invalidateRowCount();
                 rowCount--;
+            } finally {
+                store.incrementChangeCount();
+            }
         }
     }
 
@@ -245,12 +257,16 @@ public class PageBtreeIndex extends PageIndex {
     }
 
     private void removeAllRows() throws SQLException {
+        try {
             PageBtree root = getPage(rootPageId);
             root.freeRecursive();
             root = PageBtreeLeaf.create(this, rootPageId, PageBtree.ROOT);
             store.removeRecord(rootPageId);
             store.update(root);
             rowCount = 0;
+        } finally {
+            store.incrementChangeCount();
+        }
     }
 
     public void checkRename() {
@@ -286,7 +302,11 @@ public class PageBtreeIndex extends PageIndex {
         }
         // can not close the index because it might get used afterwards,
         // for example after running recovery
+        try {
             writeRowCount();
+        } finally {
+            store.incrementChangeCount();
+        }
     }
 
     /**
@@ -295,14 +315,20 @@ public class PageBtreeIndex extends PageIndex {
      * @param data the data
      * @param offset the offset
      * @param onlyPosition whether only the position of the row is stored
+     * @param needData whether the row data is required
      * @return the row
      */
-    SearchRow readRow(Data data, int offset, boolean onlyPosition) throws SQLException {
+    SearchRow readRow(Data data, int offset, boolean onlyPosition, boolean needData) throws SQLException {
         data.setPos(offset);
         long key = data.readVarLong();
         if (onlyPosition) {
+            if (needData) {
                 return tableData.getRow(null, key);
             }
+            SearchRow row = table.getTemplateSimpleRow(true);
+            row.setKey(key);
+            return row;
+        }
         SearchRow row = table.getTemplateSimpleRow(columns.length == 1);
         row.setKey(key);
         for (Column col : columns) {
@@ -312,6 +338,16 @@ public class PageBtreeIndex extends PageIndex {
         return row;
     }
 
+    /**
+     * Get the complete row from the data index.
+     *
+     * @param key the key
+     * @return the row
+     */
+    SearchRow readRow(long key) throws SQLException {
+        return tableData.getRow(null, key);
+    }
+
     /**
      * Write a row to the data page at the given offset.
      *
@@ -377,4 +413,14 @@ public class PageBtreeIndex extends PageIndex {
         root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
     }
 
+    /**
+     * Check whether the given row contains data.
+     *
+     * @param row the row
+     * @return true if it contains data
+     */
+    boolean hasData(SearchRow row) {
+        return row.getValue(columns[0].getColumnId()) != null;
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -126,6 +126,8 @@ public class PageBtreeLeaf extends PageBtree {
             }
         }
         index.getPageStore().logUndo(this, data);
+        readAllRows();
+        changeCount = index.getPageStore().getChangeCount();
         written = false;
         int offset = last - rowLength;
         int[] newOffsets = new int[entryCount + 1];
@@ -134,7 +136,6 @@ public class PageBtreeLeaf extends PageBtree {
         if (entryCount == 0) {
             x = 0;
         } else {
-            readAllRows();
             x = find(row, false, true, true);
             System.arraycopy(offsets, 0, newOffsets, 0, x);
             System.arraycopy(rows, 0, newRows, 0, x);
@@ -161,6 +162,7 @@ public class PageBtreeLeaf extends PageBtree {
         index.getPageStore().logUndo(this, data);
         entryCount--;
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         if (entryCount <= 0) {
             Message.throwInternalError();
         }

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -29,9 +29,10 @@ import org.h2.util.MemoryUtils;
  * <li>count of all children (-1 if not known): int</li>
  * <li>entry count: short</li>
  * <li>rightmost child page id: int</li>
- * <li>entries (child page id: int, offset: short) The row contains the largest
- * key of the respective child, meaning row[0] contains the largest key of
- * child[0].
+ * <li>entries (child page id: int, offset: short)</li>
+ * </ul>
+ * The row contains the largest key of the respective child,
+ * meaning row[0] contains the largest key of child[0].
  */
 public class PageBtreeNode extends PageBtree {
 
@@ -79,6 +80,9 @@ public class PageBtreeNode extends PageBtree {
         p.writeHead();
         // 4 bytes for the rightmost child page id
         p.start = p.data.length() + 4;
+        if (SysProperties.PAGE_STORE_INTERNAL_COUNT) {
+            p.rowCount = 0;
+        }
         return p;
     }
 
@@ -117,7 +121,7 @@ public class PageBtreeNode extends PageBtree {
      * @return the split point of this page, or -1 if no split is required
      */
     private int addChildTry(SearchRow row) throws SQLException {
-        if (entryCount < 2) {
+        if (entryCount < 3) {
             return -1;
         }
         int rowLength = index.getRowSize(data, row, onlyPosition);
@@ -181,8 +185,14 @@ public class PageBtreeNode extends PageBtree {
         offsets = newOffsets;
         rows = newRows;
         childPageIds = newChildPageIds;
+        if (SysProperties.PAGE_STORE_INTERNAL_COUNT) {
+            if (rowCount != UNKNOWN_ROWCOUNT) {
+                rowCount += offset;
+            }
+        }
         entryCount++;
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
     }
 
     int addRowTry(SearchRow row) throws SQLException {
@@ -208,6 +218,7 @@ public class PageBtreeNode extends PageBtree {
         }
         updateRowCount(1);
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         return -1;
     }
 
@@ -236,7 +247,7 @@ public class PageBtreeNode extends PageBtree {
         int firstChild = childPageIds[splitPoint];
         readAllRows();
         for (int i = splitPoint; i < entryCount;) {
-            p2.addChild(p2.entryCount, childPageIds[splitPoint + 1], rows[splitPoint]);
+            p2.addChild(p2.entryCount, childPageIds[splitPoint + 1], getRow(splitPoint));
             removeChild(splitPoint);
         }
         int lastChild = childPageIds[splitPoint - 1];
@@ -271,7 +282,9 @@ public class PageBtreeNode extends PageBtree {
         rows = new SearchRow[0];
         offsets = MemoryUtils.EMPTY_INT_ARRAY;
         addChild(0, page2.getPos(), pivot);
+        if (SysProperties.PAGE_STORE_INTERNAL_COUNT) {
             rowCount = page1.getRowCount() + page2.getRowCount();
+        }
         check();
     }
 
@@ -313,6 +326,7 @@ public class PageBtreeNode extends PageBtree {
         index.getPageStore().logUndo(this, data);
         updateRowCount(-1);
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         if (last == null) {
             // the last row didn't change - nothing to do
             return null;
@@ -364,11 +378,15 @@ public class PageBtreeNode extends PageBtree {
     }
 
     void setRowCountStored(int rowCount) throws SQLException {
+        if (rowCount < 0 && SysProperties.PAGE_STORE_INTERNAL_COUNT) {
+            return;
+        }
         this.rowCount = rowCount;
         if (rowCountStored != rowCount) {
             rowCountStored = rowCount;
             index.getPageStore().logUndo(this, data);
             if (written) {
+                changeCount = index.getPageStore().getChangeCount();
                 writeHead();
             }
             index.getPageStore().update(this);
@@ -422,6 +440,7 @@ public class PageBtreeNode extends PageBtree {
 
     void freeRecursive() throws SQLException {
         index.getPageStore().logUndo(this, data);
+        index.getPageStore().free(getPos());
         for (int childPageId : childPageIds) {
             index.getPage(childPageId).freeRecursive();
         }
@@ -430,7 +449,11 @@ public class PageBtreeNode extends PageBtree {
     private void removeChild(int i) throws SQLException {
         readAllRows();
         entryCount--;
+        if (SysProperties.PAGE_STORE_INTERNAL_COUNT) {
+            updateRowCount(-index.getPage(childPageIds[i]).getRowCount());
+        }
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         if (entryCount < 0) {
             Message.throwInternalError();
         }
@@ -536,8 +559,12 @@ public class PageBtreeNode extends PageBtree {
         if (parentPageId == ROOT) {
             index.setRootPageId(session, newPos);
         } else {
-            PageBtreeNode p = (PageBtreeNode) store.getPage(parentPageId);
-            p.moveChild(getPos(), newPos);
+            Page p = store.getPage(parentPageId);
+            if (!(p instanceof PageBtreeNode)) {
+                throw Message.throwInternalError();
+            }
+            PageBtreeNode n = (PageBtreeNode) p;
+            n.moveChild(getPos(), newPos);
         }
         for (int childPageId : childPageIds) {
             PageBtree p = index.getPage(childPageId);
@@ -558,6 +585,7 @@ public class PageBtreeNode extends PageBtree {
             if (childPageIds[i] == oldPos) {
                 index.getPageStore().logUndo(this, data);
                 written = false;
+                changeCount = index.getPageStore().getChangeCount();
                 childPageIds[i] = newPos;
                 index.getPageStore().update(this);
                 return;

h2/src/main/org/h2/index/PageData.java

@@ -178,6 +178,7 @@ abstract class PageData extends Page {
         index.getPageStore().logUndo(this, data);
         parentPageId = id;
         if (written) {
+            changeCount = index.getPageStore().getChangeCount();
             data.setInt(START_PARENT, parentPageId);
         }
     }
@@ -224,4 +225,11 @@ abstract class PageData extends Page {
         return parentPageId;
     }
 
+    public boolean canRemove() {
+        if (changeCount >= index.getPageStore().getChangeCount()) {
+            return false;
+        }
+        return super.canRemove();
+    }
+
 }

h2/src/main/org/h2/index/PageDataIndex.java

@@ -136,6 +136,8 @@ public class PageDataIndex extends PageIndex implements RowIndex {
                     row.setKey(row.getKey() + add);
                 }
                 add++;
+            } finally {
+                store.incrementChangeCount();
             }
         }
         lastKey = Math.max(lastKey, row.getKey() + 1);
@@ -205,9 +207,6 @@ public class PageDataIndex extends PageIndex implements RowIndex {
      */
     PageData getPage(int id, int parent) throws SQLException {
         Page pd = store.getPage(id);
-        if (pd != null && !(pd instanceof PageData)) {
-            System.out.println("test");
-        }
         PageData p = (PageData) pd;
         if (p == null) {
             PageDataLeaf empty = PageDataLeaf.create(this, id, parent);
@@ -296,11 +295,15 @@ public class PageDataIndex extends PageIndex implements RowIndex {
         if (rowCount == 1) {
             removeAllRows();
         } else {
+            try {
                 long key = row.getKey();
                 PageData root = getPage(rootPageId, 0);
                 root.remove(key);
                 invalidateRowCount();
                 rowCount--;
+            } finally {
+                store.incrementChangeCount();
+            }
         }
         if (database.isMultiVersion()) {
             // if storage is null, the delete flag is not yet set
@@ -342,6 +345,7 @@ public class PageDataIndex extends PageIndex implements RowIndex {
     }
 
     private void removeAllRows() throws SQLException {
+        try {
             PageData root = getPage(rootPageId, 0);
             root.freeRecursive();
             root = PageDataLeaf.create(this, rootPageId, PageData.ROOT);
@@ -349,6 +353,9 @@ public class PageDataIndex extends PageIndex implements RowIndex {
             store.update(root);
             rowCount = 0;
             lastKey = 0;
+        } finally {
+            store.incrementChangeCount();
+        }
     }
 
     public void checkRename() throws SQLException {
@@ -493,8 +500,12 @@ public class PageDataIndex extends PageIndex implements RowIndex {
     }
 
     public void writeRowCount() throws SQLException {
+        try {
             PageData root = getPage(rootPageId, 0);
             root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
+        } finally {
+            store.incrementChangeCount();
+        }
     }
 
 }

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -198,6 +198,7 @@ public class PageDataLeaf extends PageData {
             }
         }
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         last = x == 0 ? pageSize : offsets[x - 1];
         offset = last - rowLength;
         entryCount++;
@@ -258,6 +259,7 @@ public class PageDataLeaf extends PageData {
     private void removeRow(int i) throws SQLException {
         index.getPageStore().logUndo(this, data);
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         readAllRows();
         Row r = rows[i];
         if (r != null) {
@@ -409,7 +411,7 @@ public class PageDataLeaf extends PageData {
         freeOverflow();
     }
 
-    void freeOverflow() throws SQLException {
+    private void freeOverflow() throws SQLException {
         if (firstOverflowPageId != 0) {
             int next = firstOverflowPageId;
             do {
@@ -503,6 +505,11 @@ public class PageDataLeaf extends PageData {
 
     public void moveTo(Session session, int newPos) throws SQLException {
         PageStore store = index.getPageStore();
+        // load the pages into the cache, to ensure old pages
+        // are written
+        if (parentPageId != ROOT) {
+            store.getPage(parentPageId);
+        }
         store.logUndo(this, data);
         PageDataLeaf p2 = PageDataLeaf.create(index, newPos, parentPageId);
         readAllRows();
@@ -543,6 +550,7 @@ public class PageDataLeaf extends PageData {
         index.getPageStore().logUndo(this, data);
         firstOverflowPageId = overflow;
         if (written) {
+            changeCount = index.getPageStore().getChangeCount();
             writeHead();
             data.writeInt(firstOverflowPageId);
         }

h2/src/main/org/h2/index/PageDataNode.java

@@ -115,6 +115,7 @@ public class PageDataNode extends PageData {
     private void addChild(int x, int childPageId, long key) throws SQLException {
         index.getPageStore().logUndo(this, data);
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         long[] newKeys = new long[entryCount + 1];
         int[] newChildPageIds = new int[entryCount + 2];
         if (childPageIds != null) {
@@ -305,6 +306,7 @@ public class PageDataNode extends PageData {
             rowCountStored = rowCount;
             index.getPageStore().logUndo(this, data);
             if (written) {
+                changeCount = index.getPageStore().getChangeCount();
                 writeHead();
             }
             index.getPageStore().update(this);
@@ -363,6 +365,7 @@ public class PageDataNode extends PageData {
     private void removeChild(int i) throws SQLException {
         index.getPageStore().logUndo(this, data);
         written = false;
+        changeCount = index.getPageStore().getChangeCount();
         entryCount--;
         int removedKeyIndex = i < keys.length ? i : i - 1;
         length -= 4 + data.getVarLongLen(keys[removedKeyIndex]);
@@ -387,6 +390,14 @@ public class PageDataNode extends PageData {
 
     public void moveTo(Session session, int newPos) throws SQLException {
         PageStore store = index.getPageStore();
+        // load the pages into the cache, to ensure old pages
+        // are written
+        for (int child : childPageIds) {
+            store.getPage(child);
+        }
+        if (parentPageId != ROOT) {
+            store.getPage(parentPageId);
+        }
         store.logUndo(this, data);
         PageDataNode p2 = PageDataNode.create(index, newPos, parentPageId);
         p2.rowCountStored = rowCountStored;
@@ -402,8 +413,8 @@ public class PageDataNode extends PageData {
             PageDataNode p = (PageDataNode) store.getPage(parentPageId);
             p.moveChild(getPos(), newPos);
         }
-        for (int i = 0; i < childPageIds.length; i++) {
-            PageData p = (PageData) store.getPage(childPageIds[i]);
+        for (int child : childPageIds) {
+            PageData p = (PageData) store.getPage(child);
             p.setParentPageId(newPos);
             store.update(p);
         }
@@ -421,6 +432,7 @@ public class PageDataNode extends PageData {
             if (childPageIds[i] == oldPos) {
                 index.getPageStore().logUndo(this, data);
                 written = false;
+                changeCount = index.getPageStore().getChangeCount();
                 childPageIds[i] = newPos;
                 index.getPageStore().update(this);
                 return;

h2/src/main/org/h2/index/PageDataOverflow.java

@@ -215,26 +215,31 @@ public class PageDataOverflow extends Page {
     }
 
     public void moveTo(Session session, int newPos) throws SQLException {
+        // load the pages into the cache, to ensure old pages
+        // are written
+        Page parent = store.getPage(parentPageId);
+        if (parent == null) {
+            throw Message.throwInternalError();
+        }
+        PageDataOverflow next = null;
+        if (nextPage != 0) {
+            next = (PageDataOverflow) store.getPage(nextPage);
+        }
         store.logUndo(this, data);
         PageDataOverflow p2 = PageDataOverflow.create(store, newPos, type, parentPageId, nextPage, data, start, size);
         store.update(p2);
-        if (nextPage != 0) {
-            PageDataOverflow p3 = (PageDataOverflow) store.getPage(nextPage);
-            p3.setParentPageId(newPos);
-            store.update(p3);
+        if (next != null) {
+            next.setParentPageId(newPos);
+            store.update(next);
         }
-        Page p = store.getPage(parentPageId);
-        if (p == null) {
-            throw Message.throwInternalError();
-        }
-        if (p instanceof PageDataOverflow) {
-            PageDataOverflow p1 = (PageDataOverflow) p;
+        if (parent instanceof PageDataOverflow) {
+            PageDataOverflow p1 = (PageDataOverflow) parent;
             p1.setNext(getPos(), newPos);
         } else {
-            PageDataLeaf p1 = (PageDataLeaf) p;
+            PageDataLeaf p1 = (PageDataLeaf) parent;
             p1.setOverflow(getPos(), newPos);
         }
-        store.update(p);
+        store.update(parent);
         store.free(getPos());
     }
 
@@ -247,9 +252,16 @@ public class PageDataOverflow extends Page {
         data.setInt(START_NEXT_OVERFLOW, nextPage);
     }
 
+    /**
+     * Free this page.
+     */
     void free() throws SQLException {
         store.logUndo(this, data);
         store.free(getPos());
     }
 
+    public boolean canRemove() {
+        return super.canRemove();
+    }
+
 }

h2/src/main/org/h2/store/Data.java

@@ -14,6 +14,7 @@ import java.sql.Date;
 import java.sql.SQLException;
 import java.sql.Time;
 import java.sql.Timestamp;
+import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.message.Message;
 import org.h2.util.DateTimeUtils;
@@ -46,6 +47,8 @@ import org.h2.value.ValueUuid;
  */
 public class Data extends DataPage {
 
+    private static final int TEST_OFFSET = 0;
+
     private static final int INT_0_15 = 32;
     private static final int LONG_0_7 = 48;
     private static final int DECIMAL_0_1 = 56;
@@ -334,11 +337,14 @@ public class Data extends DataPage {
      * @param v the value
      */
     public void writeValue(Value v) throws SQLException {
+        int start = pos;
+        if (TEST_OFFSET > 0) {
+            pos += TEST_OFFSET;
+        }
         if (v == ValueNull.INSTANCE) {
             data[pos++] = 0;
             return;
         }
-        int start = pos;
         int type = v.getType();
         switch (type) {
         case Value.BOOLEAN:
@@ -539,6 +545,9 @@ public class Data extends DataPage {
      * @return the value
      */
     public Value readValue() throws SQLException {
+        if (TEST_OFFSET > 0) {
+            pos+=TEST_OFFSET;
+        }
         int type = data[pos++] & 255;
         switch (type) {
         case Value.NULL:
@@ -667,7 +676,7 @@ public class Data extends DataPage {
             } else if (type >= STRING_0_31 && type < STRING_0_31 + 32) {
                 return ValueString.get(readString(type - STRING_0_31));
             }
-            throw Message.throwInternalError("type=" + type);
+            throw Message.getSQLException(ErrorCode.FILE_CORRUPTED_1, "type: " + type);
         }
     }
 
@@ -678,6 +687,10 @@ public class Data extends DataPage {
      * @return the number of bytes required to store this value
      */
     public int getValueLen(Value v) throws SQLException {
+        return getValueLen2(v) + TEST_OFFSET;
+    }
+
+    private int getValueLen2(Value v) throws SQLException {
         if (v == ValueNull.INSTANCE) {
             return 1;
         }

h2/src/main/org/h2/store/Page.java

@@ -69,6 +69,11 @@ public abstract class Page extends Record {
      */
     public static final int TYPE_STREAM_DATA = 8;
 
+    /**
+     * When this page was changed the last time.
+     */
+    protected int changeCount;
+
     /**
      * Copy the data to a new location, change the parent to point to the new
      * location, and free up the current page.

h2/src/main/org/h2/store/PageFreeList.java

@@ -77,7 +77,9 @@ public class PageFreeList extends Page {
         while (true) {
             int free = used.nextClearBit(start);
             if (free >= pageCount) {
+                if (start == 0) {
                     full = true;
+                }
                 return -1;
             }
             if (exclude != null && exclude.get(free + getPos())) {
@@ -210,4 +212,8 @@ public class PageFreeList extends Page {
         return "page [" + getPos() + "] freeList" + (full ? "full" : "");
     }
 
+    public boolean canRemove() {
+        return false;
+    }
+
 }

h2/src/main/org/h2/store/PageInputStream.java

@@ -10,7 +10,6 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.sql.SQLException;
-import org.h2.constant.ErrorCode;
 import org.h2.message.Trace;
 import org.h2.util.BitField;
 
@@ -21,7 +20,8 @@ public class PageInputStream extends InputStream {
 
     private PageStore store;
     private final Trace trace;
-    private int trunkNext;
+    private int firstTrunkPage;
+    private PageStreamTrunk.Iterator it;
     private int dataPage;
     private PageStreamTrunk trunk;
     private PageStreamData data;
@@ -30,12 +30,13 @@ public class PageInputStream extends InputStream {
     private byte[] buffer = new byte[1];
     private int logKey;
 
-    PageInputStream(PageStore store, int logKey, int trunkPage, int dataPage) {
+    PageInputStream(PageStore store, int logKey, int firstTrunkPage, int dataPage) {
         this.store = store;
         this.trace = store.getTrace();
-        // minus one, because we increment before reading the trunk page
+        // minus one because we increment before comparing
         this.logKey = logKey - 1;
-        this.trunkNext = trunkPage;
+        this.firstTrunkPage = firstTrunkPage;
+        it = new PageStreamTrunk.Iterator(store, firstTrunkPage);
         this.dataPage = dataPage;
     }
 
@@ -80,29 +81,20 @@ public class PageInputStream extends InputStream {
         }
     }
 
-    private void fillBuffer() throws SQLException, EOFException {
+    private void fillBuffer() throws SQLException {
         if (remaining > 0 || endOfFile) {
             return;
         }
-        if (trunkNext == 0) {
-            endOfFile = true;
-            return;
-        }
         int next;
         while (true) {
             if (trunk == null) {
-                Page p = store.getPage(trunkNext);
-                if (p instanceof PageStreamTrunk) {
-                    trunk = (PageStreamTrunk) p;
-                }
+                trunk = it.next();
                 logKey++;
-                if (trunk == null) {
-                    throw new EOFException();
-                } else if (trunk.getLogKey() != logKey) {
-                    throw new EOFException();
+                if (trunk == null || trunk.getLogKey() != logKey) {
+                    endOfFile = true;
+                    return;
                 }
                 trunk.resetIndex();
-                trunkNext = trunk.getNextTrunk();
             }
             if (trunk != null) {
                 next = trunk.getNextPageData();
@@ -122,10 +114,9 @@ public class PageInputStream extends InputStream {
         if (p instanceof PageStreamData) {
             data = (PageStreamData) p;
         }
-        if (data == null) {
-            throw new EOFException();
-        } else if (data.getLogKey() != logKey) {
-            throw new EOFException();
+        if (data == null || data.getLogKey() != logKey) {
+            endOfFile = true;
+            return;
         }
         data.initRead();
         remaining = data.getRemaining();
@@ -138,25 +129,18 @@ public class PageInputStream extends InputStream {
      */
     BitField allocateAllPages() throws SQLException {
         BitField pages = new BitField();
-        int trunkPage = trunkNext;
-        while (trunkPage != 0 && trunkPage < store.getPageCount()) {
-            pages.set(trunkPage);
-            store.allocatePage(trunkPage);
-            PageStreamTrunk t = null;
-            try {
-                Page p = store.getPage(trunkPage);
-                if (p instanceof PageStreamTrunk) {
-                    t = (PageStreamTrunk) p;
-                }
-            } catch (SQLException e) {
-                if (e.getErrorCode() != ErrorCode.FILE_CORRUPTED_1) {
-                    // wrong checksum means end of stream
-                    throw e;
-                }
+        int key = logKey;
+        PageStreamTrunk.Iterator it = new PageStreamTrunk.Iterator(store, firstTrunkPage);
+        while (true) {
+            PageStreamTrunk t = it.next();
+            key++;
+            if (it.canDelete()) {
+                store.allocatePage(it.getCurrentPageId());
             }
-            if (t == null) {
+            if (t == null || t.getLogKey() != key) {
                 break;
             }
+            pages.set(t.getPos());
             t.resetIndex();
             while (true) {
                 int n = t.getNextPageData();
@@ -166,7 +150,6 @@ public class PageInputStream extends InputStream {
                 pages.set(n);
                 store.allocatePage(n);
             }
-            trunkPage = t.getNextTrunk();
         }
         return pages;
     }

h2/src/main/org/h2/store/PageLog.java

@@ -192,22 +192,16 @@ public class PageLog {
      * Free up all pages allocated by the log.
      */
     void free() throws SQLException {
-        while (firstTrunkPage != 0 && firstTrunkPage < store.getPageCount()) {
-            PageStreamTrunk t = null;
-            try {
-                Page p = store.getPage(firstTrunkPage);
-                if (p instanceof PageStreamTrunk) {
-                    t = (PageStreamTrunk) p;
-                }
-            } catch (SQLException e) {
-                if (e.getErrorCode() != ErrorCode.FILE_CORRUPTED_1) {
-                    // wrong checksum means end of stream
-                    throw e;
-                }
+        if (pageOut != null) {
+            pageOut.freeReserved();
         }
+        PageStreamTrunk.Iterator it = new PageStreamTrunk.Iterator(store, firstTrunkPage);
+        while (firstTrunkPage != 0 && firstTrunkPage < store.getPageCount()) {
+            PageStreamTrunk t = it.next();
             if (t == null) {
+                if (it.canDelete()) {
                     store.free(firstTrunkPage, false);
-                // EOF
+                }
                 break;
             }
             t.free();
@@ -371,6 +365,12 @@ public class PageLog {
                     }
                 }
             }
+        } catch (SQLException e) {
+            if (e.getErrorCode() == ErrorCode.FILE_CORRUPTED_1) {
+                trace.debug("log recovery stopped: " + e.toString());
+            } else {
+                throw e;
+            }
         } catch (EOFException e) {
             trace.debug("log recovery stopped: " + e.toString());
         } catch (IOException e) {

h2/src/main/org/h2/store/PageOutputStream.java

@@ -202,4 +202,19 @@ public class PageOutputStream extends OutputStream {
         return trunk.getLogKey();
     }
 
+    /**
+     * Free up all reserved pages.
+     */
+    void freeReserved() throws SQLException {
+        if (reservedPages.size() > 0) {
+            int[] array = new int[reservedPages.size()];
+            reservedPages.toArray(array);
+            reservedPages = new IntArray();
+            reserved = 0;
+            for (int p : array) {
+                store.free(p);
+            }
+        }
+    }
+
 }

h2/src/main/org/h2/store/PageStore.java

@@ -83,8 +83,12 @@ import org.h2.value.ValueString;
 public class PageStore implements CacheWriter {
 
     // TODO test running out of disk space (using a special file system)
+    // TODO test with recovery being the default method
+    // TODO test reservedPages does not grow unbound
 
     // TODO utf-x: test if it's faster
+    // TODO corrupt pages should be freed once in a while
+    // TODO node row counts are incorrect (not splitting row counts)
     // TODO after opening the database, delay writing until required
     // TODO optimization: try to avoid allocating a byte array per page
     // TODO optimization: check if calling Data.getValueLen slows things down
@@ -139,8 +143,8 @@ public class PageStore implements CacheWriter {
 
     private static final int INCREMENT_PAGES = 128;
 
-    private static final int READ_VERSION = 1;
-    private static final int WRITE_VERSION = 1;
+    private static final int READ_VERSION = 2;
+    private static final int WRITE_VERSION = 2;
 
     private static final int META_TYPE_SCAN_INDEX = 0;
     private static final int META_TYPE_BTREE_INDEX = 1;
@@ -196,6 +200,15 @@ public class PageStore implements CacheWriter {
 
     private ObjectArray<PageFreeList> freeLists = ObjectArray.newInstance();
 
+    /**
+     * The change count is something like a "micro-transaction-id".
+     * It is used to ensure that changed pages are not written to the file
+     * before the the current operation is not finished. This is only a problem
+     * when using a very small cache size. The value starts at 1 so that
+     * pages with change count 0 can be evicted from the cache.
+     */
+    private int changeCount = 1;
+
     /**
      * Create a new page store object.
      *
@@ -341,6 +354,9 @@ public class PageStore implements CacheWriter {
                 if (isUsed(i)) {
                     freed.clear(i);
                 } else if (!freed.get(i)) {
+                    if (trace.isDebugEnabled()) {
+                        trace.debug("free " + i);
+                    }
                     freed.set(i);
                     file.seek((long) i << pageSizeShift);
                     file.write(empty, 0, pageSize);
@@ -445,7 +461,11 @@ public class PageStore implements CacheWriter {
             if (p != null) {
                 trace.debug("move " + p.getPos() + " to " + free);
                 long logSection = log.getLogSectionId(), logPos = log.getLogPos();
+                try {
                     p.moveTo(systemSession, free);
+                } finally {
+                    changeCount++;
+                }
                 if (log.getLogSectionId() == logSection || log.getLogPos() != logPos) {
                     commit(systemSession);
                 }
@@ -717,9 +737,9 @@ public class PageStore implements CacheWriter {
     public void logUndo(Record record, Data old) throws SQLException {
         synchronized (database) {
             if (trace.isDebugEnabled()) {
-                if (!record.isChanged()) {
-                    trace.debug("logUndo " + record.toString());
-                }
+                // if (!record.isChanged()) {
+                //     trace.debug("logUndo " + record.toString());
+                // }
             }
             checkOpen();
             database.checkWritingAllowed();
@@ -837,23 +857,23 @@ public class PageStore implements CacheWriter {
     }
 
     private int allocatePage(BitField exclude, int first) throws SQLException {
-        int pos;
+        int page;
         synchronized (database) {
             // TODO could remember the first possible free list page
             for (int i = 0;; i++) {
                 PageFreeList list = getFreeList(i);
-                pos = list.allocate(exclude, first);
-                if (pos >= 0) {
+                page = list.allocate(exclude, first);
+                if (page >= 0) {
                     break;
                 }
             }
-            if (pos >= pageCount) {
+            if (page >= pageCount) {
                 increaseFileSize(INCREMENT_PAGES);
             }
             if (trace.isDebugEnabled()) {
                 // trace.debug("allocatePage " + pos);
             }
-            return pos;
+            return page;
         }
     }
 
@@ -980,6 +1000,13 @@ public class PageStore implements CacheWriter {
             Message.throwInternalError("write to page " + pageId);
         }
         byte[] bytes = data.getBytes();
+        if (SysProperties.CHECK) {
+            boolean shouldBeFreeList = (pageId - PAGE_ID_FREE_LIST_ROOT) % freeListPagesPerList == 0;
+            boolean isFreeList = bytes[0] == Page.TYPE_FREE_LIST;
+            if (bytes[0] != 0 && shouldBeFreeList != isFreeList) {
+                throw Message.throwInternalError();
+            }
+        }
         checksumSet(bytes, pageId);
         synchronized (database) {
             file.seek((long) pageId << pageSizeShift);
@@ -1558,4 +1585,20 @@ public class PageStore implements CacheWriter {
         return true;
     }
 
+    /**
+     * Increment the change count. To be done after the operation has finished.
+     */
+    public void incrementChangeCount() {
+        changeCount++;
+    }
+
+    /**
+     * Get the current change count. The first value is 1
+     *
+     * @return the change count
+     */
+    public int getChangeCount() {
+        return changeCount;
+    }
+
 }

h2/src/main/org/h2/store/PageStreamData.java

@@ -169,4 +169,8 @@ public class PageStreamData extends Page {
         return "[" + getPos() + "] stream data pos:" + data.length() + " remaining:" + remaining;
     }
 
+    public boolean canRemove() {
+        return true;
+    }
+
 }
\ No newline at end of file

h2/src/main/org/h2/store/PageStreamTrunk.java

@@ -7,6 +7,7 @@
 package org.h2.store;
 
 import java.sql.SQLException;
+import org.h2.constant.ErrorCode;
 import org.h2.engine.Session;
 
 /**
@@ -26,10 +27,18 @@ public class PageStreamTrunk extends Page {
 
     private static final int DATA_START = 17;
 
+    /**
+     * The previous stream trunk.
+     */
+    int parent;
+
+    /**
+     * The next stream trunk.
+     */
+    int nextTrunk;
+
     private final PageStore store;
-    private int parent;
     private int logKey;
-    private int nextTrunk;
     private int[] pageIds;
     private int pageCount;
     private Data data;
@@ -115,10 +124,6 @@ public class PageStreamTrunk extends Page {
         return pageIds[index++];
     }
 
-    int getNextTrunk() {
-        return nextTrunk;
-    }
-
     public int getByteCount(DataPage dummy) {
         return store.getPageSize();
     }
@@ -212,4 +217,86 @@ public class PageStreamTrunk extends Page {
         return logKey;
     }
 
+    public int getNextTrunk() {
+        return nextTrunk;
+    }
+
+    /**
+     * An iterator over page stream trunk pages.
+     */
+    static class Iterator {
+
+        private PageStore store;
+        private int first;
+        private int next;
+        private int previous;
+        private boolean canDelete;
+        private int current;
+
+        Iterator(PageStore store, int first) {
+            this.store = store;
+            this.next = first;
+        }
+
+        int getCurrentPageId() {
+            return current;
+        }
+
+        /**
+         * Get the next trunk page or null if no next trunk page.
+         *
+         * @return the next trunk page or null
+         */
+        PageStreamTrunk next() throws SQLException {
+            canDelete = false;
+            if (first == 0) {
+                first = next;
+            } else if (first == next) {
+                return null;
+            }
+            if (next == 0 || next >= store.getPageCount()) {
+                return null;
+            }
+            Page p;
+            current = next;
+            try {
+                p = store.getPage(next);
+            } catch (SQLException e) {
+                if (e.getErrorCode() != ErrorCode.FILE_CORRUPTED_1) {
+                    // wrong checksum means end of stream
+                    throw e;
+                }
+                return null;
+            }
+            if (p == null || p instanceof PageStreamTrunk || p instanceof PageStreamData) {
+                canDelete = true;
+            }
+            if (!(p instanceof PageStreamTrunk)) {
+                return null;
+            }
+            PageStreamTrunk t = (PageStreamTrunk) p;
+            if (previous > 0 && t.parent != previous) {
+                return null;
+            }
+            previous = next;
+            next = t.nextTrunk;
+            return t;
+        }
+
+        /**
+         * Check if the current page can be deleted. It can if it's empty, a
+         * stream trunk, or a stream data page.
+         *
+         * @return true if it can be deleted
+         */
+        boolean canDelete() {
+            return canDelete;
+        }
+
+    }
+
+    public boolean canRemove() {
+        return true;
+    }
+
 }