new experimental page store

h2/src/main/org/h2/command/ddl/AlterTableAlterColumn.java

@@ -240,7 +240,7 @@ public class AlterTableAlterColumn extends SchemaCommand {
         // with the old table
         // still need a new id because using 0 would mean: the new table tries
         // to use the rows of the table 0 (the meta table)
-        int id = -1;
+        int id = db.allocateObjectId(true, true);
         TableData newTable = getSchema().createTable(tempName, id, newColumns, persistent, false, Index.EMPTY_HEAD);
         newTable.setComment(table.getComment());
         StringBuffer buff = new StringBuffer(newTable.getCreateSQL());

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -201,7 +201,7 @@ class PageBtreeNode extends PageBtree {
             return false;
         }
         // this child is now empty
-        index.getPageStore().freePage(page.getPageId());
+        index.getPageStore().freePage(page.getPageId(), true, page.data);
         if (entryCount < 1) {
             // no more children - this page is empty as well
             return true;

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -6,6 +6,7 @@
  */
 package org.h2.index;
 import java.sql.SQLException;
+import java.util.Arrays;
 
 import org.h2.constant.ErrorCode;
 import org.h2.engine.Session;
@@ -92,8 +93,6 @@ class PageDataLeaf extends PageData {
     int addRow(Row row) throws SQLException {
         int rowLength = row.getByteCount(data);
         int pageSize = index.getPageStore().getPageSize();
-        // TODO currently the order is important
-        // TODO and can only add at the end
         int last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
         if (entryCount > 0 && last - rowLength < start + KEY_OFFSET_PAIR_LENGTH) {
             int todoSplitAtLastInsertionPoint;
@@ -112,11 +111,15 @@ class PageDataLeaf extends PageData {
             System.arraycopy(keys, 0, newKeys, 0, x);
             System.arraycopy(rows, 0, newRows, 0, x);
             if (x < entryCount) {
-                System.arraycopy(offsets, x, newOffsets, x + 1, entryCount - x);
+                for (int j = x; j < entryCount; j++) {
+                    newOffsets[j + 1] = offsets[j] - rowLength;
+                }
                 System.arraycopy(keys, x, newKeys, x + 1, entryCount - x);
                 System.arraycopy(rows, x, newRows, x + 1, entryCount - x);
             }
         }
+        last = x == 0 ? pageSize : offsets[x - 1];
+        offset = last - rowLength;
         entryCount++;
         start += KEY_OFFSET_PAIR_LENGTH;
         newOffsets[x] = offset;
@@ -175,7 +178,11 @@ class PageDataLeaf extends PageData {
         System.arraycopy(offsets, 0, newOffsets, 0, i);
         System.arraycopy(keys, 0, newKeys, 0, i);
         System.arraycopy(rows, 0, newRows, 0, i);
-        System.arraycopy(offsets, i + 1, newOffsets, i, entryCount - i);
+        int startNext = i < entryCount - 1 ? offsets[i + 1] : index.getPageStore().getPageSize();
+        int rowLength = offsets[i] - startNext;
+        for (int j = i; j < entryCount; j++) {
+            newOffsets[j] = offsets[j + 1] + rowLength;
+        }
         System.arraycopy(keys, i + 1, newKeys, i, entryCount - i);
         System.arraycopy(rows, i + 1, newRows, i, entryCount - i);
         start -= KEY_OFFSET_PAIR_LENGTH;
@@ -335,6 +342,9 @@ class PageDataLeaf extends PageData {
         if (firstOverflowPageId != 0) {
             data.writeInt(firstOverflowPageId);
         }
+        if (getPos() == 1) {
+            System.out.println("pause");
+        }
         for (int i = 0; i < entryCount; i++) {
             data.writeInt(keys[i]);
             data.writeShortInt(offsets[i]);

h2/src/main/org/h2/index/PageDataNode.java

@@ -196,7 +196,7 @@ class PageDataNode extends PageData {
             return false;
         }
         // this child is now empty
-        index.getPageStore().freePage(page.getPageId());
+        index.getPageStore().freePage(page.getPageId(), true, page.data);
         if (entryCount < 1) {
             // no more children - this page is empty as well
             return true;
@@ -217,6 +217,9 @@ class PageDataNode extends PageData {
             int count = 0;
             for (int i = 0; i < childPageIds.length; i++) {
                 PageData page = index.getPage(childPageIds[i]);
+                if (getPos() == page.getPos()) {
+                    throw Message.throwInternalError("Page it its own child: " + getPageId());
+                }
                 count += page.getRowCount();
             }
             rowCount = count;

h2/src/main/org/h2/index/PageScanIndex.java

@@ -40,7 +40,7 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     public PageScanIndex(TableData table, int id, IndexColumn[] columns, IndexType indexType, int headPos) throws SQLException {
         initBaseIndex(table, id, table.getName() + "_TABLE_SCAN", columns, indexType);
         int test;
-        // trace.setLevel(TraceSystem.DEBUG);
+        trace.setLevel(TraceSystem.DEBUG);
         if (database.isMultiVersion()) {
             int todoMvcc;
         }
@@ -85,9 +85,15 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     public void add(Session session, Row row) throws SQLException {
         if (row.getPos() == 0) {
             row.setPos(++lastKey);
+        } else {
+            lastKey = Math.max(lastKey, row.getPos() + 1);
         }
         if (trace.isDebugEnabled()) {
-            trace.debug("add " + row.getPos());
+            int test;
+            if (table.getId() == -1) {
+                System.out.println("pause");
+            }
+            trace.debug("add table:" + table.getId() + " " + row);
         }
         if (tableData.getContainsLargeObject()) {
             for (int i = 0; i < row.getColumnCount(); i++) {
@@ -191,6 +197,9 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     public void remove(Session session, Row row) throws SQLException {
         if (trace.isDebugEnabled()) {
             trace.debug("remove " + row.getPos());
+            if (table.getId() == -1) {
+                System.out.println("pause");
+            }
         }
         if (tableData.getContainsLargeObject()) {
             for (int i = 0; i < row.getColumnCount(); i++) {

h2/src/main/org/h2/store/PageInputStream.java

@@ -112,7 +112,7 @@ public class PageInputStream extends InputStream {
                 remaining = store.getPageSize() - page.length();
             }
             if (trace.isDebugEnabled()) {
-                trace.debug("pageIn.readPage " + parentPage + " next:" + nextPage);
+                // trace.debug("pageIn.readPage " + parentPage + " next:" + nextPage);
             }
         } catch (SQLException e) {
             throw Message.convertToIOException(e);

h2/src/main/org/h2/store/PageLog.java

@@ -234,7 +234,7 @@ e.printStackTrace();
             reservedPages[i] = store.allocatePage();
         }
         for (int i = 0; i < pageCount; i++) {
-            store.freePage(reservedPages[i]);
+            store.freePage(reservedPages[i], false, null);
         }
     }
 

h2/src/main/org/h2/store/PageStore.java

@@ -168,7 +168,7 @@ public class PageStore implements CacheWriter {
         this.database = database;
         trace = database.getTrace(Trace.PAGE_STORE);
         int test;
-// trace.setLevel(TraceSystem.DEBUG);
+trace.setLevel(TraceSystem.DEBUG);
         this.cacheSize = cacheSizeDefault;
         String cacheType = database.getCacheType();
         if (Cache2Q.TYPE_NAME.equals(cacheType)) {
@@ -407,6 +407,10 @@ public class PageStore implements CacheWriter {
         synchronized (database) {
             Record record = (Record) obj;
             if (trace.isDebugEnabled()) {
+                int test;
+                if (record.getPos() == 1) {
+                    System.out.println("pause");
+                }
                 trace.debug("writeBack " + record);
             }
             int todoRemoveParameter;
@@ -426,6 +430,10 @@ public class PageStore implements CacheWriter {
         synchronized (database) {
             if (trace.isDebugEnabled()) {
                 if (!record.isChanged()) {
+                    int test;
+                    if(record.getPos() == 1) {
+                        System.out.println("pause");
+                    }
                     trace.debug("updateRecord " + record.toString());
                 }
             }
@@ -481,7 +489,7 @@ public class PageStore implements CacheWriter {
         if (trace.isDebugEnabled()) {
             trace.debug("allocated " + id + " atEnd:" + atEnd);
         }
-        if (id > pageCount) {
+        if (id >= pageCount) {
             increaseFileSize(INCREMENT_PAGES);
         }
         return id;
@@ -498,8 +506,10 @@ public class PageStore implements CacheWriter {
      * Add a page to the free list.
      *
      * @param pageId the page id
+     * @param logUndo if an undo entry need to be logged
+     * @param old the old data (if known)
      */
-    public void freePage(int pageId) throws SQLException {
+    public void freePage(int pageId, boolean logUndo, DataPage old) throws SQLException {
         if (trace.isDebugEnabled()) {
             trace.debug("freePage " + pageId);
         }
@@ -507,7 +517,13 @@ public class PageStore implements CacheWriter {
         getFreeList().free(pageId);
         if (recoveryRunning) {
             writePage(pageId, createDataPage());
+        } else if (logUndo) {
+            if (old == null) {
+                old = readPage(pageId);
             }
+            getLog().addUndo(pageId, old);
+        }
+
     }
 
     /**
@@ -626,9 +642,8 @@ public class PageStore implements CacheWriter {
      * @param undo true if the undo step should be run
      */
     private void recover(boolean undo) throws SQLException {
-        if (undo) {
-            trace.debug("log recover");
-        } else {
+        trace.debug("log recover #" + undo);
+        if (!undo) {
             openMetaIndex();
             readMetaData();
         }
@@ -849,6 +864,9 @@ public class PageStore implements CacheWriter {
             meta = table.getScanIndex(database.getSystemSession());
         } else {
             PageScanIndex p = (PageScanIndex) metaObjects.get(ObjectUtils.getInteger(parent));
+            if (p == null) {
+                throw Message.throwInternalError("parent not found:" + parent);
+            }
             TableData table = (TableData) p.getTable();
             Column[] tableCols = table.getColumns();
             Column[] cols = new Column[columns.length];
@@ -861,6 +879,11 @@ public class PageStore implements CacheWriter {
         metaObjects.put(ObjectUtils.getInteger(id), meta);
     }
 
+    /**
+     * Add the meta data of an index.
+     *
+     * @param index the index to add
+     */
     public void addMeta(Index index) throws SQLException {
         int type = index instanceof PageScanIndex ? META_TYPE_SCAN_INDEX : META_TYPE_BTREE_INDEX;
         Column[] columns = index.getColumns();
@@ -883,6 +906,11 @@ public class PageStore implements CacheWriter {
         metaIndex.add(database.getSystemSession(), row);
     }
 
+    /**
+     * Remove the meta data of an index.
+     *
+     * @param index the index to remove
+     */
     public void removeMeta(Index index) throws SQLException {
         Session session = database.getSystemSession();
         Row row = metaIndex.getRow(session, index.getId() + 1);

h2/src/test/org/h2/test/unit/TestPageStore.java

@@ -15,37 +15,180 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.sql.SQLException;
 import java.util.Random;
+import org.h2.constant.SysProperties;
 import org.h2.engine.ConnectionInfo;
 import org.h2.engine.Database;
+import org.h2.index.Cursor;
+import org.h2.index.Index;
+import org.h2.index.IndexType;
 import org.h2.index.Page;
+import org.h2.index.PageScanIndex;
+import org.h2.result.Row;
+import org.h2.schema.Schema;
 import org.h2.store.PageInputStream;
 import org.h2.store.PageOutputStream;
 import org.h2.store.PageStore;
+import org.h2.table.Column;
+import org.h2.table.IndexColumn;
+import org.h2.table.TableData;
 import org.h2.test.TestBase;
 import org.h2.util.IntArray;
+import org.h2.util.ObjectArray;
+import org.h2.value.Value;
+import org.h2.value.ValueInt;
 
 /**
  * Test the page store.
  */
 public class TestPageStore extends TestBase {
 
+    private Database db;
+    private Schema schema;
+    private TableData table;
+    private Index index;
+
     /**
      * Run just this test.
      *
      * @param a ignored
      */
     public static void main(String[] a) throws Exception {
+        System.setProperty("h2.pageStore", "true");
         TestBase.createCaller().init().test();
     }
 
     public void test() throws Exception {
-        testAllocateFree();
-        testStreamFuzz();
-        testStreamPerformance(false, 1000);
+        testScanIndex();
+        testBtreeIndex();
+        // testAllocateFree();
+        // testStreamFuzz();
+        // testStreamPerformance(false, 1000);
         // testPerformance(true, 1000000);
         // testPerformance(false, 1000000);
     }
 
+    private void testBtreeIndex() throws SQLException {
+        if (!SysProperties.PAGE_STORE) {
+            return;
+        }
+        deleteDb("pageStore");
+        String fileName = getTestDir("/pageStore");
+        new File(fileName).delete();
+        File f = new File(fileName + ".dat");
+        f.delete();
+        db = getDatabase();
+        PageStore store = new PageStore(db, fileName, "rw", 8192);
+        store.setPageSize(1024);
+        store.open();
+        openBtreeIndex();
+        Row row;
+        for (int i = 10; i < 100; i += 10) {
+            row = table.getTemplateRow();
+            row.setValue(0, ValueInt.get(i));
+            row.setPos(i);
+            index.add(db.getSystemSession(), row);
+        }
+        row = table.getTemplateRow();
+        row.setValue(0, ValueInt.get(60));
+        row.setPos(60);
+        index.remove(db.getSystemSession(), row);
+        row = table.getTemplateRow();
+        row.setValue(0, ValueInt.get(60));
+        row.setPos(60);
+        index.add(db.getSystemSession(), row);
+        store.checkpoint();
+        store.close();
+        store = new PageStore(db, fileName, "rw", 8192);
+        store.open();
+        openBtreeIndex();
+        Cursor cursor = index.find(db.getSystemSession(), null, null);
+        for (int i = 10; i < 100; i += 10) {
+            assertTrue(cursor.next());
+            Row r = cursor.get();
+            assertEquals(i, r.getValue(0).getInt());
+        }
+        assertFalse(cursor.next());
+        store.close();
+        db.shutdownImmediately();
+    }
+
+    private void testScanIndex() throws SQLException {
+        if (!SysProperties.PAGE_STORE) {
+            return;
+        }
+        deleteDb("pageStore");
+        String fileName = getTestDir("/pageStore");
+        new File(fileName).delete();
+        File f = new File(fileName + ".dat");
+        f.delete();
+        db = getDatabase();
+        PageStore store = new PageStore(db, fileName, "rw", 8192);
+        store.setPageSize(1024);
+        store.open();
+        openScanIndex();
+        Row row;
+        for (int i = 10; i < 100; i += 10) {
+            row = table.getTemplateRow();
+            row.setValue(0, ValueInt.get(i));
+            row.setPos(i);
+            index.add(db.getSystemSession(), row);
+        }
+        row = table.getTemplateRow();
+        row.setValue(0, ValueInt.get(60));
+        row.setPos(60);
+        index.remove(db.getSystemSession(), row);
+        row = table.getTemplateRow();
+        row.setValue(0, ValueInt.get(60));
+        row.setPos(60);
+        index.add(db.getSystemSession(), row);
+        store.checkpoint();
+        store.close();
+        store = new PageStore(db, fileName, "rw", 8192);
+        store.open();
+        openScanIndex();
+        Cursor cursor = index.find(db.getSystemSession(), null, null);
+        for (int i = 10; i < 100; i += 10) {
+            assertTrue(cursor.next());
+            Row r = cursor.get();
+            assertEquals(i, r.getValue(0).getInt());
+        }
+        assertFalse(cursor.next());
+        store.close();
+        db.shutdownImmediately();
+    }
+
+    private Database getDatabase() throws SQLException {
+        String name = getTestDir("/pageStore");
+        ConnectionInfo ci = new ConnectionInfo(name);
+        return new Database(name, ci, null);
+    }
+
+    private void openScanIndex() throws SQLException {
+        ObjectArray cols = new ObjectArray();
+        cols.add(new Column("ID", Value.INT));
+        schema = new Schema(db, 0, "", null, true);
+        table = new TableData(schema, "PAGE_INDEX",
+                1, cols, true, false, 100);
+        index = (PageScanIndex) table.getScanIndex(
+                db.getSystemSession());
+    }
+
+    private void openBtreeIndex() throws SQLException {
+        ObjectArray cols = new ObjectArray();
+        cols.add(new Column("ID", Value.INT));
+        schema = new Schema(db, 0, "", null, true);
+        int id = db.allocateObjectId(true, true);
+        table = new TableData(schema, "BTREE_INDEX",
+                id, cols, true, false, 100);
+        id = db.allocateObjectId(true, true);
+        table.addIndex(db.getSystemSession(), "BTREE", id,
+                IndexColumn.wrap(table.getColumns()),
+                IndexType.createNonUnique(true),
+                Index.EMPTY_HEAD, "");
+        index = (PageScanIndex) table.getScanIndex(
+                db.getSystemSession());
+    }
+
     private void testAllocateFree() throws SQLException {
         String fileName = getTestDir("/pageStore");
         new File(fileName).delete();
@@ -63,7 +206,7 @@ public class TestPageStore extends TestBase {
         }
         for (int i = 0; i < size; i++) {
             int id = list.get(i);
-            store.freePage(id);
+            store.freePage(id, false, null);
         }
         for (int i = 0; i < size; i++) {
             int id = store.allocatePage();
@@ -76,12 +219,6 @@ public class TestPageStore extends TestBase {
         f.delete();
     }
 
-    private Database getDatabase() throws SQLException {
-        String name = "mem:pageStore";
-        ConnectionInfo ci = new ConnectionInfo(name);
-        return new Database(name, ci, null);
-    }
-
     private void testStreamPerformance(boolean file, int count) throws Exception {
         String fileName = getTestDir("/pageStore");
         new File(fileName).delete();