new experimental page store

h2/src/main/org/h2/engine/Database.java

@@ -63,6 +63,7 @@ import org.h2.util.FileUtils;
 import org.h2.util.IntHashMap;
 import org.h2.util.NetUtils;
 import org.h2.util.ObjectArray;
+import org.h2.util.ObjectUtils;
 import org.h2.util.SmallLRUCache;
 import org.h2.util.StringUtils;
 import org.h2.util.TempFileDeleter;
@@ -100,6 +101,7 @@ public class Database implements DataHandler {
     private final HashMap aggregates = new HashMap();
     private final HashMap comments = new HashMap();
     private IntHashMap tableMap = new IntHashMap();
+    private final HashMap databaseObjects = new HashMap();
 
     private final Set userSessions = Collections.synchronizedSet(new HashSet());
     private Session exclusiveSession;
@@ -519,9 +521,7 @@ public class Database implements DataHandler {
             isReconnectNeeded();
             if (SysProperties.PAGE_STORE) {
                 PageStore store = getPageStore();
-                if (!store.isNew()) {
-                    store.recover(true);
-                }
+                store.recover();
             }
             if (FileUtils.exists(dataFileName)) {
                 lobFilesInDirectories &= !ValueLob.existsLobFile(getDatabasePath());
@@ -579,7 +579,7 @@ public class Database implements DataHandler {
         cols.add(new Column("SQL", Value.STRING));
         int headPos = 0;
         if (pageStore != null) {
-            headPos = pageStore.getSystemRootPageId();
+            headPos = pageStore.getMetaTableHeadPos();
         }
         meta = mainSchema.createTable("SYS", 0, cols, persistent, false, headPos);
         tableMap.put(0, meta);
@@ -606,14 +606,6 @@ public class Database implements DataHandler {
             MetaRecord rec = (MetaRecord) records.get(i);
             rec.execute(this, systemSession, eventListener);
         }
-        if (pageStore != null) {
-            if (!pageStore.isNew()) {
-                getPageStore().recover(false);
-                if (!readOnly) {
-                    pageStore.checkpoint();
-                }
-            }
-        }
         // try to recompile the views that are invalid
         recompileInvalidViews(systemSession);
         starting = false;
@@ -630,6 +622,10 @@ public class Database implements DataHandler {
         traceSystem.getTrace(Trace.DATABASE).info("opened " + databaseName);
     }
 
+    public Schema getMainSchema() {
+        return mainSchema;
+    }
+
     private void startServer(String key) throws SQLException {
         server = Server.createTcpServer(new String[]{
                 "-tcpPort", "0",
@@ -764,13 +760,12 @@ public class Database implements DataHandler {
     }
 
     private synchronized void addMeta(Session session, DbObject obj) throws SQLException {
-        if (obj.getTemporary()) {
-            return;
-        }
+        int id = obj.getId();
+        if (id > 0 && !starting && !obj.getTemporary()) {
             Row r = meta.getTemplateRow();
             MetaRecord rec = new MetaRecord(obj);
             rec.setRecord(r);
-        objectIds.set(obj.getId());
+            objectIds.set(id);
             meta.lock(session, true, true);
             meta.addRow(session, r);
             if (isMultiVersion()) {
@@ -778,6 +773,10 @@ public class Database implements DataHandler {
                 session.log(meta, UndoLogRecord.INSERT, r);
             }
         }
+        if (SysProperties.PAGE_STORE && id > 0) {
+            databaseObjects.put(ObjectUtils.getInteger(id), obj);
+        }
+    }
 
     /**
      * Remove the given object from the meta data.
@@ -786,6 +785,7 @@ public class Database implements DataHandler {
      * @param id the id of the object to remove
      */
     public synchronized void removeMeta(Session session, int id) throws SQLException {
+        if (id > 0 && !starting) {
             SearchRow r = meta.getTemplateSimpleRow(false);
             r.setValue(0, ValueInt.get(id));
             Cursor cursor = metaIdIndex.find(session, r, r);
@@ -804,6 +804,10 @@ public class Database implements DataHandler {
                 }
             }
         }
+        if (SysProperties.PAGE_STORE) {
+            databaseObjects.remove(ObjectUtils.getInteger(id));
+        }
+    }
 
     private HashMap getMap(int type) {
         switch (type) {
@@ -842,9 +846,7 @@ public class Database implements DataHandler {
             checkWritingAllowed();
         }
         obj.getSchema().add(obj);
-        if (id > 0 && !starting) {
         addMeta(session, obj);
-        }
         if (obj instanceof TableData) {
             tableMap.put(id, obj);
         }
@@ -872,9 +874,7 @@ public class Database implements DataHandler {
         if (SysProperties.CHECK && map.get(name) != null) {
             Message.throwInternalError("object already exists");
         }
-        if (id > 0 && !starting) {
         addMeta(session, obj);
-        }
         map.put(name, obj);
     }
 
@@ -1615,6 +1615,7 @@ public class Database implements DataHandler {
             removeDatabaseObject(session, comment);
         }
         obj.getSchema().remove(obj);
+        if (!starting) {
             String invalid;
             if (SysProperties.OPTIMIZE_DROP_DEPENDENCIES) {
                 Table t = getDependentTable(obj, null);
@@ -1626,8 +1627,9 @@ public class Database implements DataHandler {
                 obj.getSchema().add(obj);
                 throw Message.getSQLException(ErrorCode.CANNOT_DROP_2, new String[] { obj.getSQL(), invalid });
             }
-        int id = obj.getId();
             obj.removeChildrenAndResources(session);
+        }
+        int id = obj.getId();
         removeMeta(session, id);
         if (obj instanceof TableData) {
             tableMap.remove(id);
@@ -2154,6 +2156,8 @@ public class Database implements DataHandler {
             if (add) {
                 objectIds.set(m.getId());
                 m.execute(this, systemSession, eventListener);
+            } else {
+                m.undo(this, systemSession, eventListener);
             }
         }
     }
@@ -2253,4 +2257,14 @@ public class Database implements DataHandler {
         }
     }
 
+    /**
+     * Get a database object.
+     *
+     * @param id the object id
+     * @return the database object
+     */
+    DbObject getDbObject(int id) {
+        return (DbObject) databaseObjects.get(ObjectUtils.getInteger(id));
+    }
+
 }

h2/src/main/org/h2/engine/MetaRecord.java

@@ -8,12 +8,12 @@ package org.h2.engine;
 
 import java.sql.SQLException;
 import java.util.Comparator;
-
 import org.h2.api.DatabaseEventListener;
 import org.h2.command.Prepared;
 import org.h2.message.Message;
 import org.h2.message.Trace;
 import org.h2.result.SearchRow;
+import org.h2.schema.SchemaObject;
 import org.h2.util.ObjectArray;
 import org.h2.value.ValueInt;
 import org.h2.value.ValueString;
@@ -95,6 +95,37 @@ public class MetaRecord {
         }
     }
 
+    /**
+     * Undo a metadata change.
+     *
+     * @param db the database
+     * @param systemSession the system session
+     * @param listener the database event listener
+     */
+    void undo(Database db, Session systemSession, DatabaseEventListener listener) throws SQLException {
+        try {
+            DbObject obj = db.getDbObject(id);
+            // null if it was already removed
+            // (a identity sequence is removed when the table is removed)
+            if (obj != null) {
+                if (obj instanceof SchemaObject) {
+                    db.removeSchemaObject(systemSession, (SchemaObject) obj);
+                } else {
+                    db.removeDatabaseObject(systemSession, obj);
+                }
+            }
+        } catch (Exception e) {
+            SQLException s = Message.addSQL(Message.convert(e), sql);
+            db.getTrace(Trace.DATABASE).error(sql, s);
+            if (listener != null) {
+                listener.exceptionThrown(s, sql);
+                // continue startup in this case
+            } else {
+                throw s;
+            }
+        }
+    }
+
     public int getId() {
         return id;
     }

h2/src/main/org/h2/index/PageBtreeIndex.java

@@ -31,6 +31,7 @@ public class PageBtreeIndex extends BaseIndex {
     private TableData tableData;
     private int headPos;
     private long rowCount;
+    private boolean needRebuild;
 
     public PageBtreeIndex(TableData table, int id, String indexName, IndexColumn[] columns,
             IndexType indexType, int headPos) throws SQLException {
@@ -46,14 +47,16 @@ public class PageBtreeIndex extends BaseIndex {
         }
         this.store = database.getPageStore();
         if (headPos == Index.EMPTY_HEAD) {
-            // new table
+            // new index
+            needRebuild = true;
             headPos = store.allocatePage();
             PageBtreeLeaf root = new PageBtreeLeaf(this, headPos, Page.ROOT, store.createDataPage());
             store.updateRecord(root, true, root.data);
-        } else if (store.isNew()) {
-            // the system table for a new database
-            PageBtreeLeaf root = new PageBtreeLeaf(this, headPos, Page.ROOT, store.createDataPage());
-            store.updateRecord(root, true, root.data);
+            int test;
+//        } else if (store.isNew()) {
+//            // the system table for a new database
+//            PageBtreeLeaf root = new PageBtreeLeaf(this, headPos, Page.ROOT, store.createDataPage());
+//            store.updateRecord(root, true, root.data);
         } else {
             rowCount = getPage(headPos).getRowCount();
             int reuseKeysIfManyDeleted;
@@ -175,7 +178,7 @@ public class PageBtreeIndex extends BaseIndex {
     }
 
     public boolean needRebuild() {
-        return false;
+        return needRebuild;
     }
 
     public void remove(Session session, Row row) throws SQLException {

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -234,4 +234,8 @@ class PageBtreeLeaf extends PageBtree {
         next.nextPage(cursor, getRow(entryCount - 1));
     }
 
+    public String toString() {
+        return "page[" + getPos() + "] btree leaf table:" + index.getId() + " entries:" + entryCount;
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -300,4 +300,9 @@ class PageBtreeNode extends PageBtree {
         cursor.setCurrent(leaf, 0);
     }
 
+    public String toString() {
+        return "page[" + getPos() + "] btree node table:" + index.getId() + " entries:" + entryCount;
+    }
+
+
 }
\ No newline at end of file

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -66,7 +66,7 @@ class PageDataLeaf extends PageData {
         if (tableId != index.getId()) {
             throw Message.getSQLException(ErrorCode.FILE_CORRUPTED_1,
                     "page:" + getPageId() + " expected table:" + index.getId() +
-                    "got:" + tableId);
+                    " got:" + tableId + " type:" + type);
         }
         entryCount = data.readShortInt();
         offsets = new int[entryCount];
@@ -351,4 +351,8 @@ class PageDataLeaf extends PageData {
         return data;
     }
 
+    public String toString() {
+        return "page[" + getPos() + "] data leaf table:" + index.getId() + " entries:" + entryCount;
+    }
+
 }

h2/src/main/org/h2/index/PageDataLeafOverflow.java

@@ -129,4 +129,8 @@ public class PageDataLeafOverflow extends Record {
         store.writePage(getPos(), overflow);
     }
 
+    public String toString() {
+        return "page[" + getPos() + "] data leaf overflow prev:" + previous + " next:" + next;
+    }
+
 }

h2/src/main/org/h2/index/PageDataNode.java

@@ -26,9 +26,9 @@ import org.h2.store.DataPage;
  */
 class PageDataNode extends PageData {
 
-    private final static int ENTRY_START = 15;
+    private static final int ENTRY_START = 15;
 
-    private final static int ENTRY_LENGTH = 8;
+    private static final int ENTRY_LENGTH = 8;
 
     /**
      * The page ids of the children.
@@ -276,4 +276,8 @@ class PageDataNode extends PageData {
         childPageIds = newChildPageIds;
     }
 
+    public String toString() {
+        return "page[" + getPos() + "] data node table:" + index.getId() + " entries:" + entryCount;
+    }
+
 }

h2/src/main/org/h2/index/PageScanIndex.java

@@ -53,13 +53,17 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
             headPos = store.allocatePage();
             PageDataLeaf root = new PageDataLeaf(this, headPos, Page.ROOT, store.createDataPage());
             store.updateRecord(root, true, root.data);
-        } else if (store.isNew()) {
-            // the system table for a new database
-            PageDataLeaf root = new PageDataLeaf(this, headPos, Page.ROOT, store.createDataPage());
-            store.updateRecord(root, true, root.data);
+            int test;
+//        } else if (store.isNew()) {
+//            // the system table for a new database
+//            PageDataLeaf root = new PageDataLeaf(this, headPos, Page.ROOT, store.createDataPage());
+//            store.updateRecord(root, true, root.data);
         } else {
-            lastKey = getPage(headPos).getLastKey();
-            rowCount = getPage(headPos).getRowCount();
+            PageData root = getPage(headPos);
+            lastKey = root.getLastKey();
+            rowCount = root.getRowCount();
+            // could have been created before, but never committed
+            store.updateRecord(root, false, null);
             int reuseKeysIfManyDeleted;
         }
         this.headPos = headPos;
@@ -127,6 +131,10 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
     PageData getPage(int id) throws SQLException {
         Record rec = store.getRecord(id);
         if (rec != null) {
+            if (rec instanceof PageDataLeafOverflow) {
+                int test;
+                System.out.println("stop");
+            }
             return (PageData) rec;
         }
         DataPage data = store.readPage(id);

h2/src/main/org/h2/store/PageFreeList.java

 /*
- * Copyright 2004-2009 H2 Group. Multiple-Licensed under the H2 License,
- * Version 1.0, and under the Eclipse Public License, Version 1.0
- * (http://h2database.com/html/license.html).
- * Initial Developer: H2 Group
+ * Copyright 2004-2009 H2 Group. Multiple-Licensed under the H2 License, Version
+ * 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html). Initial Developer: H2 Group
  */
 package org.h2.store;
 
@@ -10,82 +9,118 @@ import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
 import org.h2.index.Page;
 import org.h2.message.Message;
-import org.h2.util.IntArray;
+import org.h2.util.BitField;
 
 /**
- * The list of free pages of a page store.
- * The format of a free list trunk page is:
- * <ul><li>0-3: parent page id (always 0)
- * </li><li>4-4: page type
- * </li><li>5-8: the next page (if there are more) or number of entries
- * </li><li>9-remainder: data (4 bytes each entry)
- * </li></ul>
+ * The list of free pages of a page store. The format of a free list trunk page
+ * is:
+ * <ul>
+ * <li>0-3: parent page id (always 0)</li>
+ * <li>4-4: page type</li>
+ * <li>5-remainder: data</li>
+ * </ul>
  */
 public class PageFreeList extends Record {
 
+    private static final int DATA_START = 5;
+
     private final PageStore store;
-    private final DataPage data;
-    private final IntArray array = new IntArray();
-    private int nextPage;
+    private final BitField used = new BitField();
+    private final int firstAddressed;
+    private final int pageCount;
+    private final int nextPage;
+    private boolean full;
+    private DataPage data;
 
-    PageFreeList(PageStore store, int pageId, int nextPage) {
+    PageFreeList(PageStore store, int pageId, int firstAddressed) {
         setPos(pageId);
-        this.data = store.createDataPage();
         this.store = store;
-        this.nextPage = nextPage;
+        this.firstAddressed = firstAddressed;
+        pageCount = (store.getPageSize() - DATA_START) * 8;
+        for (int i = firstAddressed; i <= pageId; i++) {
+            used.set(getAddress(i));
+        }
+        nextPage = firstAddressed + pageCount;
+    }
+
+    private int getAddress(int pageId) {
+        return pageId - firstAddressed;
     }
 
     /**
      * Allocate a page from the free list.
      *
-     * @return the page
+     * @return the page, or -1 if all pages are used
      */
     int allocate() throws SQLException {
+        if (full) {
+            PageFreeList next = getNext();
+            if (next == null) {
+                return -1;
+            }
+            return next.allocate();
+        }
+        int free = used.nextClearBit(0);
+        if (free > pageCount) {
+            full = true;
+            return allocate();
+        }
+        used.set(free);
         store.updateRecord(this, true, data);
-        int size = array.size();
-        if (size > 0) {
-            int x = array.get(size - 1);
-            array.remove(size - 1);
-            return x;
+        return free + firstAddressed;
     }
-        store.removeRecord(getPos());
-        // no more free pages in this list:
-        // set the next page (may be 0, meaning no free pages)
-        store.setFreeListRootPage(nextPage, true, 0);
-        // and then return the page itself
-        return getPos();
+
+    /**
+     * Allocate a page at the end of the file
+     *
+     * @param min the minimum page number
+     * @return the page id
+     */
+    int allocateAtEnd(int min) throws SQLException {
+        int pos = Math.max(min, getLastUsed() + 1);
+        return allocate(pos);
     }
 
-    private int getMaxSize() {
-        return (store.getPageSize() - 9) / DataPage.LENGTH_INT;
+    public int getLastUsed() throws SQLException {
+        if (nextPage < store.getPageCount()) {
+            PageFreeList next = getNext();
+            return next.getLastUsed();
+        }
+        return used.getLastSetBit() + firstAddressed;
+    }
+
+    private PageFreeList getNext() throws SQLException {
+        PageFreeList next = (PageFreeList) store.getRecord(nextPage);
+        if (next == null) {
+            if (nextPage < store.getPageCount()) {
+                next = new PageFreeList(store, nextPage, nextPage);
+                next.read();
+                store.updateRecord(next, false, null);
+            }
+        }
+        return next;
     }
 
     /**
-     * Read the page from the disk.
+     * Mark a page as used.
+     *
+     * @param pos the page id
+     * @return the page id, or -1
      */
-    void read() throws SQLException {
-        data.reset();
-        store.readPage(getPos(), data);
-        int p = data.readInt();
-        int t = data.readByte();
-        boolean last = (t & Page.FLAG_LAST) != 0;
-        t &= ~Page.FLAG_LAST;
-        if (t != Page.TYPE_FREE_LIST || p != 0) {
-            throw Message.getSQLException(
-                    ErrorCode.FILE_CORRUPTED_1,
-                    "pos:" + getPos() + " type:" + t + " parent:" + p +
-                    " expected type:" + Page.TYPE_FREE_LIST);
-        }
-        int size;
-        if (last) {
-            nextPage = 0;
-            size = data.readInt();
+    int allocate(int pos) throws SQLException {
+        if (pos - firstAddressed > pageCount) {
+            PageFreeList next = getNext();
+            if (next == null) {
+                return -1;
+            }
+            return next.allocate(pos);
         } else {
-            nextPage = data.readInt();
-            size = getMaxSize();
+            int idx = pos - firstAddressed;
+            if (idx >= 0 && !used.get(idx)) {
+                used.set(pos - firstAddressed);
+                store.updateRecord(this, true, data);
             }
-        for (int i = 0; i < size; i++) {
-            array.add(data.readInt());
+            return pos;
         }
     }
 
@@ -95,15 +130,28 @@ public class PageFreeList extends Record {
      * @param pageId the page id to add
      */
     void free(int pageId) throws SQLException {
+        full = false;
+        used.clear(pageId - firstAddressed);
         store.updateRecord(this, true, data);
-        if (array.size() < getMaxSize()) {
-            array.add(pageId);
-        } else {
-            // this page is full:
-            // the freed page is the next list
-            this.nextPage = pageId;
-            // set the next page
-            store.setFreeListRootPage(pageId, false, getPos());
+    }
+
+    /**
+     * Read the page from the disk.
+     */
+    void read() throws SQLException {
+        data = store.createDataPage();
+        store.readPage(getPos(), data);
+        int p = data.readInt();
+        int t = data.readByte();
+        if (t == Page.TYPE_EMPTY) {
+            return;
+        }
+        if (t != Page.TYPE_FREE_LIST || p != 0) {
+            throw Message.getSQLException(ErrorCode.FILE_CORRUPTED_1, "pos:" + getPos() + " type:" + t + " parent:" + p
+                    + " expected type:" + Page.TYPE_FREE_LIST);
+        }
+        for (int i = 0; i < pageCount; i += 8) {
+            used.setByte(i, data.readByte());
         }
     }
 
@@ -112,20 +160,12 @@ public class PageFreeList extends Record {
     }
 
     public void write(DataPage buff) throws SQLException {
-        data.reset();
+        data = store.createDataPage();
         data.writeInt(0);
         int type = Page.TYPE_FREE_LIST;
-        if (nextPage == 0) {
-            type |= Page.FLAG_LAST;
-        }
         data.writeByte((byte) type);
-        if (nextPage != 0) {
-            data.writeInt(nextPage);
-        } else {
-            data.writeInt(array.size());
-        }
-        for (int i = 0; i < array.size(); i++) {
-            data.writeInt(array.get(i));
+        for (int i = 0; i < pageCount; i += 8) {
+            data.writeByte((byte) used.getByte(i));
         }
         store.writePage(getPos(), data);
     }

h2/src/main/org/h2/store/PageInputStream.java

@@ -12,6 +12,7 @@ import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
 import org.h2.index.Page;
 import org.h2.message.Message;
+import org.h2.message.Trace;
 
 /**
  * An output stream that writes into a page store.
@@ -25,6 +26,7 @@ import org.h2.message.Message;
 public class PageInputStream extends InputStream {
 
     private PageStore store;
+    private final Trace trace;
     private int parentPage;
     private int type;
     private int nextPage;
@@ -35,6 +37,7 @@ public class PageInputStream extends InputStream {
 
     public PageInputStream(PageStore store, int parentPage, int headPage, int type) {
         this.store = store;
+        this.trace = store.getTrace();
         this.parentPage = parentPage;
         this.type = type;
         nextPage = headPage;
@@ -108,6 +111,9 @@ public class PageInputStream extends InputStream {
                 nextPage = page.readInt();
                 remaining = store.getPageSize() - page.length();
             }
+            if (trace.isDebugEnabled()) {
+                trace.debug("pageIn.readPage " + parentPage + " next:" + nextPage);
+            }
         } catch (SQLException e) {
             throw Message.convertToIOException(e);
         }

h2/src/main/org/h2/store/PageLog.java

@@ -74,6 +74,7 @@ public class PageLog {
     private DataPage data;
     private long operation;
     private BitField undo = new BitField();
+    private int[] reservedPages = new int[2];
 
     PageLog(PageStore store, int firstPage) {
         this.store = store;
@@ -215,15 +216,29 @@ e.printStackTrace();
             if (trace.isDebugEnabled()) {
                 trace.debug("log undo " + pageId);
             }
+            undo.set(pageId);
+            reservePages(2);
             out.write(UNDO);
             out.writeInt(pageId);
             out.write(page.getBytes(), 0, store.getPageSize());
-            undo.set(pageId);
         } catch (IOException e) {
             throw Message.convertIOException(e, null);
         }
     }
 
+    private void reservePages(int pageCount) throws SQLException {
+        int testIfRequired;
+        if (pageCount > reservedPages.length) {
+            reservedPages = new int[pageCount];
+        }
+        for (int i = 0; i < pageCount; i++) {
+            reservedPages[i] = store.allocatePage();
+        }
+        for (int i = 0; i < pageCount; i++) {
+            store.freePage(reservedPages[i]);
+        }
+    }
+
     /**
      * Mark a committed transaction.
      *
@@ -232,6 +247,7 @@ e.printStackTrace();
     void commit(Session session) throws SQLException {
         try {
             trace.debug("log commit");
+            reservePages(1);
             out.write(COMMIT);
             out.writeInt(session.getId());
             if (store.getDatabase().getLog().getFlushOnEachCommit()) {
@@ -259,13 +275,17 @@ e.printStackTrace();
             int todoLogPosShouldBeLong;
             session.addLogPos(0, (int) operation);
             row.setLastLog(0, (int) operation);
+
+            data.reset();
+            int todoWriteIntoOutputDirectly;
+            row.write(data);
+
+            reservePages(1 + data.length() / store.getPageSize());
+
             out.write(add ? ADD : REMOVE);
             out.writeInt(session.getId());
             out.writeInt(tableId);
             out.writeInt(row.getPos());
-            data.reset();
-            int todoWriteIntoOutputDirectly;
-            row.write(data);
             out.writeInt(data.length());
             out.write(data.getBytes(), 0, data.length());
         } catch (IOException e) {