Page store: simplify overflow format

h2/src/main/org/h2/index/PageBtreeIndex.java

@@ -52,8 +52,7 @@ public class PageBtreeIndex extends PageIndex {
             // it should not for new tables, otherwise redo of other operations
             // must ensure this page is not used for other things
             store.addMeta(this, session);
-            PageBtreeLeaf root = new PageBtreeLeaf(this, rootPageId, store.createData());
-            root.parentPageId = PageBtree.ROOT;
+            PageBtreeLeaf root = PageBtreeLeaf.create(this, rootPageId, PageBtree.ROOT);
             store.updateRecord(root, true, root.data);
         } else {
             rootPageId = store.getRootPageId(id);
@@ -131,8 +130,7 @@ public class PageBtreeIndex extends PageIndex {
     PageBtree getPage(int id) throws SQLException {
         PageBtree p = (PageBtree) store.getPage(id);
         if (p == null) {
-            Data data = store.createData();
-            PageBtreeLeaf empty = new PageBtreeLeaf(this, id, data);
+            PageBtreeLeaf empty = PageBtreeLeaf.create(this, id, PageBtree.ROOT);
             return empty;
         }
         return p;
@@ -245,8 +243,7 @@ public class PageBtreeIndex extends PageIndex {
     private void removeAllRows() throws SQLException {
         PageBtree root = getPage(rootPageId);
         root.freeChildren();
-        root = new PageBtreeLeaf(this, rootPageId, store.createData());
-        root.parentPageId = PageBtree.ROOT;
+        root = PageBtreeLeaf.create(this, rootPageId, PageBtree.ROOT);
         store.removeRecord(rootPageId);
         store.updateRecord(root, true, null);
         rowCount = 0;

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -18,21 +18,21 @@ import org.h2.store.Page;
 import org.h2.store.PageStore;
 
 /**
- * A b-tree leaf page that contains index data.
- * Format:
- * <ul><li>parent page id (0 for root): int
- * </li><li>page type: byte
- * </li><li>index id: varInt
- * </li><li>entry count: short
- * </li><li>list of offsets: short
- * </li><li>data (key: varLong, value,...)
- * </li></ul>
+ * A b-tree leaf page that contains index data. Format:
+ * <ul>
+ * <li>parent page id (0 for root): int</li>
+ * <li>page type: byte</li>
+ * <li>index id: varInt</li>
+ * <li>entry count: short</li>
+ * <li>list of offsets: short</li>
+ * <li>data (key: varLong, value,...)</li>
+ * </ul>
  */
 public class PageBtreeLeaf extends PageBtree {
 
     private static final int OFFSET_LENGTH = 2;
 
-    PageBtreeLeaf(PageBtreeIndex index, int pageId, Data data) {
+    private PageBtreeLeaf(PageBtreeIndex index, int pageId, Data data) {
         super(index, pageId, data);
     }
 

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -19,17 +19,17 @@ import org.h2.store.PageStore;
 import org.h2.util.MemoryUtils;
 
 /**
- * A b-tree node page that contains index data.
- * Format:
- * <ul><li>parent page id (0 for root): int
- * </li><li>page type: byte
- * </li><li>index id: varInt
- * </li><li>count of all children (-1 if not known): int
- * </li><li>entry count: short
- * </li><li>rightmost child page id: int
- * </li><li>entries (child page id: int, offset: short)
- * The row contains the largest key of the respective child, meaning
- * row[0] contains the largest key of child[0].
+ * A b-tree node page that contains index data. Format:
+ * <ul>
+ * <li>parent page id (0 for root): int</li>
+ * <li>page type: byte</li>
+ * <li>index id: varInt</li>
+ * <li>count of all children (-1 if not known): int</li>
+ * <li>entry count: short</li>
+ * <li>rightmost child page id: int</li>
+ * <li>entries (child page id: int, offset: short) The row contains the largest
+ * key of the respective child, meaning row[0] contains the largest key of
+ * child[0].
  */
 public class PageBtreeNode extends PageBtree {
 
@@ -70,7 +70,7 @@ public class PageBtreeNode extends PageBtree {
      * @param parentPageId the parent page id
      * @return the page
      */
-    public static PageBtreeNode create(PageBtreeIndex index, int pageId, int parentPageId) {
+    static PageBtreeNode create(PageBtreeIndex index, int pageId, int parentPageId) {
         PageBtreeNode p = new PageBtreeNode(index, pageId, index.getPageStore().createData());
         p.parentPageId = parentPageId;
         p.writeHead();

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -19,17 +19,17 @@ import org.h2.store.Page;
 import org.h2.store.PageStore;
 
 /**
- * A leaf page that contains data of one or multiple rows.
- * Format:
- * <ul><li>parent page id (0 for root): int
- * </li><li>page type: byte
- * </li><li>table id: varInt
- * </li><li>column count: varInt
- * </li><li>entry count: short
- * </li><li>with overflow: the first overflow page id: int
- * </li><li>list of key / offset pairs (key: varLong, offset: shortInt)
- * </li><li>data
- * </li></ul>
+ * A leaf page that contains data of one or multiple rows. Format:
+ * <ul>
+ * <li>parent page id (0 for root): int</li>
+ * <li>page type: byte</li>
+ * <li>table id: varInt</li>
+ * <li>column count: varInt</li>
+ * <li>entry count: short</li>
+ * <li>with overflow: the first overflow page id: int</li>
+ * <li>list of key / offset pairs (key: varLong, offset: shortInt)</li>
+ * <li>data</li>
+ * </ul>
  */
 public class PageDataLeaf extends PageData {
 
@@ -223,7 +223,7 @@ public class PageDataLeaf extends PageData {
                     size = pageSize - PageDataOverflow.START_MORE;
                     next = index.getPageStore().allocatePage();
                 }
-                PageDataOverflow overflow = new PageDataOverflow(index, page, type, previous, next, all, dataOffset, size);
+                PageDataOverflow overflow = PageDataOverflow.create(index.getPageStore(), page, type, previous, next, all, dataOffset, size);
                 index.getPageStore().updateRecord(overflow, true, null);
                 dataOffset += size;
                 remaining -= size;
@@ -295,6 +295,7 @@ public class PageDataLeaf extends PageData {
                     PageDataOverflow page = index.getPageOverflow(next);
                     next = page.readInto(buff);
                 } while (next != 0);
+int checkRequired;
                 overflowRowSize = pageSize + buff.length();
                 buff.setPos(0);
                 r = index.readRow(buff, columnCount);

h2/src/main/org/h2/index/PageDataNode.java

@@ -19,18 +19,18 @@ import org.h2.store.PageStore;
 import org.h2.util.MemoryUtils;
 
 /**
- * A leaf page that contains data of one or multiple rows.
- * Format:
- * <ul><li>parent page id (0 for root): int
- * </li><li>page type: byte
- * </li><li>table id: varInt
- * </li><li>count of all children (-1 if not known): int
- * </li><li>entry count: short
- * </li><li>rightmost child page id: int
- * </li><li>entries (key: varLong, child page id: int)
- * </li></ul>
- * The key is the largest key of the respective child, meaning
- * key[0] is the largest key of child[0].
+ * A leaf page that contains data of one or multiple rows. Format:
+ * <ul>
+ * <li>parent page id (0 for root): int</li>
+ * <li>page type: byte</li>
+ * <li>table id: varInt</li>
+ * <li>count of all children (-1 if not known): int</li>
+ * <li>entry count: short</li>
+ * <li>rightmost child page id: int</li>
+ * <li>entries (key: varLong, child page id: int)</li>
+ * </ul>
+ * The key is the largest key of the respective child, meaning key[0] is the
+ * largest key of child[0].
  */
 public class PageDataNode extends PageData {
 

h2/src/main/org/h2/index/PageDataOverflow.java

@@ -16,34 +16,33 @@ import org.h2.store.Page;
 import org.h2.store.PageStore;
 
 /**
- * Overflow data for a leaf page.
- * Format:
- * <ul><li>0-3: parent page id (0 for root): int
- * </li><li>4-4: page type: byte
- * </li><li>5-8: table id: int
- * </li><li>9-12: if more data: next overflow page id: int
- * </li><li>9-10: else remaining size: short
- * </li><li>data
- * </li></ul>
+ * Overflow data for a leaf page. Format:
+ * <ul>
+ * <li>parent page id (0 for root): int (0-3)</li>
+ * <li>page type: byte (4)</li>
+ * <li>more data: next overflow page id: int (5-8)</li>
+ * <li>last remaining size: short (5-6)</li>
+ * <li>data (9-/7-)</li>
+ * </ul>
  */
 public class PageDataOverflow extends Page {
 
     /**
      * The start of the data in the last overflow page.
      */
-    static final int START_LAST = 11;
+    static final int START_LAST = 7;
 
     /**
      * The start of the data in a overflow page that is not the last one.
      */
-    static final int START_MORE = 13;
+    static final int START_MORE = 9;
 
-    private static final int START_NEXT_OVERFLOW = 9;
+    private static final int START_NEXT_OVERFLOW = 5;
 
     /**
-     * The index.
+     * The page store.
      */
-    private final PageScanIndex index;
+    private final PageStore store;
 
     /**
      * The page type.
@@ -53,38 +52,17 @@ public class PageDataOverflow extends Page {
     /**
      * The parent page (overflow or leaf).
      */
-    private int parentPage;
+    private int parentPageId;
 
     /**
      * The next overflow page, or 0.
      */
     private int nextPage;
 
-    /**
-     * The number of content bytes.
-     */
-    private int size;
-
     private Data data;
 
-    PageDataOverflow(PageScanIndex index, int pageId, int type, int previous, int next, Data allData, int offset, int size) {
-        this.index = index;
-        setPos(pageId);
-        this.type = type;
-        this.parentPage = previous;
-        this.nextPage = next;
-        this.size = size;
-        data = index.getPageStore().createData();
-        data.writeInt(parentPage);
-        data.writeByte((byte) type);
-        data.writeInt(index.getId());
-        if (type == Page.TYPE_DATA_OVERFLOW) {
-            data.writeInt(nextPage);
-        } else {
-            data.writeShortInt(size);
-        }
-        data.write(allData.getBytes(), offset, size);
-    }
+    private int start;
+    private int size;
 
     /**
      * Create an object from the given data page.
@@ -94,8 +72,8 @@ public class PageDataOverflow extends Page {
      * @param data the data page
      * @param offset the offset
      */
-    PageDataOverflow(PageScanIndex index, int pageId, Data data) {
-        this.index = index;
+    private PageDataOverflow(PageStore store, int pageId, Data data) {
+        this.store = store;
         setPos(pageId);
         this.data = data;
     }
@@ -103,39 +81,66 @@ public class PageDataOverflow extends Page {
     /**
      * Read an overflow page.
      *
-     * @param index the index
+     * @param store the page store
      * @param data the data
      * @param pageId the page id
      * @return the page
      */
-    public static Page read(PageScanIndex index, Data data, int pageId) throws SQLException {
-        PageDataOverflow p = new PageDataOverflow(index, pageId, data);
+    public static Page read(PageStore store, Data data, int pageId) throws SQLException {
+        PageDataOverflow p = new PageDataOverflow(store, pageId, data);
         p.read();
         return p;
     }
 
+    /**
+     * Create a new overflow page.
+     *
+     * @param store the page store
+     * @param page the page id
+     * @param type the page type
+     * @param parentPageId the parent page id
+     * @param next the next page or 0
+     * @param all the data
+     * @param offset the offset within the data
+     * @param size the number of bytes
+     * @return the page
+     */
+    static PageDataOverflow create(PageStore store, int page, int type, int parentPageId, int next, Data all, int offset, int size) {
+        Data data = store.createData();
+        PageDataOverflow p = new PageDataOverflow(store, page, data);
+        data.writeInt(parentPageId);
+        data.writeByte((byte) type);
+        if (type == Page.TYPE_DATA_OVERFLOW) {
+            data.writeInt(next);
+        } else {
+            data.writeShortInt(size);
+        }
+        p.start = data.length();
+        data.write(all.getBytes(), offset, size);
+        p.type = type;
+        p.parentPageId = parentPageId;
+        p.nextPage = next;
+        p.size = size;
+        return p;
+    }
+
     /**
      * Read the page.
      */
     private void read() throws SQLException {
         data.reset();
-        parentPage = data.readInt();
+        parentPageId = data.readInt();
         type = data.readByte();
-        int indexId = data.readInt();
-        if (indexId != index.getId()) {
-            throw Message.getSQLException(ErrorCode.FILE_CORRUPTED_1,
-                    "page:" + getPos() + " expected index:" + index.getId() +
-                    " got:" + indexId + " type:" + type);
-        }
         if (type == (Page.TYPE_DATA_OVERFLOW | Page.FLAG_LAST)) {
             size = data.readShortInt();
             nextPage = 0;
         } else if (type == Page.TYPE_DATA_OVERFLOW) {
-            size = index.getPageStore().getPageSize() - START_MORE;
             nextPage = data.readInt();
+            size = store.getPageSize() - data.length();
         } else {
             throw Message.getSQLException(ErrorCode.FILE_CORRUPTED_1, "page:" + getPos() + " type:" + type);
         }
+        start = data.length();
     }
 
     /**
@@ -159,15 +164,33 @@ public class PageDataOverflow extends Page {
     }
 
     public int getByteCount(DataPage dummy) {
-        return index.getPageStore().getPageSize();
+        return store.getPageSize();
+    }
+
+    private void writeHead() {
+        data.writeInt(parentPageId);
+        data.writeByte((byte) type);
     }
 
     public void write(DataPage buff) throws SQLException {
-        index.getPageStore().writePage(getPos(), data);
+        write();
+        store.writePage(getPos(), data);
+    }
+
+
+    private void write() {
+        data.reset();
+        writeHead();
+        if (type == Page.TYPE_DATA_OVERFLOW) {
+            data.writeInt(nextPage);
+        } else {
+            data.writeShortInt(size);
+        }
     }
 
+
     public String toString() {
-        return "page[" + getPos() + "] data leaf overflow parent:" + parentPage + " next:" + nextPage;
+        return "page[" + getPos() + "] data leaf overflow parent:" + parentPageId + " next:" + nextPage;
     }
 
     /**
@@ -177,29 +200,27 @@ public class PageDataOverflow extends Page {
      */
     public int getMemorySize() {
         // double the byte array size
-        return index.getPageStore().getPageSize() >> 1;
+        return store.getPageSize() >> 1;
     }
 
     void setParentPageId(int parent) {
-        this.parentPage = parent;
+        this.parentPageId = parent;
     }
 
     public void moveTo(Session session, int newPos) throws SQLException {
-        PageStore store = index.getPageStore();
-        int start =  type == Page.TYPE_DATA_OVERFLOW ? START_MORE : START_LAST;
-        PageDataOverflow p2 = new PageDataOverflow(index, newPos, type, parentPage, nextPage, data, start, size);
+        PageDataOverflow p2 = PageDataOverflow.create(store, newPos, type, parentPageId, nextPage, data, start, size);
         store.updateRecord(p2, false, null);
         if (nextPage != 0) {
             PageDataOverflow p3 = (PageDataOverflow) store.getPage(nextPage);
             p3.setParentPageId(newPos);
         }
-        Page p = store.getPage(parentPage);
+        Page p = store.getPage(parentPageId);
         if (p == null) {
             throw Message.throwInternalError();
         }
         if (p instanceof PageDataOverflow) {
             PageDataOverflow p1 = (PageDataOverflow) p;
-            p1.setOverflow(newPos);
+            p1.setNext(newPos);
         } else {
             PageDataLeaf p1 = (PageDataLeaf) p;
             p1.setOverflow(newPos);
@@ -207,10 +228,10 @@ public class PageDataOverflow extends Page {
         store.freePage(getPos(), true, data);
     }
 
-    private void setOverflow(int nextPage) throws SQLException {
+    private void setNext(int nextPage) throws SQLException {
         this.nextPage = nextPage;
         data.setInt(START_NEXT_OVERFLOW, nextPage);
-        index.getPageStore().updateRecord(this, true, data);
+        store.updateRecord(this, true, data);
     }
 
 }

h2/src/main/org/h2/store/PageFreeList.java

@@ -15,9 +15,9 @@ import org.h2.util.BitField;
  * The list of free pages of a page store. The format of a free list trunk page
  * is:
  * <ul>
- * <li>0-3: parent page id (always 0)</li>
- * <li>4-4: page type</li>
- * <li>5-remainder: data</li>
+ * <li>parent page id (always 0): int</li>
+ * <li>page type: byte</li>
+ * <li>data (5-)</li>
  * </ul>
  */
 public class PageFreeList extends Page {

h2/src/main/org/h2/store/PageStore.java

@@ -79,15 +79,15 @@ import org.h2.value.ValueString;
  */
 public class PageStore implements CacheWriter {
 
-    // TODO delete: only log the key
+    // TODO fix page format of ..
+    // TODO log: use varInt / varLong
+
     // TODO update: only log the key and changed values
-    // TODO fix page format of data overflow and so on
     // TODO implement checksum; 0 for empty pages
-    // TODO undo log: fully compress empty pages
+    // TODO undo log: (option) fully compress empty pages
     // TODO undo log: don't store empty space between head and data
     // TODO undo log: lzf compression
     // TODO long primary keys don't use delegating index yet (setPos(): int)
-    // TODO replace CRC32
     // TODO maybe remove parent pointer
     // TODO index creation: use less space (ordered, split at insertion point)
     // TODO test running out of disk space (using a special file system)
@@ -467,12 +467,7 @@ public class PageStore implements CacheWriter {
             break;
         }
         case Page.TYPE_DATA_OVERFLOW: {
-            int indexId = data.readInt();
-            PageScanIndex index = (PageScanIndex) metaObjects.get(indexId);
-            if (index == null) {
-                Message.throwInternalError("index not found " + indexId);
-            }
-            p = PageDataOverflow.read(index, data, pageId);
+            p = PageDataOverflow.read(this, data, pageId);
             break;
         }
         case Page.TYPE_BTREE_LEAF: {

h2/src/main/org/h2/store/PageStreamData.java

@@ -12,10 +12,10 @@ import org.h2.engine.Session;
 /**
  * A data page of a stream. The format is:
  * <ul>
- * <li>0-3: the trunk page id</li>
- * <li>4-4: page type</li>
- * <li>5-8: the number of bytes used</li>
- * <li>9-remainder: data</li>
+ * <li>the trunk page id: int (0-3)</li>
+ * <li>page type: byte (4)</li>
+ * <li>the number of bytes used: short (5-6)</li>
+ * <li>data (9-)</li>
  * </ul>
  */
 public class PageStreamData extends Page {

h2/src/main/org/h2/store/PageStreamTrunk.java

@@ -10,19 +10,19 @@ import java.sql.SQLException;
 import org.h2.engine.Session;
 
 /**
- * A trunk page of a stream. It contains the page numbers of the stream, and
- * the page number of the next trunk. The format is:
+ * A trunk page of a stream. It contains the page numbers of the stream, and the
+ * page number of the next trunk. The format is:
  * <ul>
- * <li>0-3: the last trunk page, or 0 if none</li>
- * <li>4-4: page type</li>
- * <li>5-8: the next trunk page</li>
- * <li>9-12: the number of pages</li>
- * <li>13-remainder: page ids</li>
+ * <li>previous trunk page, or 0 if none: int (0-3)</li>
+ * <li>page type: byte (4)</li>
+ * <li>next trunk page: int (5-8)</li>
+ * <li>number of pages (9-10)</li>
+ * <li>remainder: page ids (11-)</li>
  * </ul>
  */
 public class PageStreamTrunk extends Page {
 
-    private static final int DATA_START = 13;
+    private static final int DATA_START = 11;
 
     private final PageStore store;
     private int parent;
@@ -83,7 +83,7 @@ public class PageStreamTrunk extends Page {
         parent = data.readInt();
         data.readByte();
         nextTrunk = data.readInt();
-        pageCount = data.readInt();
+        pageCount = data.readShortInt();
         pageIds = new int[pageCount];
         for (int i = 0; i < pageCount; i++) {
             pageIds[i] = data.readInt();
@@ -121,7 +121,7 @@ public class PageStreamTrunk extends Page {
         data.writeInt(parent);
         data.writeByte((byte) Page.TYPE_STREAM_TRUNK);
         data.writeInt(nextTrunk);
-        data.writeInt(pageCount);
+        data.writeShortInt(pageCount);
         for (int i = 0; i < pageCount; i++) {
             data.writeInt(pageIds[i]);
         }

h2/src/main/org/h2/tools/Recover.java

@@ -1201,7 +1201,6 @@ public class Recover extends Tool implements DataHandler {
                 store.readFully(s2.getBytes(), 0, pageSize);
                 s2.setPos(4);
                 int type = s2.readByte();
-                int indexId = s2.readInt();
                 if (type == (Page.TYPE_DATA_OVERFLOW | Page.FLAG_LAST)) {
                     int size = s2.readShortInt();
                     writer.println("-- chain: " + next + " type: " + type + " size: " + size);