new experimental page store

h2/src/docsrc/html/performance.html

@@ -287,6 +287,20 @@ It was developed / sponsored by db4o.
 <tr><td>Barcelona delete</td><td>ms</td><td>388</td><td>319</td><td>3287</td></tr>
 <tr><td>Total</td><td>ms</td><td>26724</td><td>53962</td><td>87112</td></tr>
 </table>
+<p>
+There are a few problems with the PolePosition test:
+</p>
+<ul><li> 
+HSQLDB uses in-memory tables by default while H2 uses persistent tables. The HSQLDB version 
+included in PolePosition does not support changing this, so you need to replace 
+poleposition-0.20/lib/hsqldb.jar with a newer version (for example hsqldb-1.8.0.7.jar), 
+and then use the setting 
+hsqldb.connecturl=jdbc:hsqldb:file:data/hsqldb/dbbench2;hsqldb.default_table_type=cached;sql.enforce_size=true in Jdbc.properties.
+</li><li>HSQLDB keeps the database open between tests, while H2 closes the database (losing all the cache). 
+To change that, use the database URL jdbc:h2:file:data/h2/dbbench;DB_CLOSE_DELAY=-1
+</li><li>The amount of cache memory is quite important, specially for the PolePosition test. 
+Unfortunately, the PolePosition test does not take this into account.
+</li></ul>
 
 <br /><a name="application_profiling"></a>
 <h2>Application Profiling</h2>

h2/src/main/org/h2/index/Page.java

@@ -18,34 +18,29 @@ public class Page {
     public static final int TYPE_LOG = 8;
 
     /**
-     * An empty page.
-     */
-    static final int TYPE_EMPTY = 0;
-
-    /**
-     * A data leaf page without overflow.
+     * This is the last page of a chain.
      */
-    static final int TYPE_DATA_LEAF = 2;
+    public static final int FLAG_LAST = 16;
 
     /**
-     * A data leaf page with overflow.
+     * An empty page.
      */
-    static final int TYPE_DATA_LEAF_WITH_OVERFLOW = 3;
+    static final int TYPE_EMPTY = 0;
 
     /**
-     * A data node page without overflow.
+     * A data leaf page (without overflow: or FLAG_LAST).
      */
-    static final int TYPE_DATA_NODE = 4;
+    static final int TYPE_DATA_LEAF = 1;
 
-    /**
-     * The last overflow page.
+     /**
+     * A data node page (does never have overflow).
      */
-    static final int TYPE_DATA_OVERFLOW_LAST = 6;
+    static final int TYPE_DATA_NODE = 2;
 
     /**
-     * An overflow pages (more to come).
+     * An overflow pages (the last page: or FLAG_LAST).
      */
-    static final int TYPE_DATA_OVERFLOW_WITH_MORE = 7;
+    static final int TYPE_DATA_OVERFLOW = 3;
 
     /**
      * This is a root page.

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -72,7 +72,7 @@ class PageDataLeaf extends PageData {
         offsets = new int[entryCount];
         keys = new int[entryCount];
         rows = new Row[entryCount];
-        if (type == Page.TYPE_DATA_LEAF_WITH_OVERFLOW) {
+        if (type == (Page.TYPE_DATA_LEAF | Page.FLAG_LAST)) {
             firstOverflowPageId = data.readInt();
         }
         for (int i = 0; i < entryCount; i++) {
@@ -191,7 +191,7 @@ class PageDataLeaf extends PageData {
                     DataPageBinary page = store.readPage(next);
                     page.setPos(4);
                     int type = page.readByte();
-                    if (type == Page.TYPE_DATA_OVERFLOW_LAST) {
+                    if (type == (Page.TYPE_DATA_OVERFLOW | Page.FLAG_LAST)) {
                         int size = page.readShortInt();
                         data.write(page.getBytes(), 7, size);
                         break;
@@ -296,9 +296,9 @@ class PageDataLeaf extends PageData {
         data.writeInt(parentPageId);
         int type;
         if (firstOverflowPageId == 0) {
-            type = Page.TYPE_DATA_LEAF;
+            type = Page.TYPE_DATA_LEAF | Page.FLAG_LAST;
         } else {
-            type = Page.TYPE_DATA_LEAF_WITH_OVERFLOW;
+            type = Page.TYPE_DATA_LEAF;
         }
         data.writeByte((byte) type);
         data.writeShortInt(entryCount);
@@ -330,11 +330,11 @@ class PageDataLeaf extends PageData {
                 overflow.writeInt(parent);
                 int size;
                 if (remaining > pageSize - 7) {
-                    overflow.writeByte((byte) Page.TYPE_DATA_OVERFLOW_WITH_MORE);
+                    overflow.writeByte((byte) Page.TYPE_DATA_OVERFLOW);
                     overflow.writeInt(overflowPageIds[i + 1]);
                     size = pageSize - overflow.length();
                 } else {
-                    overflow.writeByte((byte) Page.TYPE_DATA_OVERFLOW_LAST);
+                    overflow.writeByte((byte) (Page.TYPE_DATA_OVERFLOW | Page.FLAG_LAST));
                     size = remaining;
                     overflow.writeShortInt(remaining);
                 }

h2/src/main/org/h2/index/PageScanIndex.java

@@ -118,9 +118,8 @@ public class PageScanIndex extends BaseIndex implements RowIndex {
         int parentPageId = data.readInt();
         int type = data.readByte() & 255;
         PageData result;
-        switch (type) {
+        switch (type & ~Page.FLAG_LAST) {
         case Page.TYPE_DATA_LEAF:
-        case Page.TYPE_DATA_LEAF_WITH_OVERFLOW:
             result = new PageDataLeaf(this, id, parentPageId, data);
             break;
         case Page.TYPE_DATA_NODE:

h2/src/main/org/h2/store/PageInputStream.java

+/*
+ * Copyright 2004-2008 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
+import org.h2.constant.ErrorCode;
+import org.h2.engine.Constants;
+import org.h2.index.Page;
+import org.h2.message.Message;
+
+/**
+ * An output stream that writes into a page store.
+ * The format is:
+ * <ul><li>0-3: parent page id
+ * </li><li>4-4: page type
+ * </li><li>5-8: the next page (if there are more) or length
+ * </li><li>9-remainder: data
+ * </li></ul>
+ * The data format is:
+ * <ul><li>0-0: type (0: end, 1: undo)
+ * </li><li>1-4: page id
+ * </li><li>5-: data
+ * </li></ul>
+ */
+public class PageInputStream extends InputStream {
+
+    private PageStore store;
+    private int parentPage;
+    private int type;
+    private int nextPage;
+    private DataPageBinary page;
+    private boolean endOfFile;
+    private int remaining;
+
+    public PageInputStream(PageStore store, int parentPage, int headPage, int type) {
+        this.store = store;
+        this.parentPage = parentPage;
+        this.type = type;
+        nextPage = headPage;
+        page = store.createDataPage();
+    }
+
+    public int read() throws IOException {
+        byte[] b = new byte[1];
+        int len = read(b);
+        return len < 0 ? -1 : b[0];
+    }
+
+    public int read(byte[] b) throws IOException {
+        return read(b, 0, b.length);
+    }
+
+    public int read(byte[] b, int off, int len) throws IOException {
+        if (len == 0) {
+            return 0;
+        }
+        int read = 0;
+        while (len > 0) {
+            int r = readBlock(b, off, len);
+            if (r < 0) {
+                break;
+            }
+            read += r;
+            off += r;
+            len -= r;
+        }
+        return read == 0 ? -1 : read;
+    }
+
+    private int readBlock(byte[] buff, int off, int len) throws IOException {
+        fillBuffer();
+        if (endOfFile) {
+            return -1;
+        }
+        int l = Math.min(remaining, len);
+        page.read(buff, off, l);
+        remaining -= l;
+        return l;
+    }
+
+    private void fillBuffer() throws IOException {
+        if (remaining > 0 || endOfFile) {
+            return;
+        }
+        if (nextPage == 0) {
+            endOfFile = true;
+            return;
+        }
+        page.reset();
+        try {
+            page = store.readPage(nextPage);
+            int p = page.readInt();
+            int t = page.readByte();
+            boolean last = (t & Page.FLAG_LAST) != 0;
+            t &= ~Page.FLAG_LAST;
+            if (type != t || p != parentPage) {
+                throw Message.getSQLException(
+                        ErrorCode.FILE_CORRUPTED_1,
+                        "type:" + t + " parent:" + p +
+                        " expected type:" + type + " expected parent:" + parentPage);
+            }
+            parentPage = nextPage;
+            if (last) {
+                nextPage = 0;
+                remaining = page.readInt();
+            } else {
+                nextPage = page.readInt();
+            }
+            remaining = Constants.FILE_BLOCK_SIZE - page.length();
+        } catch (SQLException e) {
+            throw Message.convertToIOException(e);
+        }
+    }
+
+}

h2/src/main/org/h2/store/PageOutputStream.java

+/*
+ * Copyright 2004-2008 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.store;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.sql.SQLException;
+import org.h2.index.Page;
+import org.h2.message.Message;
+
+/**
+ * An output stream that writes into a page store.
+ */
+public class PageOutputStream extends OutputStream {
+
+    private PageStore store;
+    private int parentPage;
+    private int type;
+    private int pageId;
+    private int nextPage;
+    private DataPageBinary page;
+    private int remaining;
+
+    /**
+     * Create a new page output stream.
+     *
+     * @param store the page store
+     * @param parentPage the parent page id
+     * @param headPage the first page
+     * @param type the page type
+     */
+    public PageOutputStream(PageStore store, int parentPage, int headPage, int type) {
+        this.store = store;
+        this.parentPage = parentPage;
+        this.nextPage = headPage;
+        this.type = type;
+        page = store.createDataPage();
+        initPage();
+    }
+
+    public void write(int b) throws IOException {
+        write(new byte[] { (byte) b });
+    }
+
+    public void write(byte[] b) throws IOException {
+        write(b, 0, b.length);
+    }
+
+    private void initPage() {
+        page.reset();
+        page.writeInt(parentPage);
+        page.writeByte((byte) type);
+        page.writeInt(0);
+        remaining = page.length();
+    }
+
+    public void write(byte[] b, int off, int len) throws IOException {
+        if (len <= 0) {
+            return;
+        }
+        while (len > remaining) {
+            page.write(b, off, remaining);
+            off += remaining;
+            len -= remaining;
+            parentPage = nextPage;
+            nextPage = store.allocatePage();
+            page.setInt(5, nextPage);
+            storePage();
+            initPage();
+        }
+        page.write(b, off, len);
+        remaining -= len;
+    }
+
+    private void storePage() throws IOException {
+        try {
+            store.writePage(pageId, page);
+        } catch (SQLException e) {
+            throw Message.convertToIOException(e);
+        }
+    }
+
+    public void close() throws IOException {
+        page.setPos(4);
+        page.writeByte((byte) (type | Page.FLAG_LAST));
+        page.writeInt(store.getPageSize() - remaining - 9);
+        storePage();
+        store = null;
+    }
+
+//    public void write(byte[] buff, int off, int len) throws IOException {
+//        if (len > 0) {
+//            try {
+//                page.reset();
+//                if (compress != null) {
+//                    if (off != 0 || len != buff.length) {
+//                        byte[] b2 = new byte[len];
+//                        System.arraycopy(buff, off, b2, 0, len);
+//                        buff = b2;
+//                        off = 0;
+//                    }
+//                    int uncompressed = len;
+//                    buff = compress.compress(buff, compressionAlgorithm);
+//                    len = buff.length;
+//                    page.writeInt(len);
+//                    page.writeInt(uncompressed);
+//                    page.write(buff, off, len);
+//                } else {
+//                    page.writeInt(len);
+//                    page.write(buff, off, len);
+//                }
+//                page.fillAligned();
+//                store.write(page.getBytes(), 0, page.length());
+//            } catch (SQLException e) {
+//                throw Message.convertToIOException(e);
+//            }
+//        }
+//    }
+
+}

h2/src/test/org/h2/test/TestAll.java

@@ -284,6 +284,11 @@ java org.h2.test.TestAll timer
 
 /*
 
+JCR: for each node type, create a table; one 'dynamic' table with parameter;
+    option to cache the results
+MySQL compatibility for @rownum:=@rownum+1
+insert into test2 (select id, id from test);
+
 http://blog.flexive.org/2008/12/05/porting-flexive-to-the-h2-database/
 postgresql generate_series?
 is in-memory scan index re-using ids?

h2/src/test/org/h2/test/unit/TestOverflow.java

@@ -27,6 +27,15 @@ public class TestOverflow extends TestBase {
     private BigInteger min, max;
     private boolean successExpected;
 
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String[] a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
     public void test() throws SQLException {
         test(Value.BYTE, Byte.MIN_VALUE, Byte.MAX_VALUE);
         test(Value.INT, Integer.MIN_VALUE, Integer.MAX_VALUE);

h2/src/test/org/h2/test/unit/TestPageStoreStreams.java

+/*
+ * Copyright 2004-2008 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.unit;
+
+import java.io.File;
+import java.util.Random;
+import org.h2.engine.ConnectionInfo;
+import org.h2.engine.Database;
+import org.h2.index.Page;
+import org.h2.store.PageInputStream;
+import org.h2.store.PageOutputStream;
+import org.h2.store.PageStore;
+import org.h2.test.TestBase;
+
+/**
+ * Test page store input and output streams.
+ */
+public class TestPageStoreStreams extends TestBase {
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String[] a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+    public void test() throws Exception {
+        String name = "mem:pageStoreStreams";
+        ConnectionInfo ci = new ConnectionInfo(name);
+        Database db = new Database(name, ci, null);
+        String fileName = getTestDir("/pageStoreStreams");
+        new File(fileName).delete();
+        PageStore store = new PageStore(db, fileName, "rw", 8192);
+        store.open();
+        Random random = new Random(1);
+        for (int i = 0; i < 10000; i += 1000) {
+            int len = i == 0 ? 0 : random.nextInt(i);
+            byte[] data = new byte[len];
+            random.nextBytes(data);
+            int head = store.allocatePage();
+            PageOutputStream out = new PageOutputStream(store, 0, head, Page.TYPE_LOG);
+            for (int p = 0; len > 0;) {
+                int l = len == 0 ? 0 : random.nextInt(len / 10);
+                out.write(data, p, l);
+                p += l;
+            }
+            out.close();
+            PageInputStream in = new PageInputStream(store, 0, head, Page.TYPE_LOG);
+            byte[] data2 = new byte[len];
+            for (int off = 0;;) {
+                int l = random.nextInt(1 + len / 10) - 1;
+                l = in.read(data2, off, l);
+                if (l < 0) {
+                    break;
+                }
+                off += l;
+            }
+            in.close();
+            assertEquals(data, data2);
+        }
+        store.close();
+        db.shutdownImmediately();
+        new File(fileName).delete();
+    }
+
+}