New experimental page store.

h2/src/main/org/h2/index/PageBtree.java

@@ -114,9 +114,6 @@ abstract class PageBtree extends Record {
                 l = i + 1;
             }
         }
-        if (bigger && comp < 0) {
-            l++;
-        }
         return l;
     }
 
@@ -210,4 +207,13 @@ abstract class PageBtree extends Record {
      */
     abstract boolean remove(SearchRow row) throws SQLException;
 
+    /**
+     * Ensure all rows are read in memory.
+     */
+    protected void readAllRows() throws SQLException {
+        for (int i = 0; i < entryCount; i++) {
+            getRow(i);
+        }
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -76,6 +76,7 @@ class PageBtreeLeaf extends PageBtree {
         if (entryCount == 0) {
             x = 0;
         } else {
+            readAllRows();
             x = find(row, false, true);
             System.arraycopy(offsets, 0, newOffsets, 0, x);
             System.arraycopy(rows, 0, newRows, 0, x);
@@ -108,7 +109,8 @@ class PageBtreeLeaf extends PageBtree {
         return 0;
     }
 
-    private void removeRow(int i) {
+    private void removeRow(int i) throws SQLException {
+        readAllRows();
         entryCount--;
         written = false;
         if (entryCount <= 0) {
@@ -118,7 +120,6 @@ class PageBtreeLeaf extends PageBtree {
         SearchRow[] newRows = new SearchRow[entryCount];
         System.arraycopy(offsets, 0, newOffsets, 0, i);
         System.arraycopy(rows, 0, newRows, 0, i);
-
         int startNext = i > 0 ? offsets[i - 1] : index.getPageStore().getPageSize();
         int rowLength = startNext - offsets[i];
         for (int j = i; j < entryCount; j++) {
@@ -186,10 +187,7 @@ class PageBtreeLeaf extends PageBtree {
         if (written) {
             return;
         }
-        // make sure rows are read
-        for (int i = 0; i < entryCount; i++) {
-            getRow(i);
-        }
+        readAllRows();
         data.reset();
         data.writeInt(parentPageId);
         data.writeByte((byte) Page.TYPE_BTREE_LEAF);
@@ -236,7 +234,7 @@ class PageBtreeLeaf extends PageBtree {
             return;
         }
         PageBtreeNode next = (PageBtreeNode) index.getPage(parentPageId);
-        next.nextPage(cursor, getRow(entryCount - 1));
+        next.nextPage(cursor, getRow(0));
     }
 
     public String toString() {

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -19,7 +19,7 @@ import org.h2.store.DataPage;
  * </li><li>5-6: entry count
  * </li><li>7-10: row count of all children (-1 if not known)
  * </li><li>11-14: rightmost child page id
- * </li><li>15- entries: 4 bytes leaf page id, 4 bytes offset
+ * </li><li>15- entries: 4 bytes leaf page id, 4 bytes offset to data
  * </li></ul>
  */
 class PageBtreeNode extends PageBtree {
@@ -80,6 +80,7 @@ class PageBtreeNode extends PageBtree {
             System.arraycopy(childPageIds, 0, newChildPageIds, 0, x + 1);
         }
         if (entryCount > 0) {
+            readAllRows();
             System.arraycopy(offsets, 0, newOffsets, 0, x);
             System.arraycopy(rows, 0, newRows, 0, x);
             if (x < entryCount) {
@@ -280,7 +281,8 @@ class PageBtreeNode extends PageBtree {
         written = true;
     }
 
-    private void removeChild(int i) {
+    private void removeChild(int i) throws SQLException {
+        readAllRows();
         entryCount--;
         written = false;
         if (entryCount < 0) {
@@ -314,7 +316,7 @@ class PageBtreeNode extends PageBtree {
      * @param row the current row
      */
     void nextPage(PageBtreeCursor cursor, SearchRow row) throws SQLException {
-        int i = find(row, true, false);
+        int i = find(row, false, false) + 1;
         if (i > entryCount) {
             if (parentPageId == Page.ROOT) {
                 cursor.setCurrent(null, 0);
@@ -322,6 +324,7 @@ class PageBtreeNode extends PageBtree {
             }
             PageBtreeNode next = (PageBtreeNode) index.getPage(parentPageId);
             next.nextPage(cursor, getRow(entryCount - 1));
+            return;
         }
         PageBtree page = index.getPage(childPageIds[i]);
         PageBtreeLeaf leaf = page.getFirstLeaf();

h2/src/main/org/h2/store/PageFreeList.java

@@ -84,6 +84,7 @@ public class PageFreeList extends Record {
     public int getLastUsed() throws SQLException {
         if (nextPage < store.getPageCount()) {
             PageFreeList next = getNext();
+            // TODO avoid recursion
             return next.getLastUsed();
         }
         return used.getLastSetBit() + firstAddressed;

h2/src/main/org/h2/store/PageInputStream.java

@@ -26,6 +26,11 @@ import org.h2.message.Trace;
  */
 public class PageInputStream extends InputStream {
 
+    /**
+     * The number of header bytes per stream page.
+     */
+    public static final int OVERHEAD = 10;
+
     private PageStore store;
     private final Trace trace;
     private int parentPage;
@@ -118,7 +123,7 @@ public class PageInputStream extends InputStream {
             remaining = store.getPageSize() - page.length();
         }
         if (trace.isDebugEnabled()) {
-            // trace.debug("pageIn.readPage " + parentPage + " next:" + nextPage);
+            trace.debug("pageIn.readPage " + parentPage + " next:" + nextPage);
         }
     }
 

h2/src/main/org/h2/store/PageLog.java

@@ -34,11 +34,6 @@ import org.h2.value.Value;
  */
 public class PageLog {
 
-    /**
-     * No operation.
-     */
-    public static final int NO_OP = 0;
-
     /**
      * An undo log entry.
      * Format: page id, page.
@@ -75,7 +70,7 @@ public class PageLog {
     private DataPage data;
     private long operation;
     private BitField undo = new BitField();
-    private int[] reservedPages = new int[2];
+    private int[] reservedPages = new int[3];
 
     PageLog(PageStore store, int firstPage) {
         this.store = store;
@@ -139,9 +134,7 @@ public class PageLog {
                     break;
                 }
                 pos++;
-                if (x == NO_OP) {
-                    // nothing to do
-                } else if (x == UNDO) {
+                if (x == UNDO) {
                     int pageId = in.readInt();
                     in.readFully(data.getBytes(), 0, store.getPageSize());
                     if (undo) {
@@ -174,6 +167,11 @@ public class PageLog {
                     if (undo) {
                         store.setLastCommitForSession(sessionId, id, pos);
                     }
+                } else {
+                    if (trace.isDebugEnabled()) {
+                        trace.debug("log end");
+                        break;
+                    }
                 }
             }
         } catch (EOFException e) {
@@ -223,7 +221,7 @@ public class PageLog {
                 trace.debug("log undo " + pageId);
             }
             undo.set(pageId);
-            reservePages(2);
+            reservePages(3);
             out.write(UNDO);
             out.writeInt(pageId);
             out.write(page.getBytes(), 0, store.getPageSize());
@@ -233,16 +231,16 @@ public class PageLog {
     }
 
     private void reservePages(int pageCount) throws SQLException {
-        int testIfRequired;
-//        if (pageCount > reservedPages.length) {
-//            reservedPages = new int[pageCount];
-//        }
-//        for (int i = 0; i < pageCount; i++) {
-//            reservedPages[i] = store.allocatePage();
-//        }
-//        for (int i = 0; i < pageCount; i++) {
-//            store.freePage(reservedPages[i], false, null);
-//        }
+        int todoThisIsSlow;
+        if (pageCount > reservedPages.length) {
+            reservedPages = new int[pageCount];
+        }
+        for (int i = 0; i < pageCount; i++) {
+            reservedPages[i] = store.allocatePage(true);
+        }
+        for (int i = pageCount - 1; i >= 0; i--) {
+            store.freePage(reservedPages[i], false, null);
+        }
     }
 
     /**
@@ -293,7 +291,7 @@ public class PageLog {
             int todoWriteIntoOutputDirectly;
             row.write(data);
 
-            reservePages(1 + data.length() / store.getPageSize());
+            reservePages(3 + data.length() / (store.getPageSize() - PageInputStream.OVERHEAD));
 
             out.write(add ? ADD : REMOVE);
             out.writeInt(session.getId());

h2/src/main/org/h2/store/PageOutputStream.java

@@ -30,6 +30,7 @@ public class PageOutputStream extends OutputStream {
     private byte[] buffer = new byte[1];
     private boolean needFlush;
     private final int streamId;
+    private boolean writing;
 
     /**
      * Create a new page output stream.
@@ -76,27 +77,35 @@ public class PageOutputStream extends OutputStream {
         if (len <= 0) {
             return;
         }
-        while (len >= remaining) {
-            page.write(b, off, remaining);
-            off += remaining;
-            len -= remaining;
-            try {
-                nextPage = store.allocatePage(allocateAtEnd);
-            } catch (SQLException e) {
-                throw Message.convertToIOException(e);
+        if (writing) {
+            throw Message.throwInternalError("writing while still writing");
+        }
+        writing = true;
+        try {
+            while (len >= remaining) {
+                page.write(b, off, remaining);
+                off += remaining;
+                len -= remaining;
+                try {
+                    nextPage = store.allocatePage(allocateAtEnd);
+                } catch (SQLException e) {
+                    throw Message.convertToIOException(e);
+                }
+                page.setPos(4);
+                page.writeByte((byte) type);
+                page.writeByte((byte) streamId);
+                page.writeInt(nextPage);
+                storePage();
+                parentPage = pageId;
+                pageId = nextPage;
+                initPage();
             }
-            page.setPos(4);
-            page.writeByte((byte) type);
-            page.writeByte((byte) streamId);
-            page.writeInt(nextPage);
-            storePage();
-            parentPage = pageId;
-            pageId = nextPage;
-            initPage();
+            page.write(b, off, len);
+            needFlush = true;
+            remaining -= len;
+        } finally {
+            writing = false;
         }
-        page.write(b, off, len);
-        needFlush = true;
-        remaining -= len;
     }
 
     private void storePage() throws IOException {

h2/src/main/org/h2/store/PageStore.java

@@ -59,10 +59,10 @@ import org.h2.value.ValueString;
  */
 public class PageStore implements CacheWriter {
 
-    // TODO check if PageLog.reservePages is required
-    // TODO PageDataLeaf and Node: support random delete/add
-    // TODO PageStore.openMetaIndex (add collation for indexes,
-    // desc columns support)
+    // TODO implement unlimited number of log files (TestPageStoreDb)
+    // TODO check if PageLog.reservePages is required - yes it is - change it
+    // TODO PageStore.openMetaIndex (desc and nulls first / last columns support)
+
     // TODO btree index with fixed size values doesn't need offset and so on
     // TODO log block allocation
     // TODO block compression: maybe http://en.wikipedia.org/wiki/LZJB
@@ -512,6 +512,9 @@ trace.setLevel(TraceSystem.DEBUG);
         if (trace.isDebugEnabled()) {
             trace.debug("freePage " + pageId);
         }
+        if (lastUsedPage == pageId) {
+            lastUsedPage--;
+        }
         cache.remove(pageId);
         getFreeList().free(pageId);
         if (recoveryRunning) {

h2/src/main/org/h2/tools/Recover.java

@@ -824,9 +824,7 @@ public class Recover extends Tool implements DataHandler {
             if (x < 0) {
                 break;
             }
-            if (x == PageLog.NO_OP) {
-                // nothing to do
-            } else if (x == PageLog.UNDO) {
+            if (x == PageLog.UNDO) {
                 int pageId = in.readInt();
                 in.readFully(new byte[pageSize]);
                 writer.println("-- undo page " + pageId);
@@ -840,6 +838,9 @@ public class Recover extends Tool implements DataHandler {
             } else if (x == PageLog.COMMIT) {
                 int sessionId = in.readInt();
                 writer.println("-- commit " + sessionId);
+            } else {
+                writer.println("-- end " + x);
+                break;
             }
         }
 

h2/src/test/org/h2/test/TestAll.java

@@ -288,6 +288,7 @@ java org.h2.test.TestAll timer
 
 // 2009-05-15: 25 tests fail with page store (first loop)
 // 2009-05-18: 18 tests fail with page store (first loop)
+// 2009-05-30: 15 tests fail with page store (first loop)
 // System.setProperty("h2.pageStore", "true");
 
 /*
@@ -337,6 +338,7 @@ kill -9 `jps -l | grep "org.h2.test.TestAll" | cut -d " " -f 1`
                 new TestRandomSQL().runTest(test);
             } else if ("join".equals(args[0])) {
                 new TestJoin().runTest(test);
+                test.endless = true;
             } else if ("btree".equals(args[0])) {
                 new TestBtreeIndex().runTest(test);
             } else if ("all".equals(args[0])) {
@@ -546,6 +548,7 @@ kill -9 `jps -l | grep "org.h2.test.TestAll" | cut -d " " -f 1`
         new TestRowLocks().runTest(this);
 
         // synth
+        new TestBtreeIndex().runTest(this);
         new TestCrashAPI().runTest(this);
         new TestFuzzOptimizations().runTest(this);
         new TestRandomSQL().runTest(this);

h2/src/test/org/h2/test/synth/TestBtreeIndex.java

@@ -33,12 +33,36 @@ public class TestBtreeIndex extends TestBase {
 
     public void test() throws SQLException {
         Random random = new Random();
-        while (true) {
+        for (int i = 0; i < getSize(1, 4); i++) {
+            testAddDelete();
             int seed = random.nextInt();
             testCase(seed);
         }
     }
 
+    private void testAddDelete() throws SQLException {
+        deleteDb("index");
+        Connection conn = getConnection("index");
+        Statement stat = conn.createStatement();
+        stat.execute("CREATE TABLE TEST(ID bigint primary key)");
+        int count = 1000;
+        stat.execute("insert into test select x from system_range(1, " + count + ")");
+        if (!config.memory) {
+            conn.close();
+            conn = getConnection("index");
+            stat = conn.createStatement();
+        }
+        for (int i = 1; i < count; i++) {
+            ResultSet rs = stat.executeQuery("select * from test order by id");
+            for (int j = i; rs.next(); j++) {
+                assertEquals(j, rs.getInt(1));
+            }
+            stat.execute("delete from test where id =" + i);
+        }
+        stat.execute("drop all objects delete files");
+        conn.close();
+    }
+
     public void testCase(int seed) throws SQLException {
         String old = baseDir;
         baseDir = TestBase.getTestDir("index");
@@ -109,7 +133,7 @@ public class TestBtreeIndex extends TestBase {
                         int deleted = prepDelete.executeUpdate();
                         if (deleted > 1) {
                             System.out.println("ERROR deleted:" + deleted);
-                            System.out.println("new TestIndex().");
+                            System.out.println("seed: " + seed);
                         }
                         count -= deleted;
                     } catch (SQLException e) {
@@ -126,12 +150,12 @@ public class TestBtreeIndex extends TestBase {
         }
         if (testCount != count) {
             System.out.println("ERROR count:" + count + " testCount:" + testCount);
-            System.out.println("new TestIndex().");
+            System.out.println("seed: " + seed);
         }
         rs = conn.createStatement().executeQuery("SELECT text, count(*) FROM a GROUP BY text HAVING COUNT(*)>1");
         if (rs.next()) {
             System.out.println("ERROR");
-            System.out.println("new TestIndex().");
+            System.out.println("seed: " + seed);
         }
         conn.close();
     }

h2/src/test/org/h2/test/unit/TestClearReferences.java

@@ -31,7 +31,8 @@ public class TestClearReferences extends TestBase {
         "org.h2.tools.CompressTool.cachedBuffer",
         "org.h2.util.MemoryUtils.reserveMemory",
         "org.h2.util.NetUtils.cachedLocalAddress",
-        "org.h2.util.RandomUtils.cachedSecureRandom"
+        "org.h2.util.RandomUtils.cachedSecureRandom",
+        "org.h2.value.CompareMode.lastUsed"
     };
 
     private boolean hasError;