Various improvements in the page store mechanism.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,8 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>The functions LENGTH, OCTET_LENGTH, and BIT_LENGTH now return BIGINT.
+<ul><li>Various improvements in the page store mechanism (still experimental).
+</li><li>The functions LENGTH, OCTET_LENGTH, and BIT_LENGTH now return BIGINT.
 </li><li>Data types CLOB and BLOB: the maximum precision was Integer.MAX_VALUE, it is now Long.MAX_VALUE.
 </li><li>Multi-threaded kernel: creating and dropping temporary database objects
     and the potentially free pages list was not correctly synchronized. Thanks a lot

h2/src/main/org/h2/index/PageBtree.java

@@ -107,7 +107,7 @@ abstract class PageBtree extends Record {
             SearchRow row = getRow(i);
             comp = index.compareRows(row, compare);
             if (comp == 0 && add) {
-                if (index.indexType.getUnique()) {
+                if (index.indexType.isUnique()) {
                     if (!index.containsNullAndAllowMultipleNull(compare)) {
                         throw index.getDuplicateKeyException();
                     }

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -9,6 +9,7 @@ package org.h2.index;
 import java.lang.ref.SoftReference;
 import java.sql.SQLException;
 import org.h2.constant.ErrorCode;
+import org.h2.constant.SysProperties;
 import org.h2.engine.Session;
 import org.h2.message.Message;
 import org.h2.result.Row;
@@ -115,6 +116,9 @@ class PageDataLeaf extends PageData {
         } else {
             readAllRows();
             x = find(row.getPos());
+            if (SysProperties.CHECK && x < keys.length && keys[x] == row.getPos()) {
+                throw Message.throwInternalError("" + row.getPos());
+            }
             System.arraycopy(offsets, 0, newOffsets, 0, x);
             System.arraycopy(keys, 0, newKeys, 0, x);
             System.arraycopy(rows, 0, newRows, 0, x);

h2/src/main/org/h2/index/PageScanCursor.java

@@ -60,7 +60,7 @@ class PageScanCursor implements Cursor {
                     continue;
                 }
                 row = delta.next();
-                if (!row.getDeleted() || row.getSessionId() == session.getId()) {
+                if (!row.isDeleted() || row.getSessionId() == session.getId()) {
                     continue;
                 }
             } else {

h2/src/main/org/h2/index/ViewCursor.java

@@ -45,7 +45,7 @@ public class ViewCursor implements Cursor {
     public boolean next() throws SQLException {
         boolean res = result.next();
         if (!res) {
-            result.reset();
+            result.close();
             current = null;
             return false;
         }

h2/src/main/org/h2/result/ResultTempTable.java

@@ -44,7 +44,8 @@ public class ResultTempTable implements ResultExternal {
         columns.add(column);
         int tableId = session.getDatabase().allocateObjectId(true, true);
         String tableName = "TEMP_RESULT_SET_" + tableId;
-        table = schema.createTable(tableName, tableId, columns, false, true, false, Index.EMPTY_HEAD, session);
+        table = schema.createTable(tableName, tableId, columns, true, false, true, false, Index.EMPTY_HEAD, session);
+        session.addLocalTempTable(table);
         int indexId = session.getDatabase().allocateObjectId(true, false);
         IndexColumn indexColumn = new IndexColumn();
         indexColumn.column = column;
@@ -57,6 +58,8 @@ public class ResultTempTable implements ResultExternal {
         } else {
             index = new BtreeIndex(session, table, indexId, tableName, indexCols, indexType, Index.EMPTY_HEAD);
         }
+        index.setTemporary(true);
+        session.addLocalTempTableIndex(index);
         table.getIndexes().add(index);
     }
 
@@ -95,10 +98,7 @@ public class ResultTempTable implements ResultExternal {
     public void close() {
         try {
             if (table != null) {
-                index.remove(session);
-                ObjectArray<Index> indexes = table.getIndexes();
-                indexes.remove(indexes.indexOf(index));
-                table.removeChildrenAndResources(session);
+                session.removeLocalTempTable(table);
             }
         } catch (SQLException e) {
             throw Message.convertToInternal(e);

h2/src/main/org/h2/store/PageStore.java

@@ -733,24 +733,35 @@ public class PageStore implements CacheWriter {
         openMetaIndex();
         readMetaData();
         log.recover(PageLog.RECOVERY_STAGE_REDO);
+        boolean setReadOnly = false;
         if (!database.isReadOnly()) {
             if (log.getInDoubtTransactions().size() == 0) {
                 log.recoverEnd();
                 switchLog();
             } else {
-                database.setReadOnly(true);
+                setReadOnly = true;
             }
         }
-        PageScanIndex index = (PageScanIndex) metaObjects.get(0);
-        if (index == null) {
+        PageScanIndex systemTable = (PageScanIndex) metaObjects.get(0);
+        if (systemTable == null) {
             systemTableHeadPos = Index.EMPTY_HEAD;
         } else {
-            systemTableHeadPos = index.getHeadPos();
+            systemTableHeadPos = systemTable.getHeadPos();
         }
         for (Index openIndex : metaObjects.values()) {
+            if (openIndex.getTable().isTemporary()) {
+                openIndex.remove(systemSession);
+                this.removeMetaIndex(openIndex, systemSession);
+            }
             openIndex.close(systemSession);
         }
         recoveryRunning = false;
+        writeBack();
+        // clear the cache because it contains pages with closed indexes
+        cache.clear();
+        if (setReadOnly) {
+            database.setReadOnly(true);
+        }
         trace.debug("log recover done");
     }
 
@@ -844,7 +855,7 @@ public class PageStore implements CacheWriter {
         metaSchema = new Schema(database, 0, "", null, true);
         int headPos = PAGE_ID_META_ROOT;
         metaTable = new TableData(metaSchema, "PAGE_INDEX",
-                META_TABLE_ID, cols, true, true, false, headPos, systemSession);
+                META_TABLE_ID, cols, false, true, true, false, headPos, systemSession);
         metaIndex = (PageScanIndex) metaTable.getScanIndex(
                 systemSession);
         metaObjects = New.hashMap();
@@ -892,8 +903,9 @@ public class PageStore implements CacheWriter {
                 Column col = new Column("C" + i, Value.INT);
                 columnArray.add(col);
             }
-            TableData table = new TableData(metaSchema, "T" + id, id, columnArray, true, true, false, headPos, session);
             String[] ops = StringUtils.arraySplit(options, ',', true);
+            boolean temp = ops.length == 3 && ops[2].equals("temp");
+            TableData table = new TableData(metaSchema, "T" + id, id, columnArray, temp, true, true, false, headPos, session);
             CompareMode mode = CompareMode.getInstance(ops[0], Integer.parseInt(ops[1]));
             table.setCompareMode(mode);
             meta = table.getScanIndex(session);
@@ -948,6 +960,9 @@ public class PageStore implements CacheWriter {
         Table table = index.getTable();
         CompareMode mode = table.getCompareMode();
         String options = mode.getName()+ "," + mode.getStrength();
+        if (table.isTemporary()) {
+            options += ",temp";
+        }
         Row row = metaTable.getTemplateRow();
         row.setValue(0, ValueInt.get(index.getId()));
         row.setValue(1, ValueInt.get(type));
@@ -967,10 +982,14 @@ public class PageStore implements CacheWriter {
      */
     public void removeMeta(Index index, Session session) throws SQLException {
         if (!recoveryRunning) {
+            removeMetaIndex(index, session);
+        }
+    }
+
+    private void removeMetaIndex(Index index, Session session) throws SQLException {
         Row row = metaIndex.getRow(session, index.getId() + 1);
         metaIndex.remove(session, row);
     }
-    }
 
     /**
      * Set the maximum log file size in megabytes.

h2/src/main/org/h2/table/TableData.java

@@ -66,11 +66,12 @@ public class TableData extends Table implements RecordReader {
     private boolean containsLargeObject;
 
     public TableData(Schema schema, String tableName, int id, ObjectArray<Column> columns,
-            boolean persistIndexes, boolean persistData, boolean clustered, int headPos, Session session) throws SQLException {
+            boolean temporary, boolean persistIndexes, boolean persistData, boolean clustered, int headPos, Session session) throws SQLException {
         super(schema, id, tableName, persistIndexes, persistData);
         Column[] cols = new Column[columns.size()];
         columns.toArray(cols);
         setColumns(cols);
+        setTemporary(temporary);
         this.clustered = clustered;
         if (!clustered) {
             if (database.isPageStoreEnabled() && persistData && database.isPersistent()) {
@@ -159,7 +160,7 @@ public class TableData extends Table implements RecordReader {
 
     public Index getUniqueIndex() {
         for (Index idx : indexes) {
-            if (idx.getIndexType().getUnique()) {
+            if (idx.getIndexType().isUnique()) {
                 return idx;
             }
         }
@@ -172,24 +173,24 @@ public class TableData extends Table implements RecordReader {
 
     public Index addIndex(Session session, String indexName, int indexId, IndexColumn[] cols, IndexType indexType,
             int headPos, String indexComment) throws SQLException {
-        if (indexType.getPrimaryKey()) {
+        if (indexType.isPrimaryKey()) {
             for (IndexColumn c : cols) {
                 Column column = c.column;
-                if (column.getNullable()) {
+                if (column.isNullable()) {
                     throw Message.getSQLException(ErrorCode.COLUMN_MUST_NOT_BE_NULLABLE_1, column.getName());
                 }
                 column.setPrimaryKey(true);
             }
         }
         Index index;
-        if (isPersistIndexes() && indexType.getPersistent()) {
+        if (isPersistIndexes() && indexType.isPersistent()) {
             if (database.isPageStoreEnabled()) {
                 index = new PageBtreeIndex(this, indexId, indexName, cols, indexType, headPos, session);
             } else {
                 index = new BtreeIndex(session, this, indexId, indexName, cols, indexType, headPos);
             }
         } else {
-            if (indexType.getHash()) {
+            if (indexType.isHash()) {
                 index = new HashIndex(this, indexId, indexName, cols, indexType);
             } else {
                 index = new TreeIndex(this, indexId, indexName, cols, indexType);
@@ -237,11 +238,11 @@ public class TableData extends Table implements RecordReader {
                 throw e;
             }
         }
-        boolean temporary = getTemporary();
+        boolean temporary = isTemporary();
         index.setTemporary(temporary);
         if (index.getCreateSQL() != null) {
             index.setComment(indexComment);
-            if (temporary && !getGlobalTemporary()) {
+            if (temporary && !isGlobalTemporary()) {
                 session.addLocalTempTableIndex(index);
             } else {
                 database.addSchemaObject(session, index);
@@ -252,7 +253,7 @@ public class TableData extends Table implements RecordReader {
                 // need to update, because maybe the index is rebuilt at startup,
                 // and so the head pos may have changed, which needs to be stored now.
                 // addSchemaObject doesn't update the sys table at startup
-                if (index.getIndexType().getPersistent() && !database.isReadOnly()
+                if (index.getIndexType().isPersistent() && !database.isReadOnly()
                         && !database.getLog().containsInDoubtTransactions()) {
                     // can not save anything in the log file if it contains in-doubt transactions
                     database.update(session, index);
@@ -302,7 +303,7 @@ public class TableData extends Table implements RecordReader {
 
     public void removeRow(Session session, Row row) throws SQLException {
         if (database.isMultiVersion()) {
-            if (row.getDeleted()) {
+            if (row.isDeleted()) {
                 throw Message.getSQLException(ErrorCode.CONCURRENT_UPDATE_1, getName());
             }
             int old = row.getSessionId();
@@ -547,7 +548,7 @@ public class TableData extends Table implements RecordReader {
 
     public String getCreateSQL() {
         StatementBuilder buff = new StatementBuilder("CREATE ");
-        if (getTemporary()) {
+        if (isTemporary()) {
             if (globalTemporary) {
                 buff.append("GLOBAL ");
             } else {
@@ -569,7 +570,7 @@ public class TableData extends Table implements RecordReader {
             buff.append(column.getCreateSQL());
         }
         buff.append("\n)");
-        if (!getTemporary() && !isPersistIndexes() && !isPersistData()) {
+        if (!isTemporary() && !isPersistIndexes() && !isPersistData()) {
             buff.append("\nNOT PERSISTENT");
         }
         return buff.toString();
@@ -687,7 +688,7 @@ public class TableData extends Table implements RecordReader {
         this.globalTemporary = globalTemporary;
     }
 
-    public boolean getGlobalTemporary() {
+    public boolean isGlobalTemporary() {
         return globalTemporary;
     }
 

h2/src/test/org/h2/test/TestAll.java

@@ -349,10 +349,10 @@ kill -9 `jps -l | grep "org.h2.test.TestAll" | cut -d " " -f 1`
             test.runTests();
             TestPerformance.main(new String[]{ "-init", "-db", "1"});
 
-            System.setProperty(SysProperties.H2_PAGE_STORE, "false");
-            test.pageStore = false;
-            test.runTests();
-            TestPerformance.main(new String[]{ "-init", "-db", "1"});
+//            System.setProperty(SysProperties.H2_PAGE_STORE, "false");
+//            test.pageStore = false;
+//            test.runTests();
+//            TestPerformance.main(new String[]{ "-init", "-db", "1"});
         }
         System.out.println(TestBase.formatTime(System.currentTimeMillis() - time) + " total");
     }
@@ -436,16 +436,18 @@ kill -9 `jps -l | grep "org.h2.test.TestAll" | cut -d " " -f 1`
         smallLog = false;
         networked = false;
         ssl = false;
-        logMode = 0;
+        logMode = 1;
         traceLevelFile = 0;
-        cipher = "AES";
         test();
 
         big = false;
-        networked = false;
-        logMode = 1;
-        cipher = null;
+        logMode = 0;
+        cipher = "AES";
+        test();
+
         mvcc = true;
+        cipher = null;
+        logMode = 1;
         test();
 
         memory = true;

h2/src/test/org/h2/test/db/TestScript.java

@@ -73,13 +73,12 @@ public class TestScript extends TestBase {
             return;
         }
         alwaysReconnect = false;
-        testScript();
         if (!config.memory) {
             if (config.big) {
                 alwaysReconnect = true;
-                testScript();
             }
         }
+        testScript();
         deleteDb("script");
     }
 
@@ -192,9 +191,9 @@ public class TestScript extends TestBase {
                     }
                     count += processPrepared(sql, prep, param);
                 }
-                writeResult("update count: " + count, null);
+                writeResult(sql, "update count: " + count, null);
             } catch (SQLException e) {
-                writeException(e);
+                writeException(sql, e);
             }
         }
         write("");
@@ -237,7 +236,7 @@ public class TestScript extends TestBase {
             }
             return prep.getUpdateCount();
         } catch (SQLException e) {
-            writeException(e);
+            writeException(sql, e);
             return 0;
         }
     }
@@ -248,10 +247,10 @@ public class TestScript extends TestBase {
                 writeResultSet(sql, stat.getResultSet());
             } else {
                 int count = stat.getUpdateCount();
-                writeResult(count < 1 ? "ok" : "update count: " + count, null);
+                writeResult(sql, count < 1 ? "ok" : "update count: " + count, null);
             }
         } catch (SQLException e) {
-            writeException(e);
+            writeException(sql, e);
         }
         return 0;
     }
@@ -287,8 +286,8 @@ public class TestScript extends TestBase {
             result.add(row);
         }
         rs.close();
-        writeResult(format(head, max), null);
-        writeResult(format(null, max), null);
+        writeResult(sql, format(head, max), null);
+        writeResult(sql, format(null, max), null);
         String[] array = new String[result.size()];
         for (int i = 0; i < result.size(); i++) {
             array[i] = format(result.get(i), max);
@@ -298,9 +297,9 @@ public class TestScript extends TestBase {
         }
         int i = 0;
         for (; i < array.length; i++) {
-            writeResult(array[i], null);
+            writeResult(sql, array[i], null);
         }
-        writeResult((ordered ? "rows (ordered): " : "rows: ") + i, null);
+        writeResult(sql, (ordered ? "rows (ordered): " : "rows: ") + i, null);
     }
 
     private String format(String[] row, int[] max) {
@@ -327,16 +326,19 @@ public class TestScript extends TestBase {
         return buff.toString();
     }
 
-    private void writeException(SQLException e) throws Exception {
-        writeResult("exception", e);
+    private void writeException(String sql, SQLException e) throws Exception {
+        writeResult(sql, "exception", e);
     }
 
-    private void writeResult(String s, SQLException e) throws Exception {
+    private void writeResult(String sql, String s, SQLException e) throws Exception {
         assertKnownException(e);
         s = ("> " + s).trim();
         String compare = readLine();
         if (compare != null && compare.startsWith(">")) {
             if (!compare.equals(s)) {
+                if (alwaysReconnect && sql.toUpperCase().startsWith("EXPLAIN")) {
+                    return;
+                }
                 errors.append("line: ");
                 errors.append(line);
                 errors.append("\n" + "exp: ");

h2/src/test/org/h2/test/jdbc/TestMetaData.java

@@ -41,6 +41,9 @@ public class TestMetaData extends TestBase {
 
     public void test() throws SQLException {
         deleteDb("metaData");
+
+        testTempTable();
+
         conn = getConnection("metaData");
 
         testColumnLobMeta();
@@ -206,7 +209,7 @@ public class TestMetaData extends TestBase {
         // meta.getUDTs()
 
         conn.close();
-        testTempTable();
+
         deleteDb("metaData");
 
     }

h2/src/test/org/h2/test/unit/TestPageStore.java

@@ -31,10 +31,31 @@ public class TestPageStore extends TestBase {
     }
 
     public void test() throws Exception {
+        testUniqueIndex();
         testCreateIndexLater();
         testFuzzOperations();
     }
 
+    private void testUniqueIndex() throws SQLException {
+        if (config.memory) {
+            return;
+        }
+        deleteDb("pageStore");
+        Connection conn = getConnection("pageStore");
+        Statement stat = conn.createStatement();
+        stat.execute("CREATE TABLE TEST(ID INT UNIQUE)");
+        stat.execute("INSERT INTO TEST VALUES(1)");
+        conn.close();
+        conn = getConnection("pageStore");
+        try {
+            conn.createStatement().execute("INSERT INTO TEST VALUES(1)");
+            fail();
+        } catch (SQLException e) {
+            assertKnownException(e);
+        }
+        conn.close();
+    }
+
     private void testCreateIndexLater() throws SQLException {
         deleteDb("pageStore");
         Connection conn = getConnection("pageStore");