Support very large transactions (work in progress).

h2/src/main/org/h2/constant/SysProperties.java

@@ -13,12 +13,21 @@ import org.h2.util.MathUtils;
 
 /**
  * The constants defined in this class are initialized from system properties.
- * Those properties can be set when starting the virtual machine:
+ * Some system properties are per machine settings, and others are as a last
+ * resort and temporary solution to work around a problem in the application or
+ * database engine. Also, there are system properties to enable features that
+ * are not yet fully tested or that are not backward compatible.
+ * <p>
+ * System properties can be set when starting the virtual machine:
+ * </p>
+ *
  * <pre>
  * java -Dh2.baseDir=/temp
  * </pre>
+ *
  * They can be set within the application, but this must be done before loading
  * any classes of this database (before loading the JDBC driver):
+ *
  * <pre>
  * System.setProperty(&quot;h2.baseDir&quot;, &quot;/temp&quot;);
  * </pre>
@@ -283,6 +292,12 @@ public class SysProperties {
      */
     public static final int LARGE_RESULT_BUFFER_SIZE = getIntSetting("h2.largeResultBufferSize", 4 * 1024);
 
+    /**
+     * System property <code>h2.largeTransactions</code> (default: false).<br />
+     * Support very large transactions
+     */
+    public static final boolean LARGE_TRANSACTIONS = getBooleanSetting("h2.largeTransactions", false);
+
     /**
      * System property <code>h2.lobCloseBetweenReads</code> (default: false).<br />
      * Close LOB files between read operations.

h2/src/main/org/h2/engine/UndoLog.java

@@ -18,12 +18,12 @@ import org.h2.util.New;
  */
 public class UndoLog {
     private Database database;
-    // TODO undo log entry: a chain would probably be faster
-    //  and use less memory than an array
+    private ArrayList<Long> storedEntriesPos = New.arrayList();
     private ArrayList<UndoLogRecord> records = New.arrayList();
     private FileStore file;
     private Data rowBuff;
     private int memoryUndo;
+    private int storedEntries;
 
     /**
      * Create a new undo log for the given session.
@@ -40,6 +40,9 @@ public class UndoLog {
      * @return the number of rows
      */
     public int size() {
+        if (SysProperties.LARGE_TRANSACTIONS) {
+            return storedEntries + records.size();
+        }
         if (SysProperties.CHECK && memoryUndo > records.size()) {
             DbException.throwInternalError();
         }
@@ -52,6 +55,8 @@ public class UndoLog {
      */
     public void clear() {
         records.clear();
+        storedEntries = 0;
+        storedEntriesPos.clear();
         memoryUndo = 0;
         if (file != null) {
             file.closeAndDeleteSilently();
@@ -67,6 +72,19 @@ public class UndoLog {
      */
     public UndoLogRecord getLast() {
         int i = records.size() - 1;
+        if (SysProperties.LARGE_TRANSACTIONS) {
+            if (i <= 0 && storedEntries > 0) {
+                int last = storedEntriesPos.size() - 1;
+                long pos = storedEntriesPos.get(last);
+                storedEntriesPos.remove(last);
+                long end = file.length();
+                while (pos < end) {
+                    int test;
+//                    UndoLogRecord e = new UndoLogRecord(null, 0, null);
+
+                }
+            }
+        }
         UndoLogRecord entry = records.get(i);
         if (entry.isStored()) {
             int start = Math.max(0, i - database.getMaxMemoryUndo() / 2);
@@ -74,7 +92,7 @@ public class UndoLog {
             for (int j = start; j <= i; j++) {
                 UndoLogRecord e = records.get(j);
                 if (e.isStored()) {
-                    e.load(rowBuff, file);
+                    e.load(rowBuff, file, this);
                     memoryUndo++;
                     if (first == null) {
                         first = e;
@@ -85,11 +103,20 @@ public class UndoLog {
                 UndoLogRecord e = records.get(k);
                 e.invalidatePos();
             }
-            first.seek(file);
+            seek(first.getFilePos());
         }
         return entry;
     }
 
+    /**
+     * Go to the right position in the file.
+     *
+     * @param file the file
+     */
+    void seek(long filePos) {
+        file.seek(filePos * Constants.FILE_BLOCK_SIZE);
+    }
+
     /**
      * Remove the last record from the list of operations.
      *

h2/src/main/org/h2/engine/UndoLogRecord.java

@@ -13,6 +13,7 @@ import org.h2.message.DbException;
 import org.h2.result.Row;
 import org.h2.store.Data;
 import org.h2.store.FileStore;
+import org.h2.table.RegularTable;
 import org.h2.table.Table;
 import org.h2.value.Value;
 
@@ -68,7 +69,15 @@ public class UndoLogRecord {
      * @return if it can be stored
      */
     boolean canStore() {
-        return table.getUniqueIndex() != null;
+        if (table.getUniqueIndex() != null) {
+            return true;
+        }
+        if (SysProperties.LARGE_TRANSACTIONS) {
+            if (table instanceof RegularTable) {
+                return true;
+            }
+        }
+        return false;
     }
 
     /**
@@ -126,15 +135,6 @@ public class UndoLogRecord {
         }
     }
 
-    /**
-     * Go to the right position in the file.
-     *
-     * @param file the file
-     */
-    void seek(FileStore file) {
-        file.seek(((long) filePos) * Constants.FILE_BLOCK_SIZE);
-    }
-
     /**
      * Save the row in the file using a buffer.
      *
@@ -146,6 +146,9 @@ public class UndoLogRecord {
         buff.writeInt(0);
         buff.writeInt(operation);
         buff.writeByte(row.isDeleted() ? (byte) 1 : (byte) 0);
+        if (SysProperties.LARGE_TRANSACTIONS) {
+            int table;
+        }
         buff.writeLong(row.getKey());
         buff.writeInt(row.getSessionId());
         buff.writeInt(row.getColumnCount());
@@ -167,10 +170,11 @@ public class UndoLogRecord {
      *
      * @param buff the buffer
      * @param file the source file
+     * @param log the log
      */
-    void load(Data buff, FileStore file) {
+    void load(Data buff, FileStore file, UndoLog log) {
         int min = Constants.FILE_BLOCK_SIZE;
-        seek(file);
+        log.seek(filePos);
         buff.reset();
         file.readFully(buff.getBytes(), 0, min);
         int len = buff.readInt() * Constants.FILE_BLOCK_SIZE;
@@ -185,6 +189,9 @@ public class UndoLogRecord {
             }
         }
         boolean deleted = buff.readByte() == 1;
+        if (SysProperties.LARGE_TRANSACTIONS) {
+            int table;
+        }
         long key = buff.readLong();
         int sessionId = buff.readInt();
         int columnCount = buff.readInt();
@@ -208,6 +215,15 @@ public class UndoLogRecord {
         return table;
     }
 
+    /**
+     * Get the position in the file.
+     *
+     * @return the file position
+     */
+    public long getFilePos() {
+        return filePos;
+    }
+
     /**
      * This method is called after the operation was committed.
      * It commits the change to the indexes.

h2/src/test/org/h2/test/todo/TestUndoLogMemory.java

+/*
+ * Copyright 2004-2010 H2 Group. Multiple-Licensed under the H2 License, Version
+ * 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html). Initial Developer: H2 Group
+ */
+package org.h2.test.todo;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import org.h2.tools.DeleteDbFiles;
+
+/**
+ * A test to reproduce out of memory using a large operation.
+ */
+public class TestUndoLogMemory {
+
+    /**
+     * Run just this test.
+     *
+     * @param args ignored
+     */
+    public static void main(String... args) throws Exception {
+        // -Xmx1m -XX:+HeapDumpOnOutOfMemoryError
+        DeleteDbFiles.execute("data", "test", true);
+        Connection conn = DriverManager.getConnection("jdbc:h2:data/test");
+        Statement stat = conn.createStatement();
+        stat.execute("set cache_size 32");
+        stat.execute("SET max_operation_memory 100");
+        stat.execute("SET max_memory_undo 100");
+
+        // also a problem: tables without unique index
+        stat.execute("create table test(id int)");
+        stat.execute("insert into test select x from system_range(1, 100000)");
+
+        // stat.execute("create table test(id int primary key)");
+
+        // INSERT problem
+        // stat.execute(
+        //     "insert into test select x from system_range(1, 400000)");
+
+        // DELETE problem
+        // stat.execute(
+        //    "insert into test select x from system_range(1, 50000)");
+        // stat.execute(
+        //     "insert into test select x from system_range(50001, 100000)");
+        // stat.execute(
+        //     "insert into test select x from system_range(100001, 150000)");
+
+        stat.execute("delete from test");
+
+        conn.close();
+    }
+}