Running out of memory could result in incomplete transactions or corrupted databases

h2/src/main/org/h2/command/Command.java

@@ -193,6 +193,7 @@ public abstract class Command implements CommandInterface {
     public int executeUpdate() throws SQLException {
         startTime = System.currentTimeMillis();
         Database database = session.getDatabase();
+        database.allocateReserveMemory();
         Object sync = database.getMultiThreaded() ? (Object) session : (Object) database;
         session.waitIfExclusiveModeEnabled();
         synchronized (sync) {
@@ -200,10 +201,10 @@ public abstract class Command implements CommandInterface {
             session.setCurrentCommand(this, startTime);
             try {
                 database.checkPowerOff();
-                int test;
                 try {
                     return update();
                 } catch (OutOfMemoryError e) {
+                    database.freeReserveMemory();
                     throw Message.convert(e);
                 } catch (Throwable e) {
                     throw Message.convert(e);

h2/src/main/org/h2/constant/SysProperties.java

@@ -389,6 +389,14 @@ public class SysProperties {
      */
     public static final int REDO_BUFFER_SIZE = getIntSetting("h2.redoBufferSize", 256 * 1024);
 
+    /**
+     * System property <code>h2.reserveMemory</code> (default: 131072).<br />
+     * This many bytes in main memory are allocated as a reserve. This reserve
+     * is freed up when if no memory is available, so that rolling back a large
+     * transaction is easier.
+     */
+    public static final int RESERVE_MEMORY = getIntSetting("h2.reserveMemory", 128 * 1024);
+
     /**
      * System property <code>h2.reuseSpaceAfter</code> (default: 16).<br />
      * Reuse space in database files after this many pages are free.

h2/src/main/org/h2/engine/Database.java

@@ -157,6 +157,8 @@ public class Database implements DataHandler {
     private boolean lobFilesInDirectories = SysProperties.LOB_FILES_IN_DIRECTORIES;
     private SmallLRUCache lobFileListCache = new SmallLRUCache(128);
 
+    private Object reserveMemory;
+
     public Database(String name, ConnectionInfo ci, String cipher) throws SQLException {
         this.compareMode = new CompareMode(null, null, 0);
         this.persistent = ci.isPersistent();
@@ -2005,4 +2007,21 @@ public class Database implements DataHandler {
         return meta.isLockedExclusively();
     }
 
+    /**
+     * Allocate a little main memory that is freed up when if no memory is
+     * available, so that rolling back a large transaction is easier.
+     */
+    public void allocateReserveMemory() {
+        if (reserveMemory == null) {
+            reserveMemory = new byte[SysProperties.RESERVE_MEMORY];
+        }
+    }
+    
+    /**
+     * Free up the reserve memory.
+     */
+    public void freeReserveMemory() {
+        reserveMemory = null;
+    }
+
 }

h2/src/main/org/h2/engine/Mode.java

@@ -115,7 +115,6 @@ public class Mode {
         add(mode);
 
         mode = new Mode("Oracle");
-        mode.uniqueIndexSingleNull = true;
         add(mode);
 
     }

h2/src/main/org/h2/store/FileLock.java

@@ -260,6 +260,8 @@ public class FileLock {
                                 save();
                             }
                             Thread.sleep(sleep);
+                        } catch (OutOfMemoryError e) {
+                            // ignore
                         } catch (Exception e) {
                             trace.debug("watchdog", e);
                         }