Merge pull request #867 from h2database/test_out_of_memory

TestOutOfMemory stability

h2/src/main/org/h2/engine/Database.java

@@ -209,6 +209,8 @@ public class Database implements DataHandler {
     private RowFactory rowFactory = RowFactory.DEFAULT;
 
     public Database(ConnectionInfo ci, String cipher) {
+        META_LOCK_DEBUGGING.set(null);
+        META_LOCK_DEBUGGING_STACK.set(null);
         String name = ci.getName();
         this.dbSettings = ci.getDbSettings();
         this.reconnectCheckDelayNs = TimeUnit.MILLISECONDS.toNanos(dbSettings.reconnectCheckDelay);
@@ -1256,6 +1258,7 @@ public class Database implements DataHandler {
      *            hook
      */
     void close(boolean fromShutdownHook) {
+        try {
             synchronized (this) {
                 if (closing) {
                     return;
@@ -1270,7 +1273,6 @@ public class Database implements DataHandler {
                         // ignore
                     }
                     traceSystem.close();
-                Engine.getInstance().close(databaseName);
                     return;
                 }
                 closing = true;
@@ -1350,7 +1352,6 @@ public class Database implements DataHandler {
                 }
                 closeOnExit = null;
             }
-        Engine.getInstance().close(databaseName);
             if (deleteFilesOnDisconnect && persistent) {
                 deleteFilesOnDisconnect = false;
                 try {
@@ -1361,6 +1362,9 @@ public class Database implements DataHandler {
                     // ignore (the trace is closed already)
                 }
             }
+        } finally {
+            Engine.getInstance().close(databaseName);
+        }
     }
 
     private void removeOrphanedLobs() {

h2/src/main/org/h2/engine/DatabaseCloser.java

@@ -69,7 +69,8 @@ class DatabaseCloser extends Thread {
                     trace.error(e, "could not close the database");
                     // if this was successful, we ignore the exception
                     // otherwise not
-                } catch (RuntimeException e2) {
+                } catch (Throwable e2) {
+                    e.addSuppressed(e2);
                     throw e;
                 }
             }

h2/src/main/org/h2/engine/Engine.java

@@ -6,6 +6,7 @@
 package org.h2.engine;
 
 import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 import org.h2.api.ErrorCode;
 import org.h2.command.CommandInterface;
@@ -27,7 +28,7 @@ import org.h2.util.Utils;
 public class Engine implements SessionFactory {
 
     private static final Engine INSTANCE = new Engine();
-    private static final HashMap<String, Database> DATABASES = new HashMap<>();
+    private static final Map<String, Database> DATABASES = new HashMap<>();
 
     private volatile long wrongPasswordDelay =
             SysProperties.DELAY_WRONG_PASSWORD_MIN;
@@ -50,13 +51,14 @@ public class Engine implements SessionFactory {
         Database database;
         ci.removeProperty("NO_UPGRADE", false);
         boolean openNew = ci.getProperty("OPEN_NEW", false);
+        boolean opened = false;
+        User user = null;
+        synchronized (DATABASES) {
             if (openNew || ci.isUnnamedInMemory()) {
                 database = null;
             } else {
                 database = DATABASES.get(name);
             }
-        User user = null;
-        boolean opened = false;
             if (database == null) {
                 if (ifExists && !Database.exists(name)) {
                     throw DbException.get(ErrorCode.DATABASE_NOT_FOUND_1, name);
@@ -76,6 +78,7 @@ public class Engine implements SessionFactory {
                     DATABASES.put(name, database);
                 }
             }
+        }
         if (opened) {
             // start the thread when already synchronizing on the database
             // otherwise a deadlock can occur when the writer thread
@@ -272,8 +275,10 @@ public class Engine implements SessionFactory {
                 throw DbException.get(ErrorCode.FEATURE_NOT_SUPPORTED_1, e, "JMX");
             }
         }
+        synchronized (DATABASES) {
             DATABASES.remove(name);
         }
+    }
 
     /**
      * This method is called after validating user name and password. If user

h2/src/main/org/h2/jdbc/JdbcConnection.java

@@ -411,7 +411,7 @@ public class JdbcConnection extends TraceObject
                     session = null;
                 }
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             throw logAndConvert(e);
         }
     }

h2/src/main/org/h2/jdbc/JdbcPreparedStatement.java

@@ -248,7 +248,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements
             } finally {
                 afterWriting();
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             throw logAndConvert(e);
         }
     }

h2/src/main/org/h2/message/DbException.java

@@ -273,7 +273,7 @@ public class DbException extends RuntimeException {
      * @param e the root cause
      * @return the SQL exception object
      */
-    public static SQLException toSQLException(Exception e) {
+    public static SQLException toSQLException(Throwable e) {
         if (e instanceof SQLException) {
             return (SQLException) e;
         }

h2/src/main/org/h2/message/TraceObject.java

@@ -351,8 +351,10 @@ public class TraceObject {
      * @param ex the exception
      * @return the SQL exception object
      */
-    protected SQLException logAndConvert(Exception ex) {
-        SQLException e = DbException.toSQLException(ex);
+    protected SQLException logAndConvert(Throwable ex) {
+        SQLException e = null;
+        try {
+            e = DbException.toSQLException(ex);
             if (trace == null) {
                 DbException.traceThrowable(e);
             } else {
@@ -363,6 +365,12 @@ public class TraceObject {
                     trace.error(e, "exception");
                 }
             }
+        } catch(Throwable ignore) {
+            if (e == null) {
+                e = new SQLException("", "HY000", ex);
+            }
+            e.addSuppressed(ignore);
+        }
         return e;
     }
 

h2/src/main/org/h2/mvstore/MVStore.java

@@ -380,9 +380,7 @@ public final class MVStore {
     }
 
     private void panic(IllegalStateException e) {
-        if (backgroundExceptionHandler != null) {
-            backgroundExceptionHandler.uncaughtException(null, e);
-        }
+        handleException(e);
         panicException = e;
         closeImmediately();
         throw e;
@@ -862,10 +860,8 @@ public final class MVStore {
     public void closeImmediately() {
         try {
             closeStore(false);
-        } catch (Exception e) {
-            if (backgroundExceptionHandler != null) {
-                backgroundExceptionHandler.uncaughtException(null, e);
-            }
+        } catch (Throwable e) {
+            handleException(e);
         }
     }
 
@@ -2468,13 +2464,11 @@ public final class MVStore {
         if (hasUnsavedChanges()) {
             try {
                 commitAndSave();
-            } catch (Exception e) {
-                if (backgroundExceptionHandler != null) {
-                    backgroundExceptionHandler.uncaughtException(null, e);
+            } catch (Throwable e) {
+                handleException(e);
                 return;
             }
         }
-        }
         if (autoCompactFillRate > 0) {
             try {
                 // whether there were file read or write operations since
@@ -2492,10 +2486,18 @@ public final class MVStore {
                 // in the bookkeeping?
                 compact(fillRate, autoCommitMemory);
                 autoCompactLastFileOpCount = fileStore.getWriteCount() + fileStore.getReadCount();
-            } catch (Exception e) {
-                if (backgroundExceptionHandler != null) {
-                    backgroundExceptionHandler.uncaughtException(null, e);
+            } catch (Throwable e) {
+                handleException(e);
+            }
+        }
     }
+
+    private void handleException(Throwable ex) {
+        if (backgroundExceptionHandler != null) {
+            try {
+                backgroundExceptionHandler.uncaughtException(null, ex);
+            } catch(Throwable ignore) {
+                ex.addSuppressed(ignore);
             }
         }
     }

h2/src/test/org/h2/test/TestBase.java

@@ -79,6 +79,8 @@ public abstract class TestBase {
 
     private final LinkedList<byte[]> memory = new LinkedList<>();
 
+    private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
     /**
      * Get the test directory for this test.
      *
@@ -545,7 +547,6 @@ public abstract class TestBase {
      * @param s the message
      */
     static synchronized void printlnWithTime(long millis, String s) {
-        SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
         s = dateFormat.format(new java.util.Date()) + " " +
                 formatTime(millis) + " " + s;
         System.out.println(s);
@@ -1470,6 +1471,12 @@ public abstract class TestBase {
      */
     protected void freeMemory() {
         memory.clear();
+        for (int i = 0; i < 5; i++) {
+            System.gc();
+            try {
+                Thread.sleep(20);
+            } catch (InterruptedException ignore) {/**/}
+        }
     }
 
     /**

h2/src/test/org/h2/test/db/TestOutOfMemory.java

@@ -13,7 +13,9 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicReference;
 import org.h2.api.ErrorCode;
+import org.h2.message.DbException;
 import org.h2.mvstore.MVStore;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FilePathMem;
@@ -37,10 +39,6 @@ public class TestOutOfMemory extends TestBase {
 
     @Override
     public void test() throws SQLException, InterruptedException {
-        if (config.travis) {
-            // fails regularly under Travis, not sure why
-            return;
-        }
         if (config.vmlens) {
             // running out of memory will cause the vmlens agent to stop working
             return;
@@ -60,7 +58,16 @@ public class TestOutOfMemory extends TestBase {
     private void testMVStoreUsingInMemoryFileSystem() {
         FilePath.register(new FilePathMem());
         String fileName = "memFS:" + getTestName();
-        MVStore store = MVStore.open(fileName);
+        final AtomicReference<Throwable> exRef = new AtomicReference<>();
+        MVStore store = new MVStore.Builder()
+                .fileName(fileName)
+                .backgroundExceptionHandler(new Thread.UncaughtExceptionHandler() {
+                    @Override
+                    public void uncaughtException(Thread t, Throwable e) {
+                        exRef.compareAndSet(null, e);
+                    }
+                })
+                .open();
         try {
             Map<Integer, byte[]> map = store.openMap("test");
             Random r = new Random(1);
@@ -70,10 +77,11 @@ public class TestOutOfMemory extends TestBase {
                     r.nextBytes(data);
                     map.put(i, data);
                 }
+                Throwable throwable = exRef.get();
+                if(throwable instanceof OutOfMemoryError) throw (OutOfMemoryError)throwable;
+                if(throwable instanceof IllegalStateException) throw (IllegalStateException)throwable;
                 fail();
-            } catch (OutOfMemoryError e) {
-                // expected
-            } catch (IllegalStateException e) {
+            } catch (OutOfMemoryError | IllegalStateException e) {
                 // expected
             }
             try {
@@ -83,7 +91,7 @@ public class TestOutOfMemory extends TestBase {
             }
             store.closeImmediately();
             store = MVStore.open(fileName);
-            map = store.openMap("test");
+            store.openMap("test");
             store.close();
         } finally {
             // just in case, otherwise if this test suffers a spurious failure,
@@ -95,47 +103,55 @@ public class TestOutOfMemory extends TestBase {
 
     private void testDatabaseUsingInMemoryFileSystem() throws SQLException, InterruptedException {
         String filename = "memFS:" + getTestName();
-        String url = "jdbc:h2:" + filename;
+        String url = "jdbc:h2:" + filename + "/test";
+        try {
             Connection conn = DriverManager.getConnection(url);
             Statement stat = conn.createStatement();
             try {
                 stat.execute("create table test(id int, name varchar) as " +
-                    "select x, space(10000000) from system_range(1, 1000)");
+                        "select x, space(10000000+x) from system_range(1, 1000)");
                 fail();
             } catch (SQLException e) {
-            int err = e.getErrorCode();
-            assertTrue(e.getMessage(), err == ErrorCode.GENERAL_ERROR_1
-                    || err == ErrorCode.OUT_OF_MEMORY);
+                assertTrue("Unexpected error code: " + e.getErrorCode(),
+                        ErrorCode.OUT_OF_MEMORY == e.getErrorCode() ||
+                        ErrorCode.FILE_CORRUPTED_1 == e.getErrorCode() ||
+                        ErrorCode.DATABASE_IS_CLOSED == e.getErrorCode() ||
+                        ErrorCode.GENERAL_ERROR_1 == e.getErrorCode());
             }
+            recoverAfterOOM();
             try {
                 conn.close();
                 fail();
             } catch (SQLException e) {
-            int err = e.getErrorCode();
-            assertTrue(e.getMessage(), err == ErrorCode.GENERAL_ERROR_1
-                    || err == ErrorCode.OUT_OF_MEMORY
-                    || err == ErrorCode.DATABASE_IS_CLOSED);
-        }
-        for (int i = 0; i < 5; i++) {
-            System.gc();
-            Thread.sleep(20);
+                assertTrue("Unexpected error code: " + e.getErrorCode(),
+                        ErrorCode.OUT_OF_MEMORY == e.getErrorCode() ||
+                        ErrorCode.FILE_CORRUPTED_1 == e.getErrorCode() ||
+                        ErrorCode.DATABASE_IS_CLOSED == e.getErrorCode() ||
+                        ErrorCode.GENERAL_ERROR_1 == e.getErrorCode());
             }
+            recoverAfterOOM();
             conn = DriverManager.getConnection(url);
             stat = conn.createStatement();
-        stat.execute("select 1");
+            stat.execute("SELECT 1");
             conn.close();
+        } finally {
             // release the static data this test generates
-        FileUtils.delete(filename);
+            FileUtils.deleteRecursive(filename, true);
         }
-
-    private void testUpdateWhenNearlyOutOfMemory() throws SQLException, InterruptedException {
-        if (config.memory || config.mvcc || config.mvStore) {
-            return;
     }
+
+    private static void recoverAfterOOM() throws InterruptedException {
         for (int i = 0; i < 5; i++) {
             System.gc();
             Thread.sleep(20);
         }
+    }
+
+    private void testUpdateWhenNearlyOutOfMemory() throws SQLException, InterruptedException {
+        if (config.memory) {
+            return;
+        }
+        recoverAfterOOM();
         deleteDb("outOfMemory");
         Connection conn = getConnection("outOfMemory;MAX_OPERATION_MEMORY=1000000");
         Statement stat = conn.createStatement();
@@ -148,8 +164,27 @@ public class TestOutOfMemory extends TestBase {
         stat.execute("checkpoint");
         eatMemory(80);
         try {
-            assertThrows(ErrorCode.OUT_OF_MEMORY, prep).execute();
-            assertThrows(ErrorCode.DATABASE_IS_CLOSED, conn).close();
+            try {
+                prep.execute();
+                fail();
+            } catch(DbException ex) {
+                freeMemory();
+                assertTrue(ErrorCode.OUT_OF_MEMORY == ex.getErrorCode() || ErrorCode.GENERAL_ERROR_1 == ex.getErrorCode());
+            } catch (SQLException ex) {
+                freeMemory();
+                assertTrue(ErrorCode.OUT_OF_MEMORY == ex.getErrorCode() || ErrorCode.GENERAL_ERROR_1 == ex.getErrorCode());
+            }
+            recoverAfterOOM();
+            try {
+                conn.close();
+                fail();
+            } catch(DbException ex) {
+                freeMemory();
+                assertEquals(ErrorCode.DATABASE_IS_CLOSED, ex.getErrorCode());
+            } catch (SQLException ex) {
+                freeMemory();
+                assertEquals(ErrorCode.DATABASE_IS_CLOSED, ex.getErrorCode());
+            }
             freeMemory();
             conn = null;
             conn = getConnection("outOfMemory");
@@ -160,7 +195,7 @@ public class TestOutOfMemory extends TestBase {
         } catch (OutOfMemoryError e) {
             freeMemory();
             // out of memory not detected
-            throw (Error) new AssertionError("Out of memory not detected").initCause(e);
+            throw new AssertionError("Out of memory not detected", e);
         } finally {
             freeMemory();
             if (conn != null) {