Merge pull request #867 from h2database/test_out_of_memory

TestOutOfMemory stability

h2/src/main/org/h2/engine/Database.java

@@ -209,6 +209,8 @@ public class Database implements DataHandler {
     private RowFactory rowFactory = RowFactory.DEFAULT;
 
     public Database(ConnectionInfo ci, String cipher) {
+        META_LOCK_DEBUGGING.set(null);
+        META_LOCK_DEBUGGING_STACK.set(null);
         String name = ci.getName();
         this.dbSettings = ci.getDbSettings();
         this.reconnectCheckDelayNs = TimeUnit.MILLISECONDS.toNanos(dbSettings.reconnectCheckDelay);
@@ -1256,110 +1258,112 @@ public class Database implements DataHandler {
      *            hook
      */
     void close(boolean fromShutdownHook) {
-        synchronized (this) {
-            if (closing) {
-                return;
-            }
-            throwLastBackgroundException();
-            if (fileLockMethod == FileLockMethod.SERIALIZED &&
-                    !reconnectChangePending) {
-                // another connection may have written something - don't write
-                try {
-                    closeOpenFilesAndUnlock(false);
-                } catch (DbException e) {
-                    // ignore
-                }
-                traceSystem.close();
-                Engine.getInstance().close(databaseName);
-                return;
-            }
-            closing = true;
-            stopServer();
-            if (!userSessions.isEmpty()) {
-                if (!fromShutdownHook) {
+        try {
+            synchronized (this) {
+                if (closing) {
                     return;
                 }
-                trace.info("closing {0} from shutdown hook", databaseName);
-                closeAllSessionsException(null);
-            }
-            trace.info("closing {0}", databaseName);
-            if (eventListener != null) {
-                // allow the event listener to connect to the database
-                closing = false;
-                DatabaseEventListener e = eventListener;
-                // set it to null, to make sure it's called only once
-                eventListener = null;
-                e.closingDatabase();
-                if (!userSessions.isEmpty()) {
-                    // if a connection was opened, we can't close the database
+                throwLastBackgroundException();
+                if (fileLockMethod == FileLockMethod.SERIALIZED &&
+                        !reconnectChangePending) {
+                    // another connection may have written something - don't write
+                    try {
+                        closeOpenFilesAndUnlock(false);
+                    } catch (DbException e) {
+                        // ignore
+                    }
+                    traceSystem.close();
                     return;
                 }
                 closing = true;
+                stopServer();
+                if (!userSessions.isEmpty()) {
+                    if (!fromShutdownHook) {
+                        return;
+                    }
+                    trace.info("closing {0} from shutdown hook", databaseName);
+                    closeAllSessionsException(null);
+                }
+                trace.info("closing {0}", databaseName);
+                if (eventListener != null) {
+                    // allow the event listener to connect to the database
+                    closing = false;
+                    DatabaseEventListener e = eventListener;
+                    // set it to null, to make sure it's called only once
+                    eventListener = null;
+                    e.closingDatabase();
+                    if (!userSessions.isEmpty()) {
+                        // if a connection was opened, we can't close the database
+                        return;
+                    }
+                    closing = true;
+                }
             }
-        }
-        removeOrphanedLobs();
-        try {
-            if (systemSession != null) {
-                if (powerOffCount != -1) {
-                    for (Table table : getAllTablesAndViews(false)) {
-                        if (table.isGlobalTemporary()) {
-                            table.removeChildrenAndResources(systemSession);
-                        } else {
-                            table.close(systemSession);
+            removeOrphanedLobs();
+            try {
+                if (systemSession != null) {
+                    if (powerOffCount != -1) {
+                        for (Table table : getAllTablesAndViews(false)) {
+                            if (table.isGlobalTemporary()) {
+                                table.removeChildrenAndResources(systemSession);
+                            } else {
+                                table.close(systemSession);
+                            }
+                        }
+                        for (SchemaObject obj : getAllSchemaObjects(
+                                DbObject.SEQUENCE)) {
+                            Sequence sequence = (Sequence) obj;
+                            sequence.close();
                         }
                     }
                     for (SchemaObject obj : getAllSchemaObjects(
-                            DbObject.SEQUENCE)) {
-                        Sequence sequence = (Sequence) obj;
-                        sequence.close();
+                            DbObject.TRIGGER)) {
+                        TriggerObject trigger = (TriggerObject) obj;
+                        try {
+                            trigger.close();
+                        } catch (SQLException e) {
+                            trace.error(e, "close");
+                        }
                     }
-                }
-                for (SchemaObject obj : getAllSchemaObjects(
-                        DbObject.TRIGGER)) {
-                    TriggerObject trigger = (TriggerObject) obj;
-                    try {
-                        trigger.close();
-                    } catch (SQLException e) {
-                        trace.error(e, "close");
+                    if (powerOffCount != -1) {
+                        meta.close(systemSession);
+                        systemSession.commit(true);
                     }
                 }
-                if (powerOffCount != -1) {
-                    meta.close(systemSession);
-                    systemSession.commit(true);
-                }
+            } catch (DbException e) {
+                trace.error(e, "close");
             }
-        } catch (DbException e) {
-            trace.error(e, "close");
-        }
-        tempFileDeleter.deleteAll();
-        try {
-            closeOpenFilesAndUnlock(true);
-        } catch (DbException e) {
-            trace.error(e, "close");
-        }
-        trace.info("closed");
-        traceSystem.close();
-        if (closeOnExit != null) {
-            closeOnExit.reset();
+            tempFileDeleter.deleteAll();
             try {
-                Runtime.getRuntime().removeShutdownHook(closeOnExit);
-            } catch (IllegalStateException e) {
-                // ignore
-            } catch (SecurityException  e) {
-                // applets may not do that - ignore
+                closeOpenFilesAndUnlock(true);
+            } catch (DbException e) {
+                trace.error(e, "close");
             }
-            closeOnExit = null;
-        }
-        Engine.getInstance().close(databaseName);
-        if (deleteFilesOnDisconnect && persistent) {
-            deleteFilesOnDisconnect = false;
-            try {
-                String directory = FileUtils.getParent(databaseName);
-                String name = FileUtils.getName(databaseName);
-                DeleteDbFiles.execute(directory, name, true);
-            } catch (Exception e) {
-                // ignore (the trace is closed already)
+            trace.info("closed");
+            traceSystem.close();
+            if (closeOnExit != null) {
+                closeOnExit.reset();
+                try {
+                    Runtime.getRuntime().removeShutdownHook(closeOnExit);
+                } catch (IllegalStateException e) {
+                    // ignore
+                } catch (SecurityException e) {
+                    // applets may not do that - ignore
+                }
+                closeOnExit = null;
+            }
+            if (deleteFilesOnDisconnect && persistent) {
+                deleteFilesOnDisconnect = false;
+                try {
+                    String directory = FileUtils.getParent(databaseName);
+                    String name = FileUtils.getName(databaseName);
+                    DeleteDbFiles.execute(directory, name, true);
+                } catch (Exception e) {
+                    // ignore (the trace is closed already)
+                }
             }
+        } finally {
+            Engine.getInstance().close(databaseName);
         }
     }
 

h2/src/main/org/h2/engine/DatabaseCloser.java

@@ -69,7 +69,8 @@ class DatabaseCloser extends Thread {
                     trace.error(e, "could not close the database");
                     // if this was successful, we ignore the exception
                     // otherwise not
-                } catch (RuntimeException e2) {
+                } catch (Throwable e2) {
+                    e.addSuppressed(e2);
                     throw e;
                 }
             }

h2/src/main/org/h2/engine/Engine.java

@@ -6,6 +6,7 @@
 package org.h2.engine;
 
 import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 import org.h2.api.ErrorCode;
 import org.h2.command.CommandInterface;
@@ -27,7 +28,7 @@ import org.h2.util.Utils;
 public class Engine implements SessionFactory {
 
     private static final Engine INSTANCE = new Engine();
-    private static final HashMap<String, Database> DATABASES = new HashMap<>();
+    private static final Map<String, Database> DATABASES = new HashMap<>();
 
     private volatile long wrongPasswordDelay =
             SysProperties.DELAY_WRONG_PASSWORD_MIN;
@@ -50,30 +51,32 @@ public class Engine implements SessionFactory {
         Database database;
         ci.removeProperty("NO_UPGRADE", false);
         boolean openNew = ci.getProperty("OPEN_NEW", false);
-        if (openNew || ci.isUnnamedInMemory()) {
-            database = null;
-        } else {
-            database = DATABASES.get(name);
-        }
-        User user = null;
         boolean opened = false;
-        if (database == null) {
-            if (ifExists && !Database.exists(name)) {
-                throw DbException.get(ErrorCode.DATABASE_NOT_FOUND_1, name);
-            }
-            database = new Database(ci, cipher);
-            opened = true;
-            if (database.getAllUsers().isEmpty()) {
-                // users is the last thing we add, so if no user is around,
-                // the database is new (or not initialized correctly)
-                user = new User(database, database.allocateObjectId(),
-                        ci.getUserName(), false);
-                user.setAdmin(true);
-                user.setUserPasswordHash(ci.getUserPasswordHash());
-                database.setMasterUser(user);
+        User user = null;
+        synchronized (DATABASES) {
+            if (openNew || ci.isUnnamedInMemory()) {
+                database = null;
+            } else {
+                database = DATABASES.get(name);
             }
-            if (!ci.isUnnamedInMemory()) {
-                DATABASES.put(name, database);
+            if (database == null) {
+                if (ifExists && !Database.exists(name)) {
+                    throw DbException.get(ErrorCode.DATABASE_NOT_FOUND_1, name);
+                }
+                database = new Database(ci, cipher);
+                opened = true;
+                if (database.getAllUsers().isEmpty()) {
+                    // users is the last thing we add, so if no user is around,
+                    // the database is new (or not initialized correctly)
+                    user = new User(database, database.allocateObjectId(),
+                            ci.getUserName(), false);
+                    user.setAdmin(true);
+                    user.setUserPasswordHash(ci.getUserPasswordHash());
+                    database.setMasterUser(user);
+                }
+                if (!ci.isUnnamedInMemory()) {
+                    DATABASES.put(name, database);
+                }
             }
         }
         if (opened) {
@@ -272,7 +275,9 @@ public class Engine implements SessionFactory {
                 throw DbException.get(ErrorCode.FEATURE_NOT_SUPPORTED_1, e, "JMX");
             }
         }
-        DATABASES.remove(name);
+        synchronized (DATABASES) {
+            DATABASES.remove(name);
+        }
     }
 
     /**

h2/src/main/org/h2/jdbc/JdbcConnection.java

@@ -411,7 +411,7 @@ public class JdbcConnection extends TraceObject
                     session = null;
                 }
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             throw logAndConvert(e);
         }
     }

h2/src/main/org/h2/jdbc/JdbcPreparedStatement.java

@@ -248,7 +248,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements
             } finally {
                 afterWriting();
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             throw logAndConvert(e);
         }
     }

h2/src/main/org/h2/message/DbException.java

@@ -273,7 +273,7 @@ public class DbException extends RuntimeException {
      * @param e the root cause
      * @return the SQL exception object
      */
-    public static SQLException toSQLException(Exception e) {
+    public static SQLException toSQLException(Throwable e) {
         if (e instanceof SQLException) {
             return (SQLException) e;
         }

h2/src/main/org/h2/message/TraceObject.java

@@ -351,17 +351,25 @@ public class TraceObject {
      * @param ex the exception
      * @return the SQL exception object
      */
-    protected SQLException logAndConvert(Exception ex) {
-        SQLException e = DbException.toSQLException(ex);
-        if (trace == null) {
-            DbException.traceThrowable(e);
-        } else {
-            int errorCode = e.getErrorCode();
-            if (errorCode >= 23000 && errorCode < 24000) {
-                trace.info(e, "exception");
+    protected SQLException logAndConvert(Throwable ex) {
+        SQLException e = null;
+        try {
+            e = DbException.toSQLException(ex);
+            if (trace == null) {
+                DbException.traceThrowable(e);
             } else {
-                trace.error(e, "exception");
+                int errorCode = e.getErrorCode();
+                if (errorCode >= 23000 && errorCode < 24000) {
+                    trace.info(e, "exception");
+                } else {
+                    trace.error(e, "exception");
+                }
+            }
+        } catch(Throwable ignore) {
+            if (e == null) {
+                e = new SQLException("", "HY000", ex);
             }
+            e.addSuppressed(ignore);
         }
         return e;
     }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -380,9 +380,7 @@ public final class MVStore {
     }
 
     private void panic(IllegalStateException e) {
-        if (backgroundExceptionHandler != null) {
-            backgroundExceptionHandler.uncaughtException(null, e);
-        }
+        handleException(e);
         panicException = e;
         closeImmediately();
         throw e;
@@ -862,10 +860,8 @@ public final class MVStore {
     public void closeImmediately() {
         try {
             closeStore(false);
-        } catch (Exception e) {
-            if (backgroundExceptionHandler != null) {
-                backgroundExceptionHandler.uncaughtException(null, e);
-            }
+        } catch (Throwable e) {
+            handleException(e);
         }
     }
 
@@ -2468,11 +2464,9 @@ public final class MVStore {
         if (hasUnsavedChanges()) {
             try {
                 commitAndSave();
-            } catch (Exception e) {
-                if (backgroundExceptionHandler != null) {
-                    backgroundExceptionHandler.uncaughtException(null, e);
-                    return;
-                }
+            } catch (Throwable e) {
+                handleException(e);
+                return;
             }
         }
         if (autoCompactFillRate > 0) {
@@ -2492,10 +2486,18 @@ public final class MVStore {
                 // in the bookkeeping?
                 compact(fillRate, autoCommitMemory);
                 autoCompactLastFileOpCount = fileStore.getWriteCount() + fileStore.getReadCount();
-            } catch (Exception e) {
-                if (backgroundExceptionHandler != null) {
-                    backgroundExceptionHandler.uncaughtException(null, e);
-                }
+            } catch (Throwable e) {
+                handleException(e);
+            }
+        }
+    }
+
+    private void handleException(Throwable ex) {
+        if (backgroundExceptionHandler != null) {
+            try {
+                backgroundExceptionHandler.uncaughtException(null, ex);
+            } catch(Throwable ignore) {
+                ex.addSuppressed(ignore);
             }
         }
     }

h2/src/test/org/h2/test/TestBase.java

@@ -79,6 +79,8 @@ public abstract class TestBase {
 
     private final LinkedList<byte[]> memory = new LinkedList<>();
 
+    private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
     /**
      * Get the test directory for this test.
      *
@@ -545,7 +547,6 @@ public abstract class TestBase {
      * @param s the message
      */
     static synchronized void printlnWithTime(long millis, String s) {
-        SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
         s = dateFormat.format(new java.util.Date()) + " " +
                 formatTime(millis) + " " + s;
         System.out.println(s);
@@ -1470,6 +1471,12 @@ public abstract class TestBase {
      */
     protected void freeMemory() {
         memory.clear();
+        for (int i = 0; i < 5; i++) {
+            System.gc();
+            try {
+                Thread.sleep(20);
+            } catch (InterruptedException ignore) {/**/}
+        }
     }
 
     /**

h2/src/test/org/h2/test/db/TestOutOfMemory.java

@@ -13,7 +13,9 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicReference;
 import org.h2.api.ErrorCode;
+import org.h2.message.DbException;
 import org.h2.mvstore.MVStore;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FilePathMem;
@@ -37,10 +39,6 @@ public class TestOutOfMemory extends TestBase {
 
     @Override
     public void test() throws SQLException, InterruptedException {
-        if (config.travis) {
-            // fails regularly under Travis, not sure why
-            return;
-        }
         if (config.vmlens) {
             // running out of memory will cause the vmlens agent to stop working
             return;
@@ -60,7 +58,16 @@ public class TestOutOfMemory extends TestBase {
     private void testMVStoreUsingInMemoryFileSystem() {
         FilePath.register(new FilePathMem());
         String fileName = "memFS:" + getTestName();
-        MVStore store = MVStore.open(fileName);
+        final AtomicReference<Throwable> exRef = new AtomicReference<>();
+        MVStore store = new MVStore.Builder()
+                .fileName(fileName)
+                .backgroundExceptionHandler(new Thread.UncaughtExceptionHandler() {
+                    @Override
+                    public void uncaughtException(Thread t, Throwable e) {
+                        exRef.compareAndSet(null, e);
+                    }
+                })
+                .open();
         try {
             Map<Integer, byte[]> map = store.openMap("test");
             Random r = new Random(1);
@@ -70,10 +77,11 @@ public class TestOutOfMemory extends TestBase {
                     r.nextBytes(data);
                     map.put(i, data);
                 }
+                Throwable throwable = exRef.get();
+                if(throwable instanceof OutOfMemoryError) throw (OutOfMemoryError)throwable;
+                if(throwable instanceof IllegalStateException) throw (IllegalStateException)throwable;
                 fail();
-            } catch (OutOfMemoryError e) {
-                // expected
-            } catch (IllegalStateException e) {
+            } catch (OutOfMemoryError | IllegalStateException e) {
                 // expected
             }
             try {
@@ -83,7 +91,7 @@ public class TestOutOfMemory extends TestBase {
             }
             store.closeImmediately();
             store = MVStore.open(fileName);
-            map = store.openMap("test");
+            store.openMap("test");
             store.close();
         } finally {
             // just in case, otherwise if this test suffers a spurious failure,
@@ -95,47 +103,55 @@ public class TestOutOfMemory extends TestBase {
 
     private void testDatabaseUsingInMemoryFileSystem() throws SQLException, InterruptedException {
         String filename = "memFS:" + getTestName();
-        String url = "jdbc:h2:" + filename;
-        Connection conn = DriverManager.getConnection(url);
-        Statement stat = conn.createStatement();
-        try {
-            stat.execute("create table test(id int, name varchar) as " +
-                    "select x, space(10000000) from system_range(1, 1000)");
-            fail();
-        } catch (SQLException e) {
-            int err = e.getErrorCode();
-            assertTrue(e.getMessage(), err == ErrorCode.GENERAL_ERROR_1
-                    || err == ErrorCode.OUT_OF_MEMORY);
-        }
+        String url = "jdbc:h2:" + filename + "/test";
         try {
+            Connection conn = DriverManager.getConnection(url);
+            Statement stat = conn.createStatement();
+            try {
+                stat.execute("create table test(id int, name varchar) as " +
+                        "select x, space(10000000+x) from system_range(1, 1000)");
+                fail();
+            } catch (SQLException e) {
+                assertTrue("Unexpected error code: " + e.getErrorCode(),
+                        ErrorCode.OUT_OF_MEMORY == e.getErrorCode() ||
+                        ErrorCode.FILE_CORRUPTED_1 == e.getErrorCode() ||
+                        ErrorCode.DATABASE_IS_CLOSED == e.getErrorCode() ||
+                        ErrorCode.GENERAL_ERROR_1 == e.getErrorCode());
+            }
+            recoverAfterOOM();
+            try {
+                conn.close();
+                fail();
+            } catch (SQLException e) {
+                assertTrue("Unexpected error code: " + e.getErrorCode(),
+                        ErrorCode.OUT_OF_MEMORY == e.getErrorCode() ||
+                        ErrorCode.FILE_CORRUPTED_1 == e.getErrorCode() ||
+                        ErrorCode.DATABASE_IS_CLOSED == e.getErrorCode() ||
+                        ErrorCode.GENERAL_ERROR_1 == e.getErrorCode());
+            }
+            recoverAfterOOM();
+            conn = DriverManager.getConnection(url);
+            stat = conn.createStatement();
+            stat.execute("SELECT 1");
             conn.close();
-            fail();
-        } catch (SQLException e) {
-            int err = e.getErrorCode();
-            assertTrue(e.getMessage(), err == ErrorCode.GENERAL_ERROR_1
-                    || err == ErrorCode.OUT_OF_MEMORY
-                    || err == ErrorCode.DATABASE_IS_CLOSED);
+        } finally {
+            // release the static data this test generates
+            FileUtils.deleteRecursive(filename, true);
         }
+    }
+
+    private static void recoverAfterOOM() throws InterruptedException {
         for (int i = 0; i < 5; i++) {
             System.gc();
             Thread.sleep(20);
         }
-        conn = DriverManager.getConnection(url);
-        stat = conn.createStatement();
-        stat.execute("select 1");
-        conn.close();
-        // release the static data this test generates
-        FileUtils.delete(filename);
     }
 
     private void testUpdateWhenNearlyOutOfMemory() throws SQLException, InterruptedException {
-        if (config.memory || config.mvcc || config.mvStore) {
+        if (config.memory) {
             return;
         }
-        for (int i = 0; i < 5; i++) {
-            System.gc();
-            Thread.sleep(20);
-        }
+        recoverAfterOOM();
         deleteDb("outOfMemory");
         Connection conn = getConnection("outOfMemory;MAX_OPERATION_MEMORY=1000000");
         Statement stat = conn.createStatement();
@@ -148,8 +164,27 @@ public class TestOutOfMemory extends TestBase {
         stat.execute("checkpoint");
         eatMemory(80);
         try {
-            assertThrows(ErrorCode.OUT_OF_MEMORY, prep).execute();
-            assertThrows(ErrorCode.DATABASE_IS_CLOSED, conn).close();
+            try {
+                prep.execute();
+                fail();
+            } catch(DbException ex) {
+                freeMemory();
+                assertTrue(ErrorCode.OUT_OF_MEMORY == ex.getErrorCode() || ErrorCode.GENERAL_ERROR_1 == ex.getErrorCode());
+            } catch (SQLException ex) {
+                freeMemory();
+                assertTrue(ErrorCode.OUT_OF_MEMORY == ex.getErrorCode() || ErrorCode.GENERAL_ERROR_1 == ex.getErrorCode());
+            }
+            recoverAfterOOM();
+            try {
+                conn.close();
+                fail();
+            } catch(DbException ex) {
+                freeMemory();
+                assertEquals(ErrorCode.DATABASE_IS_CLOSED, ex.getErrorCode());
+            } catch (SQLException ex) {
+                freeMemory();
+                assertEquals(ErrorCode.DATABASE_IS_CLOSED, ex.getErrorCode());
+            }
             freeMemory();
             conn = null;
             conn = getConnection("outOfMemory");
@@ -160,7 +195,7 @@ public class TestOutOfMemory extends TestBase {
         } catch (OutOfMemoryError e) {
             freeMemory();
             // out of memory not detected
-            throw (Error) new AssertionError("Out of memory not detected").initCause(e);
+            throw new AssertionError("Out of memory not detected", e);
         } finally {
             freeMemory();
             if (conn != null) {