MVStore: out of memory while storing could corrupt the store

h2/src/docsrc/html/changelog.html

@@ -21,6 +21,11 @@ Change Log
 
 <h2>Next Version (unreleased)</h2>
 <ul>
+<li>MVStore: out of memory while storing could corrupt the store
+    (theoretically, a rollback would be possible, but this case is not yet implemented).
+</li>
+<li>The compressed in-memory file systems (memLZF:) could not be used in the MVStore.
+</li>
 <li>The in-memory file systems (memFS: and memLZF:) did not support files larger than 2 GB
     due to an integer overflow.
 </li>

h2/src/main/org/h2/engine/Database.java

@@ -1973,7 +1973,7 @@ public class Database implements DataHandler {
             backgroundException = null;
             if (b != null) {
                 // wrap the exception, so we see it was thrown here
-                throw DbException.get(b.getErrorCode(), b);
+                throw DbException.get(b.getErrorCode(), b, b.getMessage());
             }
         }
     }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -1000,9 +1000,19 @@ public class MVStore {
             time = Math.max(lastChunk.time, time);
         }
         int newChunkId = lastChunkId;
-        do {
+        while (true) {
             newChunkId = (newChunkId + 1) % Chunk.MAX_ID;
-        } while (chunks.containsKey(newChunkId));
+            Chunk old = chunks.get(newChunkId);
+            if (old == null) {
+                break;
+            }
+            if (old.block == Long.MAX_VALUE) {
+                IllegalStateException e = DataUtils.newIllegalStateException(
+                        DataUtils.ERROR_INTERNAL,
+                        "Last block not stored, possibly due to out-of-memory");
+                panic(e);
+            }
+        }
         Chunk c = new Chunk(newChunkId);
 
         c.pageCount = Integer.MAX_VALUE;

h2/src/main/org/h2/mvstore/db/MVPrimaryIndex.java

@@ -143,8 +143,7 @@ public class MVPrimaryIndex extends BaseIndex {
         try {
             map.put(key, ValueArray.get(row.getValueList()));
         } catch (IllegalStateException e) {
-            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
-                    e, table.getName());
+            throw mvTable.convertException(e);
         }
         lastKey = Math.max(lastKey, row.getKey());
     }
@@ -167,8 +166,7 @@ public class MVPrimaryIndex extends BaseIndex {
                         getSQL() + ": " + row.getKey());
             }
         } catch (IllegalStateException e) {
-            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
-                    e, table.getName());
+            throw mvTable.convertException(e);
         }
     }
 

h2/src/main/org/h2/mvstore/db/MVSecondaryIndex.java

@@ -192,8 +192,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex {
         try {
             map.put(array, ValueNull.INSTANCE);
         } catch (IllegalStateException e) {
-            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
-                    e, table.getName());
+            throw mvTable.convertException(e);
         }
         if (indexType.isUnique()) {
             Iterator<Value> it = map.keyIterator(unique, true);
@@ -247,8 +246,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex {
                         getSQL() + ": " + row.getKey());
             }
         } catch (IllegalStateException e) {
-            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
-                    e, table.getName());
+            throw mvTable.convertException(e);
         }
     }
 

h2/src/main/org/h2/mvstore/db/MVSpatialIndex.java

@@ -140,8 +140,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex {
         try {
             map.put(key, ValueLong.get(0));
         } catch (IllegalStateException e) {
-            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
-                    e, table.getName());
+            throw mvTable.convertException(e);
         }
         if (indexType.isUnique()) {
             // check if there is another (uncommitted) entry
@@ -176,8 +175,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex {
                         getSQL() + ": " + row.getKey());
             }
         } catch (IllegalStateException e) {
-            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
-                    e, table.getName());
+            throw mvTable.convertException(e);
         }
     }
 

h2/src/main/org/h2/mvstore/db/MVTable.java

@@ -27,6 +27,7 @@ import org.h2.index.IndexType;
 import org.h2.index.MultiVersionIndex;
 import org.h2.message.DbException;
 import org.h2.message.Trace;
+import org.h2.mvstore.DataUtils;
 import org.h2.mvstore.db.MVTableEngine.Store;
 import org.h2.mvstore.db.TransactionStore.Transaction;
 import org.h2.result.Row;
@@ -66,12 +67,14 @@ public class MVTable extends TableBase {
     private boolean containsLargeObject;
     private Column rowIdColumn;
 
-    private final TransactionStore store;
+    private final MVTableEngine.Store store;
+    private final TransactionStore transactionStore;
 
     public MVTable(CreateTableData data, MVTableEngine.Store store) {
         super(data);
         nextAnalyze = database.getSettings().analyzeAuto;
-        this.store = store.getTransactionStore();
+        this.store = store;
+        this.transactionStore = store.getTransactionStore();
         this.isHidden = data.isHidden;
         for (Column col : getColumns()) {
             if (DataType.isLargeObject(col.getType())) {
@@ -420,7 +423,7 @@ public class MVTable extends TableBase {
         int mainIndexColumn;
         mainIndexColumn = getMainIndexColumn(indexType, cols);
         if (database.isStarting()) {
-            if (store.store.hasMap("index." + indexId)) {
+            if (transactionStore.store.hasMap("index." + indexId)) {
                 mainIndexColumn = -1;
             }
         } else if (primaryIndex.getRowCountMax() != 0) {
@@ -786,7 +789,7 @@ public class MVTable extends TableBase {
     Transaction getTransaction(Session session) {
         if (session == null) {
             // TODO need to commit/rollback the transaction
-            return store.begin();
+            return transactionStore.begin();
         }
         return session.getTransaction();
     }
@@ -820,4 +823,13 @@ public class MVTable extends TableBase {
         }
     }
 
+    DbException convertException(IllegalStateException e) {
+        if (DataUtils.getErrorCode(e.getMessage()) ==
+                DataUtils.ERROR_TRANSACTION_LOCKED) {
+            throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
+                    e, getName());
+        }
+        return store.convertIllegalStateException(e);
+    }
+
 }

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -54,9 +54,9 @@ public class MVTableEngine implements TableEngine {
         byte[] key = db.getFileEncryptionKey();
         String dbPath = db.getDatabasePath();
         MVStore.Builder builder = new MVStore.Builder();
-        if (dbPath == null) {
-            store = new Store(db, builder);
-        } else {
+        store = new Store();
+        boolean encrypted = false;
+        if (dbPath != null) {
             String fileName = dbPath + Constants.SUFFIX_MV_FILE;
             MVStoreTool.compactCleanUp(fileName);
             builder.fileName(fileName);
@@ -74,6 +74,7 @@ public class MVTableEngine implements TableEngine {
                 }
             }
             if (key != null) {
+                encrypted = true;
                 char[] password = new char[key.length / 2];
                 for (int i = 0; i < password.length; i++) {
                     password[i] = (char) (((key[i + i] & 255) << 16) |
@@ -94,30 +95,8 @@ public class MVTableEngine implements TableEngine {
                 }
 
             });
-            try {
-                store = new Store(db, builder);
-            } catch (IllegalStateException e) {
-                int errorCode = DataUtils.getErrorCode(e.getMessage());
-                if (errorCode == DataUtils.ERROR_FILE_CORRUPT) {
-                    if (key != null) {
-                        throw DbException.get(
-                                ErrorCode.FILE_ENCRYPTION_ERROR_1,
-                                e, fileName);
-                    }
-                } else if (errorCode == DataUtils.ERROR_FILE_LOCKED) {
-                    throw DbException.get(
-                            ErrorCode.DATABASE_ALREADY_OPEN_1,
-                            e, fileName);
-                } else if (errorCode == DataUtils.ERROR_READING_FAILED) {
-                    throw DbException.get(
-                            ErrorCode.IO_EXCEPTION_1,
-                            e, fileName);
-                }
-                throw DbException.get(
-                        ErrorCode.FILE_CORRUPTED_1,
-                        e, fileName);
-            }
         }
+        store.open(db, builder, encrypted);
         db.setMvStore(store);
         return store;
     }
@@ -147,26 +126,62 @@ public class MVTableEngine implements TableEngine {
         /**
          * The store.
          */
-        private final MVStore store;
+        private MVStore store;
 
         /**
          * The transaction store.
          */
-        private final TransactionStore transactionStore;
+        private TransactionStore transactionStore;
 
         private long statisticsStart;
 
         private int temporaryMapId;
 
-        public Store(Database db, MVStore.Builder builder) {
-            this.store = builder.open();
-            if (!db.getSettings().reuseSpace) {
-                store.setReuseSpace(false);
+        private boolean encrypted;
+
+        private String fileName;
+
+        void open(Database db, MVStore.Builder builder, boolean encrypted) {
+            this.encrypted = encrypted;
+            try {
+                this.store = builder.open();
+                FileStore fs = store.getFileStore();
+                if (fs != null) {
+                    this.fileName = fs.getFileName();
+                }
+                if (!db.getSettings().reuseSpace) {
+                    store.setReuseSpace(false);
+                }
+                this.transactionStore = new TransactionStore(
+                        store,
+                        new ValueDataType(null, db, null));
+                transactionStore.init();
+            } catch (IllegalStateException e) {
+                throw convertIllegalStateException(e);
             }
-            this.transactionStore = new TransactionStore(
-                    store,
-                    new ValueDataType(null, db, null));
-            transactionStore.init();
+        }
+
+        DbException convertIllegalStateException(IllegalStateException e) {
+            int errorCode = DataUtils.getErrorCode(e.getMessage());
+            if (errorCode == DataUtils.ERROR_FILE_CORRUPT) {
+                if (encrypted) {
+                    throw DbException.get(
+                            ErrorCode.FILE_ENCRYPTION_ERROR_1,
+                            e, fileName);
+                }
+            } else if (errorCode == DataUtils.ERROR_FILE_LOCKED) {
+                throw DbException.get(
+                        ErrorCode.DATABASE_ALREADY_OPEN_1,
+                        e, fileName);
+            } else if (errorCode == DataUtils.ERROR_READING_FAILED) {
+                throw DbException.get(
+                        ErrorCode.IO_EXCEPTION_1,
+                        e, fileName);
+            }
+            throw DbException.get(
+                    ErrorCode.FILE_CORRUPTED_1,
+                    e, fileName);
+
         }
 
         public MVStore getStore() {

h2/src/test/org/h2/test/db/TestOutOfMemory.java

@@ -6,13 +6,20 @@
 package org.h2.test.db;
 
 import java.sql.Connection;
+import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Map;
 
 import org.h2.api.ErrorCode;
+import org.h2.mvstore.MVStore;
+import org.h2.store.fs.FilePath;
+import org.h2.store.fs.FilePathMem;
+import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
+import org.h2.tools.DeleteDbFiles;
 
 /**
  * Tests out of memory situations. The database must not get corrupted, and
@@ -31,6 +38,64 @@ public class TestOutOfMemory extends TestBase {
 
     @Override
     public void test() throws SQLException {
+        testMVStoreUsingInMemoryFileSystem();
+        testDatabaseUsingInMemoryFileSystem();
+        testUpdateWhenNearlyOutOfMemory();
+    }
+
+    private void testMVStoreUsingInMemoryFileSystem() {
+        FilePath.register(new FilePathMem());
+        String fileName = "memLZF:" + getTestName();
+        MVStore store = MVStore.open(fileName);
+        Map<Integer, byte[]> map = store.openMap("test");
+        byte[] data = new byte[10 * 1024 * 1024];
+        try {
+            for (int i = 0; i < 100; i++) {
+                map.put(i, data);
+            }
+            fail();
+        } catch (OutOfMemoryError e) {
+            // expected
+        }
+        data = null;
+        try {
+            store.close();
+            fail();
+        } catch (IllegalStateException e) {
+            // expected
+        }
+        store.closeImmediately();
+        store = MVStore.open(fileName);
+        map = store.openMap("test");
+        store.close();
+        FileUtils.delete(fileName);
+    }
+
+    private void testDatabaseUsingInMemoryFileSystem() throws SQLException {
+        String url = "jdbc:h2:memLZF:" + getTestName();
+        Connection conn = DriverManager.getConnection(url);
+        Statement stat = conn.createStatement();
+        try {
+            stat.execute("create table test(id int, name varchar) as " +
+                    "select x, space(10000000) from system_range(1, 1000)");
+            fail();
+        } catch (SQLException e) {
+            assertEquals(ErrorCode.GENERAL_ERROR_1, e.getErrorCode());
+        }
+        try {
+            conn.close();
+            fail();
+        } catch (SQLException e) {
+            assertEquals(ErrorCode.GENERAL_ERROR_1, e.getErrorCode());
+        }
+        conn = DriverManager.getConnection(url);
+        stat = conn.createStatement();
+        stat.execute("select 1");
+        conn.close();
+        DeleteDbFiles.execute("memLZF:", getTestName(), true);
+    }
+
+    private void testUpdateWhenNearlyOutOfMemory() throws SQLException {
         if (config.memory || config.mvcc) {
             return;
         }