提交 383aa317 authored 作者: Thomas Mueller's avatar Thomas Mueller

MVStore: out of memory while storing could corrupt the store

上级 888407d5
...@@ -21,6 +21,11 @@ Change Log ...@@ -21,6 +21,11 @@ Change Log
<h2>Next Version (unreleased)</h2> <h2>Next Version (unreleased)</h2>
<ul> <ul>
<li>MVStore: out of memory while storing could corrupt the store
(theoretically, a rollback would be possible, but this case is not yet implemented).
</li>
<li>The compressed in-memory file systems (memLZF:) could not be used in the MVStore.
</li>
<li>The in-memory file systems (memFS: and memLZF:) did not support files larger than 2 GB <li>The in-memory file systems (memFS: and memLZF:) did not support files larger than 2 GB
due to an integer overflow. due to an integer overflow.
</li> </li>
......
...@@ -1973,7 +1973,7 @@ public class Database implements DataHandler { ...@@ -1973,7 +1973,7 @@ public class Database implements DataHandler {
backgroundException = null; backgroundException = null;
if (b != null) { if (b != null) {
// wrap the exception, so we see it was thrown here // wrap the exception, so we see it was thrown here
throw DbException.get(b.getErrorCode(), b); throw DbException.get(b.getErrorCode(), b, b.getMessage());
} }
} }
} }
......
...@@ -1000,9 +1000,19 @@ public class MVStore { ...@@ -1000,9 +1000,19 @@ public class MVStore {
time = Math.max(lastChunk.time, time); time = Math.max(lastChunk.time, time);
} }
int newChunkId = lastChunkId; int newChunkId = lastChunkId;
do { while (true) {
newChunkId = (newChunkId + 1) % Chunk.MAX_ID; newChunkId = (newChunkId + 1) % Chunk.MAX_ID;
} while (chunks.containsKey(newChunkId)); Chunk old = chunks.get(newChunkId);
if (old == null) {
break;
}
if (old.block == Long.MAX_VALUE) {
IllegalStateException e = DataUtils.newIllegalStateException(
DataUtils.ERROR_INTERNAL,
"Last block not stored, possibly due to out-of-memory");
panic(e);
}
}
Chunk c = new Chunk(newChunkId); Chunk c = new Chunk(newChunkId);
c.pageCount = Integer.MAX_VALUE; c.pageCount = Integer.MAX_VALUE;
......
...@@ -143,8 +143,7 @@ public class MVPrimaryIndex extends BaseIndex { ...@@ -143,8 +143,7 @@ public class MVPrimaryIndex extends BaseIndex {
try { try {
map.put(key, ValueArray.get(row.getValueList())); map.put(key, ValueArray.get(row.getValueList()));
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, throw mvTable.convertException(e);
e, table.getName());
} }
lastKey = Math.max(lastKey, row.getKey()); lastKey = Math.max(lastKey, row.getKey());
} }
...@@ -167,8 +166,7 @@ public class MVPrimaryIndex extends BaseIndex { ...@@ -167,8 +166,7 @@ public class MVPrimaryIndex extends BaseIndex {
getSQL() + ": " + row.getKey()); getSQL() + ": " + row.getKey());
} }
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, throw mvTable.convertException(e);
e, table.getName());
} }
} }
......
...@@ -192,8 +192,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex { ...@@ -192,8 +192,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex {
try { try {
map.put(array, ValueNull.INSTANCE); map.put(array, ValueNull.INSTANCE);
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, throw mvTable.convertException(e);
e, table.getName());
} }
if (indexType.isUnique()) { if (indexType.isUnique()) {
Iterator<Value> it = map.keyIterator(unique, true); Iterator<Value> it = map.keyIterator(unique, true);
...@@ -247,8 +246,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex { ...@@ -247,8 +246,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex {
getSQL() + ": " + row.getKey()); getSQL() + ": " + row.getKey());
} }
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, throw mvTable.convertException(e);
e, table.getName());
} }
} }
......
...@@ -140,8 +140,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex { ...@@ -140,8 +140,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex {
try { try {
map.put(key, ValueLong.get(0)); map.put(key, ValueLong.get(0));
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, throw mvTable.convertException(e);
e, table.getName());
} }
if (indexType.isUnique()) { if (indexType.isUnique()) {
// check if there is another (uncommitted) entry // check if there is another (uncommitted) entry
...@@ -176,8 +175,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex { ...@@ -176,8 +175,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex {
getSQL() + ": " + row.getKey()); getSQL() + ": " + row.getKey());
} }
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, throw mvTable.convertException(e);
e, table.getName());
} }
} }
......
...@@ -27,6 +27,7 @@ import org.h2.index.IndexType; ...@@ -27,6 +27,7 @@ import org.h2.index.IndexType;
import org.h2.index.MultiVersionIndex; import org.h2.index.MultiVersionIndex;
import org.h2.message.DbException; import org.h2.message.DbException;
import org.h2.message.Trace; import org.h2.message.Trace;
import org.h2.mvstore.DataUtils;
import org.h2.mvstore.db.MVTableEngine.Store; import org.h2.mvstore.db.MVTableEngine.Store;
import org.h2.mvstore.db.TransactionStore.Transaction; import org.h2.mvstore.db.TransactionStore.Transaction;
import org.h2.result.Row; import org.h2.result.Row;
...@@ -66,12 +67,14 @@ public class MVTable extends TableBase { ...@@ -66,12 +67,14 @@ public class MVTable extends TableBase {
private boolean containsLargeObject; private boolean containsLargeObject;
private Column rowIdColumn; private Column rowIdColumn;
private final TransactionStore store; private final MVTableEngine.Store store;
private final TransactionStore transactionStore;
public MVTable(CreateTableData data, MVTableEngine.Store store) { public MVTable(CreateTableData data, MVTableEngine.Store store) {
super(data); super(data);
nextAnalyze = database.getSettings().analyzeAuto; nextAnalyze = database.getSettings().analyzeAuto;
this.store = store.getTransactionStore(); this.store = store;
this.transactionStore = store.getTransactionStore();
this.isHidden = data.isHidden; this.isHidden = data.isHidden;
for (Column col : getColumns()) { for (Column col : getColumns()) {
if (DataType.isLargeObject(col.getType())) { if (DataType.isLargeObject(col.getType())) {
...@@ -420,7 +423,7 @@ public class MVTable extends TableBase { ...@@ -420,7 +423,7 @@ public class MVTable extends TableBase {
int mainIndexColumn; int mainIndexColumn;
mainIndexColumn = getMainIndexColumn(indexType, cols); mainIndexColumn = getMainIndexColumn(indexType, cols);
if (database.isStarting()) { if (database.isStarting()) {
if (store.store.hasMap("index." + indexId)) { if (transactionStore.store.hasMap("index." + indexId)) {
mainIndexColumn = -1; mainIndexColumn = -1;
} }
} else if (primaryIndex.getRowCountMax() != 0) { } else if (primaryIndex.getRowCountMax() != 0) {
...@@ -786,7 +789,7 @@ public class MVTable extends TableBase { ...@@ -786,7 +789,7 @@ public class MVTable extends TableBase {
Transaction getTransaction(Session session) { Transaction getTransaction(Session session) {
if (session == null) { if (session == null) {
// TODO need to commit/rollback the transaction // TODO need to commit/rollback the transaction
return store.begin(); return transactionStore.begin();
} }
return session.getTransaction(); return session.getTransaction();
} }
...@@ -820,4 +823,13 @@ public class MVTable extends TableBase { ...@@ -820,4 +823,13 @@ public class MVTable extends TableBase {
} }
} }
DbException convertException(IllegalStateException e) {
if (DataUtils.getErrorCode(e.getMessage()) ==
DataUtils.ERROR_TRANSACTION_LOCKED) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, getName());
}
return store.convertIllegalStateException(e);
}
} }
...@@ -54,9 +54,9 @@ public class MVTableEngine implements TableEngine { ...@@ -54,9 +54,9 @@ public class MVTableEngine implements TableEngine {
byte[] key = db.getFileEncryptionKey(); byte[] key = db.getFileEncryptionKey();
String dbPath = db.getDatabasePath(); String dbPath = db.getDatabasePath();
MVStore.Builder builder = new MVStore.Builder(); MVStore.Builder builder = new MVStore.Builder();
if (dbPath == null) { store = new Store();
store = new Store(db, builder); boolean encrypted = false;
} else { if (dbPath != null) {
String fileName = dbPath + Constants.SUFFIX_MV_FILE; String fileName = dbPath + Constants.SUFFIX_MV_FILE;
MVStoreTool.compactCleanUp(fileName); MVStoreTool.compactCleanUp(fileName);
builder.fileName(fileName); builder.fileName(fileName);
...@@ -74,6 +74,7 @@ public class MVTableEngine implements TableEngine { ...@@ -74,6 +74,7 @@ public class MVTableEngine implements TableEngine {
} }
} }
if (key != null) { if (key != null) {
encrypted = true;
char[] password = new char[key.length / 2]; char[] password = new char[key.length / 2];
for (int i = 0; i < password.length; i++) { for (int i = 0; i < password.length; i++) {
password[i] = (char) (((key[i + i] & 255) << 16) | password[i] = (char) (((key[i + i] & 255) << 16) |
...@@ -94,30 +95,8 @@ public class MVTableEngine implements TableEngine { ...@@ -94,30 +95,8 @@ public class MVTableEngine implements TableEngine {
} }
}); });
try {
store = new Store(db, builder);
} catch (IllegalStateException e) {
int errorCode = DataUtils.getErrorCode(e.getMessage());
if (errorCode == DataUtils.ERROR_FILE_CORRUPT) {
if (key != null) {
throw DbException.get(
ErrorCode.FILE_ENCRYPTION_ERROR_1,
e, fileName);
}
} else if (errorCode == DataUtils.ERROR_FILE_LOCKED) {
throw DbException.get(
ErrorCode.DATABASE_ALREADY_OPEN_1,
e, fileName);
} else if (errorCode == DataUtils.ERROR_READING_FAILED) {
throw DbException.get(
ErrorCode.IO_EXCEPTION_1,
e, fileName);
}
throw DbException.get(
ErrorCode.FILE_CORRUPTED_1,
e, fileName);
}
} }
store.open(db, builder, encrypted);
db.setMvStore(store); db.setMvStore(store);
return store; return store;
} }
...@@ -147,26 +126,62 @@ public class MVTableEngine implements TableEngine { ...@@ -147,26 +126,62 @@ public class MVTableEngine implements TableEngine {
/** /**
* The store. * The store.
*/ */
private final MVStore store; private MVStore store;
/** /**
* The transaction store. * The transaction store.
*/ */
private final TransactionStore transactionStore; private TransactionStore transactionStore;
private long statisticsStart; private long statisticsStart;
private int temporaryMapId; private int temporaryMapId;
public Store(Database db, MVStore.Builder builder) { private boolean encrypted;
this.store = builder.open();
if (!db.getSettings().reuseSpace) { private String fileName;
store.setReuseSpace(false);
void open(Database db, MVStore.Builder builder, boolean encrypted) {
this.encrypted = encrypted;
try {
this.store = builder.open();
FileStore fs = store.getFileStore();
if (fs != null) {
this.fileName = fs.getFileName();
}
if (!db.getSettings().reuseSpace) {
store.setReuseSpace(false);
}
this.transactionStore = new TransactionStore(
store,
new ValueDataType(null, db, null));
transactionStore.init();
} catch (IllegalStateException e) {
throw convertIllegalStateException(e);
} }
this.transactionStore = new TransactionStore( }
store,
new ValueDataType(null, db, null)); DbException convertIllegalStateException(IllegalStateException e) {
transactionStore.init(); int errorCode = DataUtils.getErrorCode(e.getMessage());
if (errorCode == DataUtils.ERROR_FILE_CORRUPT) {
if (encrypted) {
throw DbException.get(
ErrorCode.FILE_ENCRYPTION_ERROR_1,
e, fileName);
}
} else if (errorCode == DataUtils.ERROR_FILE_LOCKED) {
throw DbException.get(
ErrorCode.DATABASE_ALREADY_OPEN_1,
e, fileName);
} else if (errorCode == DataUtils.ERROR_READING_FAILED) {
throw DbException.get(
ErrorCode.IO_EXCEPTION_1,
e, fileName);
}
throw DbException.get(
ErrorCode.FILE_CORRUPTED_1,
e, fileName);
} }
public MVStore getStore() { public MVStore getStore() {
......
...@@ -6,13 +6,20 @@ ...@@ -6,13 +6,20 @@
package org.h2.test.db; package org.h2.test.db;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.Map;
import org.h2.api.ErrorCode; import org.h2.api.ErrorCode;
import org.h2.mvstore.MVStore;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FilePathMem;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase; import org.h2.test.TestBase;
import org.h2.tools.DeleteDbFiles;
/** /**
* Tests out of memory situations. The database must not get corrupted, and * Tests out of memory situations. The database must not get corrupted, and
...@@ -31,6 +38,64 @@ public class TestOutOfMemory extends TestBase { ...@@ -31,6 +38,64 @@ public class TestOutOfMemory extends TestBase {
@Override @Override
public void test() throws SQLException { public void test() throws SQLException {
testMVStoreUsingInMemoryFileSystem();
testDatabaseUsingInMemoryFileSystem();
testUpdateWhenNearlyOutOfMemory();
}
private void testMVStoreUsingInMemoryFileSystem() {
FilePath.register(new FilePathMem());
String fileName = "memLZF:" + getTestName();
MVStore store = MVStore.open(fileName);
Map<Integer, byte[]> map = store.openMap("test");
byte[] data = new byte[10 * 1024 * 1024];
try {
for (int i = 0; i < 100; i++) {
map.put(i, data);
}
fail();
} catch (OutOfMemoryError e) {
// expected
}
data = null;
try {
store.close();
fail();
} catch (IllegalStateException e) {
// expected
}
store.closeImmediately();
store = MVStore.open(fileName);
map = store.openMap("test");
store.close();
FileUtils.delete(fileName);
}
private void testDatabaseUsingInMemoryFileSystem() throws SQLException {
String url = "jdbc:h2:memLZF:" + getTestName();
Connection conn = DriverManager.getConnection(url);
Statement stat = conn.createStatement();
try {
stat.execute("create table test(id int, name varchar) as " +
"select x, space(10000000) from system_range(1, 1000)");
fail();
} catch (SQLException e) {
assertEquals(ErrorCode.GENERAL_ERROR_1, e.getErrorCode());
}
try {
conn.close();
fail();
} catch (SQLException e) {
assertEquals(ErrorCode.GENERAL_ERROR_1, e.getErrorCode());
}
conn = DriverManager.getConnection(url);
stat = conn.createStatement();
stat.execute("select 1");
conn.close();
DeleteDbFiles.execute("memLZF:", getTestName(), true);
}
private void testUpdateWhenNearlyOutOfMemory() throws SQLException {
if (config.memory || config.mvcc) { if (config.memory || config.mvcc) {
return; return;
} }
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论