提交 383aa317 authored 作者: Thomas Mueller's avatar Thomas Mueller

MVStore: out of memory while storing could corrupt the store

上级 888407d5
......@@ -21,6 +21,11 @@ Change Log
<h2>Next Version (unreleased)</h2>
<ul>
<li>MVStore: out of memory while storing could corrupt the store
(theoretically, a rollback would be possible, but this case is not yet implemented).
</li>
<li>The compressed in-memory file systems (memLZF:) could not be used in the MVStore.
</li>
<li>The in-memory file systems (memFS: and memLZF:) did not support files larger than 2 GB
due to an integer overflow.
</li>
......
......@@ -1973,7 +1973,7 @@ public class Database implements DataHandler {
backgroundException = null;
if (b != null) {
// wrap the exception, so we see it was thrown here
throw DbException.get(b.getErrorCode(), b);
throw DbException.get(b.getErrorCode(), b, b.getMessage());
}
}
}
......
......@@ -1000,9 +1000,19 @@ public class MVStore {
time = Math.max(lastChunk.time, time);
}
int newChunkId = lastChunkId;
do {
while (true) {
newChunkId = (newChunkId + 1) % Chunk.MAX_ID;
} while (chunks.containsKey(newChunkId));
Chunk old = chunks.get(newChunkId);
if (old == null) {
break;
}
if (old.block == Long.MAX_VALUE) {
IllegalStateException e = DataUtils.newIllegalStateException(
DataUtils.ERROR_INTERNAL,
"Last block not stored, possibly due to out-of-memory");
panic(e);
}
}
Chunk c = new Chunk(newChunkId);
c.pageCount = Integer.MAX_VALUE;
......
......@@ -143,8 +143,7 @@ public class MVPrimaryIndex extends BaseIndex {
try {
map.put(key, ValueArray.get(row.getValueList()));
} catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, table.getName());
throw mvTable.convertException(e);
}
lastKey = Math.max(lastKey, row.getKey());
}
......@@ -167,8 +166,7 @@ public class MVPrimaryIndex extends BaseIndex {
getSQL() + ": " + row.getKey());
}
} catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, table.getName());
throw mvTable.convertException(e);
}
}
......
......@@ -192,8 +192,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex {
try {
map.put(array, ValueNull.INSTANCE);
} catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, table.getName());
throw mvTable.convertException(e);
}
if (indexType.isUnique()) {
Iterator<Value> it = map.keyIterator(unique, true);
......@@ -247,8 +246,7 @@ public class MVSecondaryIndex extends BaseIndex implements MVIndex {
getSQL() + ": " + row.getKey());
}
} catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, table.getName());
throw mvTable.convertException(e);
}
}
......
......@@ -140,8 +140,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex {
try {
map.put(key, ValueLong.get(0));
} catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, table.getName());
throw mvTable.convertException(e);
}
if (indexType.isUnique()) {
// check if there is another (uncommitted) entry
......@@ -176,8 +175,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex, MVIndex {
getSQL() + ": " + row.getKey());
}
} catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, table.getName());
throw mvTable.convertException(e);
}
}
......
......@@ -27,6 +27,7 @@ import org.h2.index.IndexType;
import org.h2.index.MultiVersionIndex;
import org.h2.message.DbException;
import org.h2.message.Trace;
import org.h2.mvstore.DataUtils;
import org.h2.mvstore.db.MVTableEngine.Store;
import org.h2.mvstore.db.TransactionStore.Transaction;
import org.h2.result.Row;
......@@ -66,12 +67,14 @@ public class MVTable extends TableBase {
private boolean containsLargeObject;
private Column rowIdColumn;
private final TransactionStore store;
private final MVTableEngine.Store store;
private final TransactionStore transactionStore;
public MVTable(CreateTableData data, MVTableEngine.Store store) {
super(data);
nextAnalyze = database.getSettings().analyzeAuto;
this.store = store.getTransactionStore();
this.store = store;
this.transactionStore = store.getTransactionStore();
this.isHidden = data.isHidden;
for (Column col : getColumns()) {
if (DataType.isLargeObject(col.getType())) {
......@@ -420,7 +423,7 @@ public class MVTable extends TableBase {
int mainIndexColumn;
mainIndexColumn = getMainIndexColumn(indexType, cols);
if (database.isStarting()) {
if (store.store.hasMap("index." + indexId)) {
if (transactionStore.store.hasMap("index." + indexId)) {
mainIndexColumn = -1;
}
} else if (primaryIndex.getRowCountMax() != 0) {
......@@ -786,7 +789,7 @@ public class MVTable extends TableBase {
Transaction getTransaction(Session session) {
if (session == null) {
// TODO need to commit/rollback the transaction
return store.begin();
return transactionStore.begin();
}
return session.getTransaction();
}
......@@ -820,4 +823,13 @@ public class MVTable extends TableBase {
}
}
DbException convertException(IllegalStateException e) {
if (DataUtils.getErrorCode(e.getMessage()) ==
DataUtils.ERROR_TRANSACTION_LOCKED) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1,
e, getName());
}
return store.convertIllegalStateException(e);
}
}
......@@ -54,9 +54,9 @@ public class MVTableEngine implements TableEngine {
byte[] key = db.getFileEncryptionKey();
String dbPath = db.getDatabasePath();
MVStore.Builder builder = new MVStore.Builder();
if (dbPath == null) {
store = new Store(db, builder);
} else {
store = new Store();
boolean encrypted = false;
if (dbPath != null) {
String fileName = dbPath + Constants.SUFFIX_MV_FILE;
MVStoreTool.compactCleanUp(fileName);
builder.fileName(fileName);
......@@ -74,6 +74,7 @@ public class MVTableEngine implements TableEngine {
}
}
if (key != null) {
encrypted = true;
char[] password = new char[key.length / 2];
for (int i = 0; i < password.length; i++) {
password[i] = (char) (((key[i + i] & 255) << 16) |
......@@ -94,30 +95,8 @@ public class MVTableEngine implements TableEngine {
}
});
try {
store = new Store(db, builder);
} catch (IllegalStateException e) {
int errorCode = DataUtils.getErrorCode(e.getMessage());
if (errorCode == DataUtils.ERROR_FILE_CORRUPT) {
if (key != null) {
throw DbException.get(
ErrorCode.FILE_ENCRYPTION_ERROR_1,
e, fileName);
}
} else if (errorCode == DataUtils.ERROR_FILE_LOCKED) {
throw DbException.get(
ErrorCode.DATABASE_ALREADY_OPEN_1,
e, fileName);
} else if (errorCode == DataUtils.ERROR_READING_FAILED) {
throw DbException.get(
ErrorCode.IO_EXCEPTION_1,
e, fileName);
}
throw DbException.get(
ErrorCode.FILE_CORRUPTED_1,
e, fileName);
}
}
store.open(db, builder, encrypted);
db.setMvStore(store);
return store;
}
......@@ -147,26 +126,62 @@ public class MVTableEngine implements TableEngine {
/**
* The store.
*/
private final MVStore store;
private MVStore store;
/**
* The transaction store.
*/
private final TransactionStore transactionStore;
private TransactionStore transactionStore;
private long statisticsStart;
private int temporaryMapId;
public Store(Database db, MVStore.Builder builder) {
this.store = builder.open();
if (!db.getSettings().reuseSpace) {
store.setReuseSpace(false);
private boolean encrypted;
private String fileName;
void open(Database db, MVStore.Builder builder, boolean encrypted) {
this.encrypted = encrypted;
try {
this.store = builder.open();
FileStore fs = store.getFileStore();
if (fs != null) {
this.fileName = fs.getFileName();
}
if (!db.getSettings().reuseSpace) {
store.setReuseSpace(false);
}
this.transactionStore = new TransactionStore(
store,
new ValueDataType(null, db, null));
transactionStore.init();
} catch (IllegalStateException e) {
throw convertIllegalStateException(e);
}
this.transactionStore = new TransactionStore(
store,
new ValueDataType(null, db, null));
transactionStore.init();
}
DbException convertIllegalStateException(IllegalStateException e) {
int errorCode = DataUtils.getErrorCode(e.getMessage());
if (errorCode == DataUtils.ERROR_FILE_CORRUPT) {
if (encrypted) {
throw DbException.get(
ErrorCode.FILE_ENCRYPTION_ERROR_1,
e, fileName);
}
} else if (errorCode == DataUtils.ERROR_FILE_LOCKED) {
throw DbException.get(
ErrorCode.DATABASE_ALREADY_OPEN_1,
e, fileName);
} else if (errorCode == DataUtils.ERROR_READING_FAILED) {
throw DbException.get(
ErrorCode.IO_EXCEPTION_1,
e, fileName);
}
throw DbException.get(
ErrorCode.FILE_CORRUPTED_1,
e, fileName);
}
public MVStore getStore() {
......
......@@ -6,13 +6,20 @@
package org.h2.test.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import org.h2.api.ErrorCode;
import org.h2.mvstore.MVStore;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FilePathMem;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.tools.DeleteDbFiles;
/**
* Tests out of memory situations. The database must not get corrupted, and
......@@ -31,6 +38,64 @@ public class TestOutOfMemory extends TestBase {
@Override
public void test() throws SQLException {
testMVStoreUsingInMemoryFileSystem();
testDatabaseUsingInMemoryFileSystem();
testUpdateWhenNearlyOutOfMemory();
}
private void testMVStoreUsingInMemoryFileSystem() {
FilePath.register(new FilePathMem());
String fileName = "memLZF:" + getTestName();
MVStore store = MVStore.open(fileName);
Map<Integer, byte[]> map = store.openMap("test");
byte[] data = new byte[10 * 1024 * 1024];
try {
for (int i = 0; i < 100; i++) {
map.put(i, data);
}
fail();
} catch (OutOfMemoryError e) {
// expected
}
data = null;
try {
store.close();
fail();
} catch (IllegalStateException e) {
// expected
}
store.closeImmediately();
store = MVStore.open(fileName);
map = store.openMap("test");
store.close();
FileUtils.delete(fileName);
}
private void testDatabaseUsingInMemoryFileSystem() throws SQLException {
String url = "jdbc:h2:memLZF:" + getTestName();
Connection conn = DriverManager.getConnection(url);
Statement stat = conn.createStatement();
try {
stat.execute("create table test(id int, name varchar) as " +
"select x, space(10000000) from system_range(1, 1000)");
fail();
} catch (SQLException e) {
assertEquals(ErrorCode.GENERAL_ERROR_1, e.getErrorCode());
}
try {
conn.close();
fail();
} catch (SQLException e) {
assertEquals(ErrorCode.GENERAL_ERROR_1, e.getErrorCode());
}
conn = DriverManager.getConnection(url);
stat = conn.createStatement();
stat.execute("select 1");
conn.close();
DeleteDbFiles.execute("memLZF:", getTestName(), true);
}
private void testUpdateWhenNearlyOutOfMemory() throws SQLException {
if (config.memory || config.mvcc) {
return;
}
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论