提交 cd85f642 authored 作者: Thomas Mueller's avatar Thomas Mueller

References to BLOB and CLOB objects now have a timeout. The configuration…

References to BLOB and CLOB objects now have a timeout. The configuration setting is LOB_TIMEOUT (default 5 minutes). This should avoid growing the database file if there are many queries that return BLOB or CLOB objects, and the database is not closed for a longer time.
上级 842d1042
...@@ -804,6 +804,13 @@ public class ErrorCode { ...@@ -804,6 +804,13 @@ public class ErrorCode {
*/ */
public static final int VIEW_ALREADY_EXISTS_1 = 90038; public static final int VIEW_ALREADY_EXISTS_1 = 90038;
/**
* The error with code <code>90039</code> is thrown when
* trying to access a CLOB or BLOB object that timed out.
* See the database setting LOB_TIMEOUT.
*/
public static final int LOB_CLOSED_ON_TIMEOUT_1 = 90039;
/** /**
* The error with code <code>90040</code> is thrown when * The error with code <code>90040</code> is thrown when
* a user that is not administrator tries to execute a statement * a user that is not administrator tries to execute a statement
...@@ -1907,7 +1914,7 @@ public class ErrorCode { ...@@ -1907,7 +1914,7 @@ public class ErrorCode {
public static final int STEP_SIZE_MUST_NOT_BE_ZERO = 90142; public static final int STEP_SIZE_MUST_NOT_BE_ZERO = 90142;
// next are 90039, 90051, 90056, 90110, 90122, 90143 // next are 90051, 90056, 90110, 90122, 90143
private ErrorCode() { private ErrorCode() {
// utility class // utility class
......
...@@ -133,6 +133,15 @@ public class DbSettings extends SettingsBase { ...@@ -133,6 +133,15 @@ public class DbSettings extends SettingsBase {
*/ */
public final boolean largeTransactions = get("LARGE_TRANSACTIONS", true); public final boolean largeTransactions = get("LARGE_TRANSACTIONS", true);
/**
* Database setting <code>LOB_TIMEOUT</code> (default: 300000,
* which means 5 minutes).<br />
* The number of milliseconds a temporary LOB reference is kept until it
* times out. After the timeout, the LOB is no longer accessible using this
* reference.
*/
public final int lobTimeout = get("LOB_TIMEOUT", 300000);
/** /**
* Database setting <code>MAX_COMPACT_COUNT</code> * Database setting <code>MAX_COMPACT_COUNT</code>
* (default: Integer.MAX_VALUE).<br /> * (default: Integer.MAX_VALUE).<br />
......
...@@ -9,6 +9,7 @@ import java.util.ArrayList; ...@@ -9,6 +9,7 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList;
import java.util.Random; import java.util.Random;
import org.h2.api.ErrorCode; import org.h2.api.ErrorCode;
...@@ -108,6 +109,20 @@ public class Session extends SessionWithState { ...@@ -108,6 +109,20 @@ public class Session extends SessionWithState {
private final int queryCacheSize; private final int queryCacheSize;
private SmallLRUCache<String, Command> queryCache; private SmallLRUCache<String, Command> queryCache;
private long modificationMetaID = -1; private long modificationMetaID = -1;
/**
* Temporary LOBs from result sets. Those are kept for some time. The
* problem is that transactions are committed before the result is returned,
* and in some cases the next transaction is already started before the
* result is read (for example when using the server mode, when accessing
* metadata methods). We can't simply free those values up when starting the
* next transaction, because they would be removed too early.
*/
private LinkedList<TimeoutValue> temporaryResultLobs;
/**
* The temporary LOBs that need to be removed on commit.
*/
private ArrayList<Value> temporaryLobs; private ArrayList<Value> temporaryLobs;
private Transaction transaction; private Transaction transaction;
...@@ -497,14 +512,7 @@ public class Session extends SessionWithState { ...@@ -497,14 +512,7 @@ public class Session extends SessionWithState {
// (create/drop table and so on) // (create/drop table and so on)
database.commit(this); database.commit(this);
} }
if (temporaryLobs != null) { removeTemporaryLobs(true);
for (Value v : temporaryLobs) {
if (!v.isLinked()) {
v.close();
}
}
temporaryLobs.clear();
}
if (undoLog.size() > 0) { if (undoLog.size() > 0) {
// commit the rows when using MVCC // commit the rows when using MVCC
if (database.isMultiVersion()) { if (database.isMultiVersion()) {
...@@ -536,6 +544,31 @@ public class Session extends SessionWithState { ...@@ -536,6 +544,31 @@ public class Session extends SessionWithState {
endTransaction(); endTransaction();
} }
private void removeTemporaryLobs(boolean onTimeout) {
if (temporaryLobs != null) {
for (Value v : temporaryLobs) {
if (!v.isLinked()) {
v.close();
}
}
temporaryLobs.clear();
}
if (temporaryResultLobs != null && temporaryResultLobs.size() > 0) {
long keepYoungerThan = System.currentTimeMillis() -
database.getSettings().lobTimeout;
while (temporaryResultLobs.size() > 0) {
TimeoutValue tv = temporaryResultLobs.getFirst();
if (onTimeout && tv.created >= keepYoungerThan) {
break;
}
Value v = temporaryResultLobs.removeFirst().value;
if (!v.isLinked()) {
v.close();
}
}
}
}
private void checkCommitRollback() { private void checkCommitRollback() {
if (commitOrRollbackDisabled && locks.size() > 0) { if (commitOrRollbackDisabled && locks.size() > 0) {
throw DbException.get(ErrorCode.COMMIT_ROLLBACK_NOT_ALLOWED); throw DbException.get(ErrorCode.COMMIT_ROLLBACK_NOT_ALLOWED);
...@@ -545,8 +578,8 @@ public class Session extends SessionWithState { ...@@ -545,8 +578,8 @@ public class Session extends SessionWithState {
private void endTransaction() { private void endTransaction() {
if (unlinkLobMap != null && unlinkLobMap.size() > 0) { if (unlinkLobMap != null && unlinkLobMap.size() > 0) {
if (database.getMvStore() == null) { if (database.getMvStore() == null) {
// need to flush the transaction log, because we can't unlink lobs // need to flush the transaction log, because we can't unlink
// if the commit record is not written // lobs if the commit record is not written
database.flush(); database.flush();
} }
for (Value v : unlinkLobMap.values()) { for (Value v : unlinkLobMap.values()) {
...@@ -673,6 +706,7 @@ public class Session extends SessionWithState { ...@@ -673,6 +706,7 @@ public class Session extends SessionWithState {
if (!closed) { if (!closed) {
try { try {
database.checkPowerOff(); database.checkPowerOff();
removeTemporaryLobs(false);
cleanTempTables(true); cleanTempTables(true);
undoLog.clear(); undoLog.clear();
database.removeSession(this); database.removeSession(this);
...@@ -1447,11 +1481,18 @@ public class Session extends SessionWithState { ...@@ -1447,11 +1481,18 @@ public class Session extends SessionWithState {
@Override @Override
public void addTemporaryLob(Value v) { public void addTemporaryLob(Value v) {
if (v.getTableId() == LobStorageFrontend.TABLE_RESULT) {
if (temporaryResultLobs == null) {
temporaryResultLobs = new LinkedList<TimeoutValue>();
}
temporaryResultLobs.add(new TimeoutValue(v));
} else {
if (temporaryLobs == null) { if (temporaryLobs == null) {
temporaryLobs = new ArrayList<Value>(); temporaryLobs = new ArrayList<Value>();
} }
temporaryLobs.add(v); temporaryLobs.add(v);
} }
}
/** /**
* Represents a savepoint (a position in a transaction to where one can roll * Represents a savepoint (a position in a transaction to where one can roll
...@@ -1470,4 +1511,25 @@ public class Session extends SessionWithState { ...@@ -1470,4 +1511,25 @@ public class Session extends SessionWithState {
long transactionSavepoint; long transactionSavepoint;
} }
/**
* An object with a timeout.
*/
public static class TimeoutValue {
/**
* The time when this object was created.
*/
final long created = System.currentTimeMillis();
/**
* The value.
*/
final Value value;
TimeoutValue(Value v) {
this.value = v;
}
}
} }
...@@ -42,6 +42,7 @@ public class LocalResult implements ResultInterface, ResultTarget { ...@@ -42,6 +42,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
private boolean distinct; private boolean distinct;
private boolean randomAccess; private boolean randomAccess;
private boolean closed; private boolean closed;
private boolean containsLobs;
/** /**
* Construct a local result object. * Construct a local result object.
...@@ -114,12 +115,15 @@ public class LocalResult implements ResultInterface, ResultTarget { ...@@ -114,12 +115,15 @@ public class LocalResult implements ResultInterface, ResultTarget {
* (if there is any) is not copied. * (if there is any) is not copied.
* *
* @param targetSession the session of the copy * @param targetSession the session of the copy
* @return the copy * @return the copy if possible, or null if copying is not possible
*/ */
public LocalResult createShallowCopy(Session targetSession) { public LocalResult createShallowCopy(Session targetSession) {
if (external == null && (rows == null || rows.size() < rowCount)) { if (external == null && (rows == null || rows.size() < rowCount)) {
return null; return null;
} }
if (containsLobs) {
return null;
}
ResultExternal e2 = null; ResultExternal e2 = null;
if (external != null) { if (external != null) {
e2 = external.createShallowCopy(); e2 = external.createShallowCopy();
...@@ -260,6 +264,7 @@ public class LocalResult implements ResultInterface, ResultTarget { ...@@ -260,6 +264,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
Value v = values[i]; Value v = values[i];
Value v2 = v.copyToResult(); Value v2 = v.copyToResult();
if (v2 != v) { if (v2 != v) {
containsLobs = true;
session.addTemporaryLob(v2); session.addTemporaryLob(v2);
values[i] = v2; values[i] = v2;
} }
......
...@@ -268,7 +268,14 @@ public class LobStorageMap implements LobStorageInterface { ...@@ -268,7 +268,14 @@ public class LobStorageMap implements LobStorageInterface {
init(); init();
Object[] value = lobMap.get(lob.getLobId()); Object[] value = lobMap.get(lob.getLobId());
if (value == null) { if (value == null) {
throw DbException.throwInternalError("Lob not found: " + lob.getLobId()); if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT ||
lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
throw DbException.get(
ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" +
lob.getLobId() + "/" + lob.getTableId());
}
throw DbException.throwInternalError("Lob not found: " +
lob.getLobId() + "/" + lob.getTableId());
} }
byte[] streamStoreId = (byte[]) value[0]; byte[] streamStoreId = (byte[]) value[0];
return streamStore.get(streamStoreId); return streamStore.get(streamStoreId);
...@@ -348,7 +355,7 @@ public class LobStorageMap implements LobStorageInterface { ...@@ -348,7 +355,7 @@ public class LobStorageMap implements LobStorageInterface {
} }
private static void trace(String op) { private static void trace(String op) {
System.out.println(Thread.currentThread().getName() + " LOB " + op); System.out.println("[" + Thread.currentThread().getName() + "] LOB " + op);
} }
} }
...@@ -204,6 +204,7 @@ public class ValueLobDb extends Value implements Value.ValueClob, ...@@ -204,6 +204,7 @@ public class ValueLobDb extends Value implements Value.ValueClob,
@Override @Override
public boolean isLinked() { public boolean isLinked() {
return tableId != LobStorageFrontend.TABLE_ID_SESSION_VARIABLE && return tableId != LobStorageFrontend.TABLE_ID_SESSION_VARIABLE &&
tableId != LobStorageFrontend.TABLE_RESULT &&
small == null; small == null;
} }
......
...@@ -59,6 +59,7 @@ public class TestLob extends TestBase { ...@@ -59,6 +59,7 @@ public class TestLob extends TestBase {
@Override @Override
public void test() throws Exception { public void test() throws Exception {
testRemovedAfterTimeout();
testConcurrentRemoveRead(); testConcurrentRemoveRead();
testCloseLobTwice(); testCloseLobTwice();
testCleaningUpLobsOnRollback(); testCleaningUpLobsOnRollback();
...@@ -112,6 +113,42 @@ public class TestLob extends TestBase { ...@@ -112,6 +113,42 @@ public class TestLob extends TestBase {
FileUtils.deleteRecursive(TEMP_DIR, true); FileUtils.deleteRecursive(TEMP_DIR, true);
} }
private void testRemovedAfterTimeout() throws Exception {
deleteDb("lob");
final String url = getURL("lob;lob_timeout=50", true);
Connection conn = getConnection(url);
Statement stat = conn.createStatement();
stat.execute("create table test(id int primary key, data clob)");
PreparedStatement prep = conn.prepareStatement("insert into test values(?, ?)");
prep.setInt(1, 1);
prep.setString(2, "aaa" + new String(new char[1024 * 16]).replace((char) 0, 'x'));
prep.execute();
prep.setInt(1, 2);
prep.setString(2, "bbb" + new String(new char[1024 * 16]).replace((char) 0, 'x'));
prep.execute();
ResultSet rs = stat.executeQuery("select * from test order by id");
rs.next();
Clob c1 = rs.getClob(2);
assertEquals("aaa", c1.getSubString(1, 3));
rs.next();
assertEquals("aaa", c1.getSubString(1, 3));
rs.close();
assertEquals("aaa", c1.getSubString(1, 3));
stat.execute("delete from test");
c1.getSubString(1, 3);
// wait until it times out
Thread.sleep(100);
// start a new transaction, to be sure
stat.execute("delete from test");
try {
c1.getSubString(1, 3);
fail();
} catch (SQLException e) {
// expected
}
conn.close();
}
private void testConcurrentRemoveRead() throws Exception { private void testConcurrentRemoveRead() throws Exception {
deleteDb("lob"); deleteDb("lob");
final String url = getURL("lob", true); final String url = getURL("lob", true);
...@@ -1255,7 +1292,6 @@ public class TestLob extends TestBase { ...@@ -1255,7 +1292,6 @@ public class TestLob extends TestBase {
"CREATE TABLE IF NOT EXISTS p( id int primary key, rawbyte BLOB ); "); "CREATE TABLE IF NOT EXISTS p( id int primary key, rawbyte BLOB ); ");
prep.execute(); prep.execute();
prep.close(); prep.close();
prep = conn.prepareStatement("INSERT INTO p(id) VALUES(?);"); prep = conn.prepareStatement("INSERT INTO p(id) VALUES(?);");
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
prep.setInt(1, i); prep.setInt(1, i);
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论