提交 a580bf1a authored 作者: Thomas Mueller's avatar Thomas Mueller

Mainly MVStore improvements

上级 464c791c
......@@ -18,7 +18,11 @@ Change Log
<h1>Change Log</h1>
<h2>Next Version (unreleased)</h2>
<ul><li>The license has changed to MPL 2.0 + EPL 1.0.
<ul><li>Recursive queries with many result rows (more than the setting "max_memory_rows")
did not work correctly.
</li><li>The license has changed to MPL 2.0 + EPL 1.0.
</li><li>MVStore: temporary tables from result sets could survive re-opening a database,
which could result in a ClassCastException.
</li><li>MVStore: unique indexes that were created later on did not work correctly
if there were over 5000 rows in the table.
</li><li>MVStore: creating secondary indexes on large tables
......
......@@ -603,7 +603,6 @@ public class Select extends Query {
}
if (randomAccessResult) {
result = createLocalResult(result);
result.setRandomAccess();
}
if (isGroupQuery && !isGroupSortedQuery) {
result = createLocalResult(result);
......
......@@ -18,6 +18,7 @@ import java.util.StringTokenizer;
import org.h2.api.DatabaseEventListener;
import org.h2.api.ErrorCode;
import org.h2.api.JavaObjectSerializer;
import org.h2.command.CommandInterface;
import org.h2.command.ddl.CreateTableData;
import org.h2.command.dml.SetTypes;
import org.h2.constraint.Constraint;
......@@ -725,7 +726,7 @@ public class Database implements DataHandler {
}
if (mvStore != null) {
mvStore.initTransactions();
mvStore.removeTemporaryMaps();
mvStore.removeTemporaryMaps(objectIds);
}
recompileInvalidViews(systemSession);
starting = false;
......@@ -1325,8 +1326,12 @@ public class Database implements DataHandler {
}
reconnectModified(false);
if (mvStore != null) {
if (!readOnly && compactMode != 0) {
mvStore.compactFile(dbSettings.maxCompactTime);
if (!readOnly) {
if (compactMode == CommandInterface.SHUTDOWN_COMPACT) {
mvStore.compactFile(dbSettings.maxCompactTime);
} else if (compactMode == CommandInterface.SHUTDOWN_DEFRAG) {
mvStore.compactFile(Long.MAX_VALUE);
}
}
mvStore.close(dbSettings.maxCompactTime);
}
......
......@@ -27,7 +27,7 @@ import org.h2.message.TraceSystem;
import org.h2.mvstore.db.MVTable;
import org.h2.mvstore.db.TransactionStore.Change;
import org.h2.mvstore.db.TransactionStore.Transaction;
import org.h2.result.ResultInterface;
import org.h2.result.LocalResult;
import org.h2.result.Row;
import org.h2.schema.Schema;
import org.h2.store.DataHandler;
......@@ -99,7 +99,7 @@ public class Session extends SessionWithState {
private long transactionStart;
private long currentCommandStart;
private HashMap<String, Value> variables;
private HashSet<ResultInterface> temporaryResults;
private HashSet<LocalResult> temporaryResults;
private int queryTimeout;
private boolean commitOrRollbackDisabled;
private Table waitForLock;
......@@ -1268,7 +1268,7 @@ public class Session extends SessionWithState {
*
* @param result the temporary result set
*/
public void addTemporaryResult(ResultInterface result) {
public void addTemporaryResult(LocalResult result) {
if (!result.needToClose()) {
return;
}
......@@ -1283,7 +1283,7 @@ public class Session extends SessionWithState {
private void closeTemporaryResults() {
if (temporaryResults != null) {
for (ResultInterface result : temporaryResults) {
for (LocalResult result : temporaryResults) {
result.close();
}
temporaryResults = null;
......
......@@ -21,6 +21,7 @@ public class Cursor<K, V> implements Iterator<K> {
private CursorPos pos;
private K current, last;
private V currentValue, lastValue;
private Page lastPage;
private final Page root;
private boolean initialized;
......@@ -46,6 +47,7 @@ public class Cursor<K, V> implements Iterator<K> {
K c = current;
last = current;
lastValue = currentValue;
lastPage = pos == null ? null : pos.page;
fetchNext();
return c;
}
......@@ -67,6 +69,10 @@ public class Cursor<K, V> implements Iterator<K> {
public V getValue() {
return lastValue;
}
Page getPage() {
return lastPage;
}
/**
* Skip over that many entries. This method is relatively fast (for this map
......@@ -147,5 +153,5 @@ public class Cursor<K, V> implements Iterator<K> {
}
current = null;
}
}
......@@ -772,6 +772,72 @@ public class MVMap<K, V> extends AbstractMap<K, V>
public Iterator<K> keyIterator(K from) {
return new Cursor<K, V>(this, root, from);
}
/**
* Re-write any pages that belong to one of the chunks in the given set.
*
* @param set the set of chunk ids
*/
public void rewrite(Set<Integer> set) {
rewrite(root, set);
}
public int rewrite(Page p, Set<Integer> set) {
; // TODO write more tests
if (p.isLeaf()) {
long pos = p.getPos();
if (pos == 0) {
return 0;
}
int chunkId = DataUtils.getPageChunkId(pos);
if (!set.contains(chunkId)) {
return 0;
}
@SuppressWarnings("unchecked")
K key = (K) p.getKey(0);
@SuppressWarnings("unchecked")
V value = (V) p.getValue(0);
put(key, value);
return 1;
}
int writtenPageCount = 0;
for (int i = 0; i < p.getChildPageCount(); i++) {
long pos = p.getChildPagePos(i);
if (pos == 0) {
continue;
}
if (DataUtils.getPageType(pos) == DataUtils.PAGE_TYPE_LEAF) {
int chunkId = DataUtils.getPageChunkId(pos);
if (!set.contains(chunkId)) {
continue;
}
}
writtenPageCount += rewrite(p.getChildPage(i), set);
}
if (writtenPageCount == 0) {
long pos = p.getPos();
if (pos != 0) {
int chunkId = DataUtils.getPageChunkId(pos);
if (set.contains(chunkId)) {
// an inner node page that is in one of the chunks,
// but only points to chunks that are not in the set:
// if no child was changed, we need to do that now
Page p2 = p;
while (!p2.isLeaf()) {
p2 = p2.getChildPage(0);
}
@SuppressWarnings("unchecked")
K key = (K) p2.getKey(0);
@SuppressWarnings("unchecked")
V value = (V) p2.getValue(0);
put(key, value);
writtenPageCount++;
}
}
}
return writtenPageCount;
}
/**
* Get a cursor to iterate over a number of keys and values.
......
......@@ -203,12 +203,12 @@ public class MVStore {
private long lastStoredVersion;
/**
* The estimated number of average-sized unsaved pages. This number may not
* be completely accurate, because it may be changed concurrently, and
* because temporary pages are counted.
* The estimated memory used by unsaved pages. This number is not accurate,
* also because it may be changed concurrently, and because temporary pages
* are counted.
*/
private int unsavedPageCount;
private int autoCommitPageCount;
private int unsavedMemory;
private int autoCommitMemory;
private boolean saveNeeded;
/**
......@@ -287,9 +287,7 @@ public class MVStore {
o = config.get("autoCommitBufferSize");
int kb = o == null ? 512 : (Integer) o;
// 19 KB memory is about 1 KB storage
int autoCommitBufferSize = kb * 1024 * 19;
int div = pageSplitSize;
autoCommitPageCount = autoCommitBufferSize / (div == 0 ? 1 : div);
autoCommitMemory = kb * 1024 * 19;
char[] encryptionKey = (char[]) config.get("encryptionKey");
try {
fileStore.open(fileName, readOnly, encryptionKey);
......@@ -916,7 +914,7 @@ public class MVStore {
}
private long storeNow() {
int currentUnsavedPageCount = unsavedPageCount;
int currentUnsavedPageCount = unsavedMemory;
long storeVersion = currentStoreVersion;
long version = ++currentVersion;
setWriteVersion(version);
......@@ -1099,18 +1097,6 @@ public class MVStore {
chunkId++;
}
}
// }
// while (chunkId <= lastChunk.id) {
// if (chunks.get(chunkId) == null) {
// // one of the chunks in between
// // was removed
// needHeader = true;
// break;
// }
// chunkId++;
// }
// }
}
}
......@@ -1133,7 +1119,7 @@ public class MVStore {
// some pages might have been changed in the meantime (in the newest
// version)
unsavedPageCount = Math.max(0, unsavedPageCount
unsavedMemory = Math.max(0, unsavedMemory
- currentUnsavedPageCount);
metaChanged = false;
......@@ -1321,7 +1307,40 @@ public class MVStore {
ByteBuffer buff = fileStore.readFully(p, Chunk.MAX_HEADER_LENGTH);
return Chunk.readChunkHeader(buff, p);
}
/**
* Compact the store by moving all live pages to new chunks.
*
* @return if anything was written
*/
public synchronized boolean compactRewriteFully() {
checkOpen();
if (lastChunk == null) {
// nothing to do
return false;
}
; // TODO write tests
for (MVMap<?, ?> m : maps.values()) {
@SuppressWarnings("unchecked")
MVMap<Object, Object> map = (MVMap<Object, Object>) m;
Cursor<Object, Object> cursor = map.cursor(null);
Page lastPage = null;
while (cursor.hasNext()) {
cursor.next();
Page p = cursor.getPage();
if (p == lastPage) {
continue;
}
Object k = p.getKey(0);
Object v = p.getValue(0);
map.put(k, v);
lastPage = p;
}
}
commitAndSave();
return true;
}
/**
* Compact the store by moving all chunks next to each other, if there is
* free space between chunks. This might temporarily double the file size.
......@@ -1447,6 +1466,132 @@ public class MVStore {
public void sync() {
fileStore.sync();
}
/**
* Try to increase the fill rate by re-writing partially full chunks. Chunks
* with a low number of live items are re-written.
* <p>
* If the current fill rate is higher than the target fill rate, nothing is
* done. If not at least a minimum amount of space can be saved, nothing is
* done.
* <p>
* Please note this method will not necessarily reduce the file size, as
* empty chunks are not overwritten.
* <p>
* Only data of open maps can be moved. For maps that are not open, the old
* chunk is still referenced. Therefore, it is recommended to open all maps
* before calling this method.
*
* @param targetFillRate the minimum percentage of live entries
* @param saving the amount of saved space
* @return if a chunk was re-written
*/
public synchronized boolean compact(int targetFillRate, int saving) {
checkOpen();
if (lastChunk == null) {
// nothing to do
return false;
}
// calculate the fill rate
long maxLengthSum = 0;
long maxLengthLiveSum = 0;
for (Chunk c : chunks.values()) {
maxLengthSum += c.maxLen;
maxLengthLiveSum += c.maxLenLive;
}
// the fill rate of all chunks combined
if (maxLengthSum <= 0) {
// avoid division by 0
maxLengthSum = 1;
}
int fillRate = (int) (100 * maxLengthLiveSum / maxLengthSum);
if (fillRate >= targetFillRate) {
return false;
}
long time = getTime();
// the 'old' list contains the chunks we want to free up
ArrayList<Chunk> old = New.arrayList();
Chunk last = chunks.get(lastChunk.id);
for (Chunk c : chunks.values()) {
if (canOverwriteChunk(c, time)) {
long age = last.version - c.version + 1;
c.collectPriority = (int) (c.getFillRate() / age);
old.add(c);
}
}
if (old.size() == 0) {
return false;
}
// sort the list, so the first entry should be collected first
Collections.sort(old, new Comparator<Chunk>() {
@Override
public int compare(Chunk o1, Chunk o2) {
int comp = new Integer(o1.collectPriority).
compareTo(o2.collectPriority);
if (comp == 0) {
comp = new Long(o1.maxLenLive).
compareTo(o2.maxLenLive);
}
return comp;
}
});
// find out up to were in the old list we need to move
long saved = 0;
int chunkCount = 0;
Chunk move = null;
for (Chunk c : old) {
long size = c.maxLen - c.maxLenLive;
if (move != null) {
if (saved > saving) {
break;
}
}
saved += size;
chunkCount++;
move = c;
}
if (chunkCount <= 1) {
return false;
}
// remove the chunks we want to keep from this list
boolean remove = false;
for (Iterator<Chunk> it = old.iterator(); it.hasNext();) {
Chunk c = it.next();
if (move == c) {
remove = true;
} else if (remove) {
it.remove();
}
}
HashSet<Integer> set = New.hashSet();
for (Chunk c : old) {
set.add(c.id);
}
for (MVMap<?, ?> m : maps.values()) {
@SuppressWarnings("unchecked")
MVMap<Object, Object> map = (MVMap<Object, Object>) m;
; // TODO write more tests
map.rewrite(set);
}
commitAndSave();
boolean again = false;
for (Chunk c : old) {
if (c.maxLenLive > 0) {
// not cleared - that means bookkeeping of live pages
// is broken; copyLive will fix this
again = true;
copyLive(c);
}
}
if (again) {
commitAndSave();
}
return true;
}
/**
* Try to increase the fill rate by re-writing partially full chunks. Chunks
......@@ -1468,7 +1613,7 @@ public class MVStore {
* which is also the size of the new chunk
* @return if a chunk was re-written
*/
public synchronized boolean compact(int targetFillRate, int minSaving) {
public synchronized boolean compactOld(int targetFillRate, int minSaving) {
checkOpen();
if (lastChunk == null) {
// nothing to do
......@@ -1689,8 +1834,7 @@ public class MVStore {
// the value could be smaller than 0 because
// in some cases a page is allocated,
// but never stored
int count = 1 + memory / pageSplitSize;
unsavedPageCount = Math.max(0, unsavedPageCount - count);
unsavedMemory = Math.max(0, unsavedMemory - memory);
return;
}
......@@ -1891,10 +2035,9 @@ public class MVStore {
* @param memory the memory usage of the page
*/
void registerUnsavedPage(int memory) {
int count = 1 + memory / pageSplitSize;
unsavedPageCount += count;
int newValue = unsavedPageCount;
if (newValue > autoCommitPageCount && autoCommitPageCount > 0) {
unsavedMemory += memory;
int newValue = unsavedMemory;
if (newValue > autoCommitMemory && autoCommitMemory > 0) {
saveNeeded = true;
}
}
......@@ -1905,7 +2048,10 @@ public class MVStore {
void beforeWrite() {
if (saveNeeded) {
saveNeeded = false;
commitAndSave();
// check again, because it could have been written by now
if (unsavedMemory > autoCommitMemory && autoCommitMemory > 0) {
commitAndSave();
}
}
}
......@@ -2162,7 +2308,7 @@ public class MVStore {
* Commit and save all changes, if there are any.
*/
void commitInBackground() {
if (unsavedPageCount == 0 || closed) {
if (unsavedMemory == 0 || closed) {
return;
}
......@@ -2266,26 +2412,25 @@ public class MVStore {
}
/**
* Get the maximum number of unsaved pages. If this number is exceeded,
* unsaved changes are stored to disk.
*
* @return the number of maximum unsaved pages
* Get the maximum memory (in bytes) used for unsaved pages. If this number
* is exceeded, unsaved changes are stored to disk.
*
* @return the memory in bytes
*/
public int getAutoCommitPageCount() {
return autoCommitPageCount;
public int getAutoCommitMemory() {
return autoCommitMemory;
}
/**
* Get the estimated number of unsaved pages. If the value exceeds the
* auto-commit page count, the changes are committed.
* Get the estimated memory (in bytes) of unsaved data. If the value exceeds the
* auto-commit memory, the changes are committed.
* <p>
* The returned value may not be completely accurate, but can be used to
* estimate the memory usage for unsaved data.
* The returned value is an estimation only.
*
* @return the number of unsaved pages
* @return the memory in bytes
*/
public int getUnsavedPageCount() {
return unsavedPageCount;
public int getUnsavedMemory() {
return unsavedMemory;
}
/**
......@@ -2399,15 +2544,15 @@ public class MVStore {
}
/**
* Set the size of the write buffer, in KB (for file-based stores).
* Unless auto-commit is disabled, changes are automatically saved if
* there are more than this amount of changes.
* Set the size of the write buffer, in KB disk space (for file-based
* stores). Unless auto-commit is disabled, changes are automatically
* saved if there are more than this amount of changes.
* <p>
* The default is 512 KB.
* <p>
* When the value is set to 0 or lower, data is not automatically
* stored.
*
*
* @param kb the write buffer size, in kilobytes
* @return this
*/
......
......@@ -221,6 +221,10 @@ public class Page {
Page p = childrenPages[index];
return p != null ? p : map.readPage(children[index]);
}
public long getChildPagePos(int index) {
return children[index];
}
/**
* Get the child page at the given index, if it is live.
......
......@@ -14,11 +14,9 @@ import java.io.OutputStream;
import java.io.Reader;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import org.h2.api.ErrorCode;
......
......@@ -320,5 +320,36 @@ public abstract class FilePath {
public FilePath unwrap() {
return this;
}
/**
* The options to open a file.
*/
public enum FileOpenOption {
/**
* Append at the end of the file.
*/
APPEND,
/**
* Written data is not buffered.
*/
DSYNC,
/**
* Open the file for read access.
*/
READ,
/**
* Written data and metadata is not buffered.
*/
SYNC,
/**
* Open the file for write access.
*/
WRITE,
}
}
......@@ -6,9 +6,7 @@
*/
package org.h2.table;
import java.sql.Date;
import java.sql.ResultSetMetaData;
import java.sql.Timestamp;
import org.h2.api.ErrorCode;
import org.h2.command.Parser;
......
......@@ -26,7 +26,6 @@ import org.h2.index.IndexType;
import org.h2.index.ViewIndex;
import org.h2.message.DbException;
import org.h2.result.LocalResult;
import org.h2.result.ResultInterface;
import org.h2.result.Row;
import org.h2.result.SortOrder;
import org.h2.schema.Schema;
......@@ -46,41 +45,6 @@ public class TableView extends Table {
private static final long ROW_COUNT_APPROXIMATION = 100;
private static final class CacheKey {
private final int[] masks;
private final Session session;
public CacheKey(int[] masks, Session session) {
this.masks = masks;
this.session = session;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(masks);
result = prime * result + session.hashCode();
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
CacheKey other = (CacheKey) obj;
if (!Arrays.equals(masks, other.masks))
return false;
if (session != other.session)
return false;
return true;
}
}
private String querySQL;
private ArrayList<Table> tables;
private String[] columnNames;
......@@ -572,7 +536,7 @@ public class TableView extends Table {
this.recursiveResult = value;
}
public ResultInterface getRecursiveResult() {
public LocalResult getRecursiveResult() {
return recursiveResult;
}
......@@ -595,5 +559,49 @@ public class TableView extends Table {
}
}
}
/**
* The key of the index cache for views.
*/
private static final class CacheKey {
private final int[] masks;
private final Session session;
public CacheKey(int[] masks, Session session) {
this.masks = masks;
this.session = session;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(masks);
result = prime * result + session.hashCode();
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
CacheKey other = (CacheKey) obj;
if (session != other.session) {
return false;
}
if (!Arrays.equals(masks, other.masks)) {
return false;
}
return true;
}
}
}
......@@ -4,6 +4,11 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
/*
* Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
* and the EPL 1.0 (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.tools;
//## AWT ##
......
......@@ -362,10 +362,10 @@ java org.h2.test.TestAll timer
*/
String cacheType;
private Server server;
AbbaLockingDetector abbaLockingDetector;
private Server server;
/**
* Run all tests.
*
......@@ -385,7 +385,15 @@ java org.h2.test.TestAll timer
// use lower values, to better test those cases,
// and to speed up the tests (for delays)
System.setProperty("h2.maxMemoryRows", "128");
; // TEST
// System.setProperty("h2.maxMemoryRows", "2");
System.setProperty("h2.maxMemoryRows", "100");
// System.setProperty("h2.maxMemoryRows", "1000");
// System.setProperty("h2.maxMemoryRows", "2");
System.setProperty("h2.check2", "true");
System.setProperty("h2.delayWrongPasswordMin", "0");
System.setProperty("h2.delayWrongPasswordMax", "0");
......@@ -610,7 +618,6 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
new TestAlterSchemaRename().runTest(this);
new TestAutoRecompile().runTest(this);
new TestBitField().runTest(this);
new TestBnf().runTest(this);
new TestBackup().runTest(this);
new TestBigDb().runTest(this);
new TestBigResult().runTest(this);
......@@ -752,6 +759,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
// unit
new TestAutoReconnect().runTest(this);
new TestBnf().runTest(this);
new TestCache().runTest(this);
new TestClearReferences().runTest(this);
new TestCollation().runTest(this);
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论