Skip to content

H
h2database
提交 09769f17 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVStore: larger stores (multiple GB) are now much faster.

上级 f8db2f80
隐藏空白字符变更
内嵌 并排
正在显示 包含 261 行增加 和 426 行删除
h2/src/docsrc/html/changelog.html
浏览文件 @ 09769f17
......@@ -18,7 +18,8 @@ Change Log
<h1>Change Log</h1>
<h2>Next Version (unreleased)</h2>
<ul><li>When using local temporary tables and not dropping them manually before closing the session,
<ul><li>MVStore: larger stores (multiple GB) are now much faster.
</li><li>When using local temporary tables and not dropping them manually before closing the session,
and then killing the process could result in a database that couldn't be opened (except when using
the recover tool).
</li><li>Support TRUNC(timestamp) for improved Oracle compatiblity.
......
h2/src/main/org/h2/mvstore/ChangeCursor.java deleted 100644 → 0
浏览文件 @ f8db2f80
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.mvstore;
import java.util.Iterator;
/**
* A cursor to iterate over all keys in new pages.
*
* @param <K> the key type
* @param <V> the value type
*/
public class ChangeCursor<K, V> implements Iterator<K> {
private final MVMap<K, V> map;
private final Page root1, root2;
/**
* The state of this cursor.
* 0: not initialized
* 1: reading from root1
* 2: reading from root2
* 3: closed
*/
private int state;
private CursorPos pos1, pos2;
private K current;
ChangeCursor(MVMap<K, V> map, Page root1, Page root2) {
this.map = map;
this.root1 = root1;
this.root2 = root2;
}
public K next() {
K c = current;
fetchNext();
return c;
}
public boolean hasNext() {
if (state == 0) {
pos1 = new CursorPos(root1, 0, null);
pos1 = min(pos1);
state = 1;
fetchNext();
}
return current != null;
}
public void remove() {
throw DataUtils.newUnsupportedOperationException(
"Removing is not supported");
}
private void fetchNext() {
while (fetchNextKey()) {
if (pos1 == null || pos2 == null) {
break;
}
@SuppressWarnings("unchecked")
V v1 = (V) map.binarySearch(root1, current);
@SuppressWarnings("unchecked")
V v2 = (V) map.binarySearch(root2, current);
if (!v1.equals(v2)) {
break;
}
}
}
private boolean fetchNextKey() {
while (true) {
if (state == 3) {
return false;
}
if (state == 1) {
// read from root1
pos1 = fetchNext(pos1);
if (pos1 == null) {
// reached the end of pos1
state = 2;
pos2 = null;
continue;
}
pos2 = find(root2, current);
if (pos2 == null) {
// not found in root2
return true;
}
if (!pos1.page.equals(pos2.page)) {
// the page is different,
// so the entry has possibly changed
return true;
}
while (true) {
pos1 = pos1.parent;
if (pos1 == null) {
// reached end of pos1
state = 2;
pos2 = null;
break;
}
pos2 = pos2.parent;
if (pos2 == null || !pos1.page.equals(pos2.page)) {
if (pos1.index + 1 < map.getChildPageCount(pos1.page)) {
pos1 = new CursorPos(pos1.page.getChildPage(++pos1.index), 0, pos1);
pos1 = min(pos1);
break;
}
}
}
}
if (state == 2) {
if (pos2 == null) {
// init reading from root2
pos2 = new CursorPos(root2, 0, null);
pos2 = min(pos2);
}
// read from root2
pos2 = fetchNext(pos2);
if (pos2 == null) {
// reached the end of pos2
state = 3;
current = null;
continue;
}
pos1 = find(root1, current);
if (pos1 != null) {
// found a corresponding record
// so it was not deleted
// but now we may need to skip pages
if (!pos1.page.equals(pos2.page)) {
// the page is different
pos1 = null;
continue;
}
while (true) {
pos2 = pos2.parent;
if (pos2 == null) {
// reached end of pos1
state = 3;
current = null;
pos1 = null;
break;
}
pos1 = pos1.parent;
if (pos1 == null || !pos2.page.equals(pos1.page)) {
if (pos2.index + 1 < map.getChildPageCount(pos2.page)) {
pos2 = new CursorPos(pos2.page.getChildPage(++pos2.index), 0, pos2);
pos2 = min(pos2);
break;
}
}
}
pos1 = null;
continue;
}
// found no corresponding record
// so it was deleted
return true;
}
}
}
private CursorPos find(Page p, K key) {
// TODO combine with RangeCursor.min
// possibly move to MVMap
CursorPos pos = null;
while (true) {
if (p.isLeaf()) {
int x = key == null ? 0 : p.binarySearch(key);
if (x < 0) {
return null;
}
return new CursorPos(p, x, pos);
}
int x = key == null ? -1 : p.binarySearch(key);
if (x < 0) {
x = -x - 1;
} else {
x++;
}
pos = new CursorPos(p, x, pos);
p = p.getChildPage(x);
}
}
@SuppressWarnings("unchecked")
private CursorPos fetchNext(CursorPos p) {
while (p != null) {
if (p.index < p.page.getKeyCount()) {
current = (K) p.page.getKey(p.index++);
return p;
}
p = p.parent;
if (p == null) {
break;
}
if (p.index + 1 < map.getChildPageCount(p.page)) {
p = new CursorPos(p.page.getChildPage(++p.index), 0, p);
p = min(p);
}
}
current = null;
return p;
}
private static CursorPos min(CursorPos p) {
while (true) {
if (p.page.isLeaf()) {
return p;
}
Page c = p.page.getChildPage(0);
p = new CursorPos(c, 0, p);
}
}
}
h2/src/main/org/h2/mvstore/Chunk.java
浏览文件 @ 09769f17
......@@ -42,9 +42,14 @@ public class Chunk {
int length;
/**
* The number of pages.
* The total number of pages in this chunk.
*/
int pageCount;
/**
* The number of pages still alive.
*/
int pageCountLive;
/**
* The sum of the max length of all pages.
......@@ -101,6 +106,7 @@ public class Chunk {
Chunk c = new Chunk(chunkId);
c.length = length;
c.pageCount = pageCount;
c.pageCountLive = pageCount;
c.start = start;
c.metaRootPos = metaRootPos;
c.maxLength = maxLength;
......@@ -136,6 +142,7 @@ public class Chunk {
c.start = Long.parseLong(map.get("start"));
c.length = Integer.parseInt(map.get("length"));
c.pageCount = Integer.parseInt(map.get("pageCount"));
c.pageCountLive = Integer.parseInt(map.get("pageCountLive"));
c.maxLength = Long.parseLong(map.get("maxLength"));
c.maxLengthLive = Long.parseLong(map.get("maxLengthLive"));
c.metaRootPos = Long.parseLong(map.get("metaRoot"));
......@@ -169,6 +176,7 @@ public class Chunk {
"maxLengthLive:" + maxLengthLive + "," +
"metaRoot:" + metaRootPos + "," +
"pageCount:" + pageCount + "," +
"pageCountLive:" + pageCountLive + "," +
"start:" + start + "," +
"time:" + time + "," +
"version:" + version;
......
h2/src/main/org/h2/mvstore/FreeSpaceList.java
浏览文件 @ 09769f17
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.mvstore;
import java.util.ArrayList;
import java.util.List;
/**
* A list that maintains ranges of free space (in pages) in a file.
*/
public class FreeSpaceList {
/** the first 2 pages are occupied by the file header. */
/**
* The first 2 pages are occupied by the file header.
*/
private static final int FIRST_FREE_PAGE = 2;
/** I use max_value/2 to avoid overflow errors during arithmetic. */
private static final int MAX_NO_PAGES = Integer.MAX_VALUE / 2;
/**
* The maximum number of pages. Smaller than than MAX_VALUE to avoid
* overflow errors during arithmetic operations.
*/
private static final int MAX_PAGE_COUNT = Integer.MAX_VALUE / 2;
private List<PageRange> freeSpaceList = new ArrayList<PageRange>();
public FreeSpaceList() {
freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_NO_PAGES));
clear();
}
private static final class PageRange {
/** the start point, in pages */
public int start;
/** the length, in pages */
public int length;
public PageRange(int start, int length) {
this.start = start;
this.length = length;
}
@Override
public String toString() {
return "start:" + start + " length:" + length;
}
/**
* Reset the list.
*/
public synchronized void clear() {
freeSpaceList.clear();
freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_PAGE_COUNT));
}
/**
* @return position in pages
* Allocate a number of pages.
*
* @param length the number of bytes to allocate
* @return the position in pages
*/
public synchronized int allocatePages(long length) {
int required = (int) (length / MVStore.BLOCK_SIZE) + 1;
......@@ -43,9 +52,15 @@ public class FreeSpaceList {
return pr.start;
}
}
throw DataUtils.newIllegalStateException("could not find a free page to allocate");
throw DataUtils.newIllegalStateException(
"Could not find a free page to allocate");
}
/**
* Mark a chunk as used.
*
* @param c the chunk
*/
public synchronized void markUsed(Chunk c) {
int chunkStart = (int) (c.start / MVStore.BLOCK_SIZE);
int required = (int) ((c.start + c.length) / MVStore.BLOCK_SIZE) + 2 - chunkStart;
......@@ -59,11 +74,12 @@ public class FreeSpaceList {
i++;
}
if (found == null) {
throw DataUtils.newIllegalStateException("cannot find spot to mark chunk as used in free list, "
+ c.toString());
throw DataUtils.newIllegalStateException(
"Cannot find spot to mark chunk as used in free list: {0}", c);
}
if (chunkStart + required > found.start + found.length) {
throw DataUtils.newIllegalStateException("chunk runs over edge of free space, " + c.toString());
throw DataUtils.newIllegalStateException(
"Chunk runs over edge of free space: {0}", c);
}
if (found.start == chunkStart) {
// if the used-chunk is at the beginning of a free-space-range
......@@ -92,6 +108,11 @@ public class FreeSpaceList {
}
}
/**
* Mark the chunk as free.
*
* @param c the chunk
*/
public synchronized void markFree(Chunk c) {
int chunkStart = (int) (c.start / MVStore.BLOCK_SIZE);
int required = (c.length / MVStore.BLOCK_SIZE) + 1;
......@@ -105,15 +126,15 @@ public class FreeSpaceList {
i++;
}
if (found == null) {
throw DataUtils.newIllegalStateException("cannot find spot to mark chunk as unused in free list, "
+ c.toString());
throw DataUtils.newIllegalStateException(
"Cannot find spot to mark chunk as unused in free list: {0}", c);
}
if (chunkStart + required + 1 == found.start) {
// if the used-chunk is adjacent to the beginning of a
// free-space-range
found.start = chunkStart;
found.length += required;
// compactify-free-list: merge the previous entry into this one if
// compact: merge the previous entry into this one if
// they are now adjacent
if (i > 0) {
PageRange previous = freeSpaceList.get(i - 1);
......@@ -129,7 +150,7 @@ public class FreeSpaceList {
PageRange previous = freeSpaceList.get(i - 1);
if (previous.start + previous.length + 1 == chunkStart) {
previous.length += required;
// compactify-free-list: merge the next entry into this one if
// compact: merge the next entry into this one if
// they are now adjacent
if (previous.start + previous.length + 1 == found.start) {
previous.length += found.length;
......@@ -144,23 +165,44 @@ public class FreeSpaceList {
freeSpaceList.add(i, newRange);
}
public synchronized void clear() {
freeSpaceList.clear();
freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_NO_PAGES));
}
/** debug method */
public void dumpFreeList() {
StringBuilder buf = new StringBuilder("free list : ");
public String toString() {
StringBuilder buff = new StringBuilder();
boolean first = true;
for (PageRange pr : freeSpaceList) {
for (PageRange r : freeSpaceList) {
if (first) {
first = false;
} else {
buf.append(", ");
buff.append(", ");
}
buf.append(pr.start + "-" + (pr.start + pr.length - 1));
buff.append(r.start + "-" + (r.start + r.length - 1));
}
System.out.println(buf.toString());
return buff.toString();
}
/**
* A range of free pages.
*/
private static final class PageRange {
/**
* The starting point, in pages.
*/
public int start;
/**
* The length, in pages.
*/
public int length;
public PageRange(int start, int length) {
this.start = start;
this.length = length;
}
@Override
public String toString() {
return "start:" + start + " length:" + length;
}
}
}
h2/src/main/org/h2/mvstore/MVMap.java
浏览文件 @ 09769f17
......@@ -773,18 +773,6 @@ public class MVMap<K, V> extends AbstractMap<K, V>
return new Cursor<K>(this, root, from);
}
/**
* Iterate over all keys in changed pages.
*
* @param version the old version
* @return the iterator
*/
public Iterator<K> changeIterator(long version) {
checkOpen();
MVMap<K, V> old = openVersion(version);
return new ChangeCursor<K, V>(this, root, old.root);
}
public Set<Map.Entry<K, V>> entrySet() {
HashMap<K, V> map = new HashMap<K, V>();
for (K k : keySet()) {
......
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ 09769f17
......@@ -44,8 +44,6 @@ H:3,...
TODO:
- rolling docs review: at convert "Features" to top-level (linked) entries
- mvcc with multiple transactions
- test LZ4
- additional test async write / read algorithm for speed and errors
- move setters to the builder, except for setRetainVersion, setReuseSpace,
and settings that are persistent (setStoreVersion)
......@@ -96,7 +94,9 @@ TODO:
-- use a transaction log where only the deltas are stored
- serialization for lists, sets, sets, sorted sets, maps, sorted maps
- maybe rename 'rollback' to 'revert'
- support other compression algorithms (deflate,...)
- support other compression algorithms (deflate, LZ4,...)
- only retain the last version, unless explicitly set (setRetainVersion)
- unit test for the FreeSpaceList; maybe find a simpler implementation
*/
......@@ -155,12 +155,11 @@ public class MVStore {
/**
* The map of temporarily freed entries in the chunks. The key is the
* unsaved version, the value is the map of chunks. The maps of chunks
* contains the number of freed entries per chunk.
* Access is synchronized.
* The map of temporarily removed pages. The key is the unsaved version, the
* value is the map of chunks. The maps of chunks contains the number of
* freed entries per chunk. Access is synchronized.
*/
private final HashMap<Long, HashMap<Integer, Chunk>> freedChunks = New.hashMap();
private final HashMap<Long, HashMap<Integer, Chunk>> freedPages = New.hashMap();
private MVMapConcurrent<String, String> meta;
......@@ -227,8 +226,8 @@ public class MVStore {
MVStore(HashMap<String, Object> config) {
String f = (String) config.get("fileName");
if (f != null && !f.startsWith("nio:")) {
// NIO is used by default
if (f != null && f.indexOf(':') < 0) {
// NIO is used, unless a different file system is specified
// the following line is to ensure the NIO file system is compiled
FilePathNio.class.getName();
f = "nio:" + f;
......@@ -551,23 +550,32 @@ public class MVStore {
lastChunkId = header.id;
chunks.put(header.id, header);
meta.setRootPos(header.metaRootPos, -1);
Iterator<String> it = meta.keyIterator("chunk.");
while (it.hasNext()) {
String s = it.next();
if (!s.startsWith("chunk.")) {
break;
// load chunks in reverse order, because data about previous chunks
// might only be available in later chunks
// if this is a performance problem when there are many
// chunks, the id of the previous / next chunk might need to
// be maintained
for (int id = lastChunkId; id >= 0; id--) {
String s = meta.get("chunk." + id);
if (s == null) {
continue;
}
Chunk c = Chunk.fromString(meta.get(s));
Chunk c = Chunk.fromString(s);
if (c.id == header.id) {
c.start = header.start;
c.length = header.length;
c.metaRootPos = header.metaRootPos;
c.maxLengthLive = header.maxLengthLive;
c.pageCount = header.pageCount;
c.pageCountLive = header.pageCountLive;
c.maxLength = header.maxLength;
c.maxLengthLive = header.maxLengthLive;
}
lastChunkId = Math.max(c.id, lastChunkId);
chunks.put(c.id, c);
if (c.pageCountLive == 0) {
// remove this chunk in the next save operation
registerFreePage(currentVersion, c.id, 0, 0);
}
}
// rebuild the free space list
freeSpaceList.clear();
......@@ -835,7 +843,6 @@ public class MVStore {
c.maxLengthLive = Long.MAX_VALUE;
c.start = Long.MAX_VALUE;
c.length = Integer.MAX_VALUE;
time = Math.max(0, time - creationTime);
c.time = time;
c.version = version;
chunks.put(c.id, c);
......@@ -848,7 +855,9 @@ public class MVStore {
if (v >= 0 && m.getVersion() >= lastStoredVersion) {
MVMap<?, ?> r = m.openVersion(storeVersion);
r.waitUntilWritten(r.getRoot());
changed.add(r);
if (r.getRoot().getPos() == 0) {
changed.add(r);
}
}
}
}
......@@ -859,23 +868,8 @@ public class MVStore {
} else {
meta.put("root." + m.getId(), String.valueOf(Integer.MAX_VALUE));
}
}
applyFreedChunks(storeVersion);
Set<Integer> removedChunks = New.hashSet();
// do it twice, because changing the meta table
// could cause a chunk to become empty
for (int i = 0; i < 2; i++) {
for (Chunk x : chunks.values()) {
if (x.maxLengthLive == 0 && canOverwriteChunk(x, time)) {
meta.remove("chunk." + x.id);
removedChunks.add(x.id);
} else {
meta.put("chunk." + x.id, x.asString());
}
applyFreedChunks(storeVersion);
}
}
}
Set<Chunk> removedChunks = applyFreedPages(storeVersion, time);
ByteBuffer buff;
if (writeBuffer != null) {
buff = writeBuffer;
......@@ -899,7 +893,7 @@ public class MVStore {
meta.put("chunk." + c.id, c.asString());
if (ASSERT) {
if (freedChunks.size() > 0) {
if (freedPages.size() > 0) {
throw DataUtils.newIllegalStateException("Temporary freed chunks");
}
}
......@@ -920,13 +914,9 @@ public class MVStore {
long filePos = reuseSpace ? allocateChunk(length) : fileLength;
boolean storeAtEndOfFile = filePos + length >= fileLength;
// need to keep old chunks
// until they are are no longer referenced
// by an old version
// so empty space is not reused too early
for (int x : removedChunks) {
freeSpaceList.markFree(chunks.get(x));
chunks.remove(x);
// free up the space of unused chunks now
for (Chunk x : removedChunks) {
freeSpaceList.markFree(x);
}
c.start = filePos;
......@@ -971,7 +961,6 @@ public class MVStore {
currentStoreVersion = -1;
metaChanged = false;
lastStoredVersion = storeVersion;
return version;
}
......@@ -990,25 +979,64 @@ public class MVStore {
return System.currentTimeMillis() - creationTime;
}
private void applyFreedChunks(long storeVersion) {
synchronized (freedChunks) {
for (Iterator<Long> it = freedChunks.keySet().iterator(); it.hasNext();) {
long v = it.next();
if (v > storeVersion) {
continue;
/**
* Apply the freed pages to the chunk metadata. The metadata is updated, but
* freed chunks are not removed yet.
*
* @param storeVersion apply up to the given version
* @return the set of freed chunks (might be empty)
*/
private Set<Chunk> applyFreedPages(long storeVersion, long time) {
Set<Chunk> removedChunks = New.hashSet();
synchronized (freedPages) {
while (true) {
ArrayList<Chunk> modified = New.arrayList();
for (Iterator<Long> it = freedPages.keySet().iterator(); it.hasNext();) {
long v = it.next();
if (v > storeVersion) {
continue;
}
Map<Integer, Chunk> freed = freedPages.get(v);
for (Chunk f : freed.values()) {
Chunk c = chunks.get(f.id);
c.maxLengthLive += f.maxLengthLive;
c.pageCountLive += f.pageCountLive;
if (c.pageCountLive < 0) {
throw DataUtils.newIllegalStateException(
"Corrupt page count {0}", c.pageCountLive);
}
if (c.maxLengthLive < 0) {
throw DataUtils.newIllegalStateException(
"Corrupt max length {0}", c.maxLengthLive);
}
if (c.pageCount == 0 && c.maxLengthLive > 0) {
throw DataUtils.newIllegalStateException(
"Corrupt max length {0}", c.maxLengthLive);
}
modified.add(c);
}
it.remove();
}
Map<Integer, Chunk> freed = freedChunks.get(v);
for (Chunk f : freed.values()) {
Chunk c = chunks.get(f.id);
c.maxLengthLive += f.maxLengthLive;
if (c.maxLengthLive < 0) {
throw DataUtils.newIllegalStateException(
"Corrupt max length {0}", c.maxLengthLive);
for (Chunk c : modified) {
if (c.maxLengthLive == 0) {
if (canOverwriteChunk(c, time)) {
removedChunks.add(c);
chunks.remove(c.id);
meta.remove("chunk." + c.id);
} else {
// remove this chunk in the next save operation
registerFreePage(storeVersion + 1, c.id, 0, 0);
}
} else {
meta.put("chunk." + c.id, c.asString());
}
}
it.remove();
if (modified.size() == 0) {
break;
}
}
}
return removedChunks;
}
/**
......@@ -1267,18 +1295,23 @@ public class MVStore {
// can be re-used
version = currentStoreVersion;
}
synchronized (freedChunks) {
HashMap<Integer, Chunk>freed = freedChunks.get(version);
registerFreePage(version, c.id, DataUtils.getPageMaxLength(pos), 1);
}
private void registerFreePage(long version, int chunkId, long maxLengthLive, int pageCount) {
synchronized (freedPages) {
HashMap<Integer, Chunk>freed = freedPages.get(version);
if (freed == null) {
freed = New.hashMap();
freedChunks.put(version, freed);
freedPages.put(version, freed);
}
Chunk f = freed.get(c.id);
Chunk f = freed.get(chunkId);
if (f == null) {
f = new Chunk(c.id);
freed.put(c.id, f);
f = new Chunk(chunkId);
freed.put(chunkId, f);
}
f.maxLengthLive -= DataUtils.getPageMaxLength(pos);
f.maxLengthLive -= maxLengthLive;
f.pageCountLive -= pageCount;
}
}
......@@ -1492,8 +1525,8 @@ public class MVStore {
chunks.clear();
freeSpaceList.clear();
maps.clear();
synchronized (freedChunks) {
freedChunks.clear();
synchronized (freedPages) {
freedPages.clear();
}
currentVersion = version;
metaChanged = false;
......@@ -1506,10 +1539,10 @@ public class MVStore {
m.rollbackTo(version);
}
for (long v = currentVersion; v >= version; v--) {
if (freedChunks.size() == 0) {
if (freedPages.size() == 0) {
break;
}
freedChunks.remove(v);
freedPages.remove(v);
}
meta.rollbackTo(version);
metaChanged = false;
......@@ -1556,8 +1589,8 @@ public class MVStore {
}
private void revertTemp(long storeVersion) {
synchronized (freedChunks) {
for (Iterator<Long> it = freedChunks.keySet().iterator(); it.hasNext();) {
synchronized (freedPages) {
for (Iterator<Long> it = freedPages.keySet().iterator(); it.hasNext();) {
long v = it.next();
if (v > storeVersion) {
continue;
......
h2/src/main/org/h2/mvstore/Page.java
浏览文件 @ 09769f17
......@@ -814,11 +814,15 @@ public class Page {
^ DataUtils.getCheckValue(start)
^ DataUtils.getCheckValue(pageLength);
buff.putShort(start + 4, (short) check);
this.pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
if (pos != 0) {
throw DataUtils.newIllegalStateException("Page already stored");
}
pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
long max = DataUtils.getPageMaxLength(pos);
chunk.maxLength += max;
chunk.maxLengthLive += max;
chunk.pageCount++;
chunk.pageCountLive++;
return buff;
}
......
h2/src/main/org/h2/mvstore/TransactionStore.java
浏览文件 @ 09769f17
......@@ -90,6 +90,7 @@ public class TransactionStore {
MVMapConcurrent.Builder<long[], Object[]> builder =
new MVMapConcurrent.Builder<long[], Object[]>().
valueType(valueType);
// TODO escape other map names, to avoid conflicts
undoLog = store.openMap("undoLog", builder);
init();
}
......@@ -220,8 +221,9 @@ public class TransactionStore {
void commit(Transaction t, long maxLogId) {
store.incrementVersion();
for (long logId = 0; logId < maxLogId; logId++) {
Object[] op = undoLog.get(new long[] {
t.getId(), logId });
long[] undoKey = new long[] {
t.getId(), logId };
Object[] op = undoLog.get(undoKey);
int opType = (Integer) op[0];
if (opType == Transaction.OP_REMOVE) {
int mapId = (Integer) op[1];
......@@ -238,7 +240,7 @@ public class TransactionStore {
map.remove(key);
}
}
undoLog.remove(logId);
undoLog.remove(undoKey);
}
openTransactions.remove(t.getId());
openTransactionMap.remove(t.getId());
......@@ -858,6 +860,7 @@ public class TransactionStore {
}
public void removeMap() {
// TODO remove in a transaction
mapWrite.removeMap();
}
......
h2/src/main/org/h2/mvstore/db/MVTableEngine.java
浏览文件 @ 09769f17
......@@ -116,7 +116,8 @@ public class MVTableEngine implements TableEngine {
* @param store the store
*/
static void store(MVStore store) {
store.compact(50);
int test;
// store.compact(50);
store.store();
}
......
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ 09769f17
......@@ -203,6 +203,9 @@ java org.pitest.mutationtest.MutationCoverageReport
--targetTests org.h2.test.store.TestStreamStore
--sourceDirs src/test,src/tools
Dump heap on out of memory:
-XX:+HeapDumpOnOutOfMemoryError
Random test:
java15
cd h2database/h2/bin
......@@ -536,6 +539,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
ssl = false;
traceLevelFile = 0;
test();
testUnit();
big = false;
cipher = "AES";
......
h2/src/test/org/h2/test/TestBase.java
浏览文件 @ 09769f17
......@@ -600,9 +600,9 @@ public abstract class TestBase {
*/
public void assertEquals(java.util.Date expected, java.util.Date actual) {
if (expected != actual && !expected.equals(actual)) {
final DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
final SimpleTimeZone GMT_TIMEZONE = new SimpleTimeZone(0, "Z");
df.setTimeZone(GMT_TIMEZONE);
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
SimpleTimeZone gmt = new SimpleTimeZone(0, "Z");
df.setTimeZone(gmt);
fail("Expected: " + df.format(expected) + " actual: " + df.format(actual));
}
}
......
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ 09769f17
......@@ -8,7 +8,6 @@ package org.h2.test.store;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
......@@ -22,7 +21,6 @@ import org.h2.mvstore.type.StringDataType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.New;
/**
* Tests the MVStore.
......@@ -35,7 +33,10 @@ public class TestMVStore extends TestBase {
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
TestBase test = TestBase.createCaller().init();
test.config.traceTest = true;
test.config.big = true;
test.test();
}
public void test() throws Exception {
......@@ -61,7 +62,6 @@ public class TestMVStore extends TestBase {
testIterateOldVersion();
testObjects();
testExample();
testIterateOverChanges();
testOpenStoreCloseLoop();
testVersion();
testTruncateFile();
......@@ -79,10 +79,10 @@ public class TestMVStore extends TestBase {
testIterate();
testCloseTwice();
testSimple();
// this test will run out of memory on a 32-bit VM before it gets to the part we need it to test
if ("64".equals(System.getProperty("sun.arch.data.model"))) {
testLargerThan2G();
}
// longer running tests
testLargerThan2G();
}
private void testAtomicOperations() {
......@@ -705,39 +705,6 @@ public class TestMVStore extends TestBase {
}
}
private void testIterateOverChanges() {
String fileName = getBaseDir() + "/testIterate.h3";
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
s.setPageSize(6);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 100; i++) {
m.put(i, "Hi");
}
s.incrementVersion();
s.store();
for (int i = 20; i < 40; i++) {
assertEquals("Hi", m.put(i, "Hello"));
}
long old = s.getCurrentVersion();
s.incrementVersion();
for (int i = 10; i < 15; i++) {
m.put(i, "Hallo");
}
m.put(50, "Hallo");
for (int i = 90; i < 93; i++) {
assertEquals("Hi", m.remove(i));
}
assertEquals(null, m.put(100, "Hallo"));
Iterator<Integer> it = m.changeIterator(old);
ArrayList<Integer> list = New.arrayList();
while (it.hasNext()) {
list.add(it.next());
}
assertEquals("[10, 11, 12, 13, 14, 50, 100, 90, 91, 92]", list.toString());
s.close();
}
private void testOldVersion() {
MVStore s;
for (int op = 0; op <= 1; op++) {
......@@ -1031,8 +998,8 @@ public class TestMVStore extends TestBase {
assertEquals("name:data", m.get("map." + id));
assertTrue(m.get("root.1").length() > 0);
assertTrue(m.containsKey("chunk.1"));
assertEquals("id:1,length:246,maxLength:224,maxLengthLive:0," +
"metaRoot:274877910922,pageCount:2," +
assertEquals("id:1,length:263,maxLength:288,maxLengthLive:288," +
"metaRoot:274877910924,pageCount:2,pageCountLive:2," +
"start:8192,time:0,version:1", m.get("chunk.1"));
assertTrue(m.containsKey("chunk.2"));
......@@ -1397,30 +1364,36 @@ public class TestMVStore extends TestBase {
s.close();
}
private void testLargerThan2G() {
private void testLargerThan2G() throws IOException {
if (!config.big) {
return;
}
String fileName = getBaseDir() + "/testLargerThan2G.h3";
FileUtils.delete(fileName);
MVStore store = new MVStore.Builder().cacheSize(800).fileName(fileName).writeBufferSize(0).writeDelay(-1)
.open();
MVMap<String, String> map = store.openMap("test");
int totalWrite = 15000000;
int lineStored = 0;
while (lineStored <= totalWrite) {
lineStored++;
String actualKey = lineStored + " just for length length length " + lineStored;
String value = "Just a a string that is kinda long long long but not too much much much much much much much much much "
+ lineStored;
map.put(actualKey, value);
if (lineStored % 1000000 == 0) {
store.store();
MVStore store = new MVStore.Builder().cacheSize(16).
fileName(fileName).open();
MVMap<Integer, String> map = store.openMap("test");
long last = System.currentTimeMillis();
String data = new String(new char[2500]).replace((char) 0, 'x');
for (int i = 0;; i++) {
map.put(i, data);
if (i % 10000 == 0) {
long version = store.commit();
store.setRetainVersion(version);
long time = System.currentTimeMillis();
if (time - last > 2000) {
long mb = store.getFile().size() / 1024 / 1024;
trace(mb + "/4500");
if (mb > 4500) {
break;
}
last = time;
}
}
}
store.store();
store.commit();
store.close();
FileUtils.delete(fileName);
}
/**
......
h2/src/test/org/h2/test/store/TestMVTableEngine.java
浏览文件 @ 09769f17
......@@ -36,7 +36,7 @@ public class TestMVTableEngine extends TestBase {
public void test() throws Exception {
testEncryption();
testReadOnly();
// testReuseDiskSpace();
testReuseDiskSpace();
testDataTypes();
testLocking();
testSimple();
......@@ -87,8 +87,8 @@ public class TestMVTableEngine extends TestBase {
";DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
Connection conn;
Statement stat;
long maxSizeHalf = 0;
for (int i = 0; i < 10; i++) {
long maxSize = 0;
for (int i = 0; i < 20; i++) {
conn = getConnection(dbName);
for (MVTableEngine.Store s : MVTableEngine.getStores()) {
s.getStore().setRetentionTime(0);
......@@ -100,10 +100,10 @@ public class TestMVTableEngine extends TestBase {
conn.close();
long size = FileUtils.size(getBaseDir() + "/mvstore"
+ Constants.SUFFIX_MV_FILE);
if (i < 6) {
maxSizeHalf = Math.max(size, maxSizeHalf);
} else if (size > maxSizeHalf) {
fail(i + " size: " + size + " max: " + maxSizeHalf);
if (i < 10) {
maxSize = (int) (Math.max(size, maxSize) * 1.1);
} else if (size > maxSize) {
fail(i + " size: " + size + " max: " + maxSize);
}
}
}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论