Skip to content

H
h2database
提交 09769f17 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVStore: larger stores (multiple GB) are now much faster.

上级 f8db2f80
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 261 行增加 和 426 行删除
h2/src/docsrc/html/changelog.html
浏览文件 @ 09769f17
......@@ -18,7 +18,8 @@ Change Log
<h1>Change Log</h1>
<h2>Next Version (unreleased)</h2>
<ul><li>When using local temporary tables and not dropping them manually before closing the session,
<ul><li>MVStore: larger stores (multiple GB) are now much faster.
</li><li>When using local temporary tables and not dropping them manually before closing the session,
and then killing the process could result in a database that couldn't be opened (except when using
the recover tool).
</li><li>Support TRUNC(timestamp) for improved Oracle compatiblity.
......
h2/src/main/org/h2/mvstore/ChangeCursor.java deleted 100644 → 0
浏览文件 @ f8db2f80
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.mvstore;
import java.util.Iterator;
/**
* A cursor to iterate over all keys in new pages.
*
* @param <K> the key type
* @param <V> the value type
*/
public class ChangeCursor<K, V> implements Iterator<K> {
private final MVMap<K, V> map;
private final Page root1, root2;
/**
* The state of this cursor.
* 0: not initialized
* 1: reading from root1
* 2: reading from root2
* 3: closed
*/
private int state;
private CursorPos pos1, pos2;
private K current;
ChangeCursor(MVMap<K, V> map, Page root1, Page root2) {
this.map = map;
this.root1 = root1;
this.root2 = root2;
}
public K next() {
K c = current;
fetchNext();
return c;
}
public boolean hasNext() {
if (state == 0) {
pos1 = new CursorPos(root1, 0, null);
pos1 = min(pos1);
state = 1;
fetchNext();
}
return current != null;
}
public void remove() {
throw DataUtils.newUnsupportedOperationException(
"Removing is not supported");
}
private void fetchNext() {
while (fetchNextKey()) {
if (pos1 == null || pos2 == null) {
break;
}
@SuppressWarnings("unchecked")
V v1 = (V) map.binarySearch(root1, current);
@SuppressWarnings("unchecked")
V v2 = (V) map.binarySearch(root2, current);
if (!v1.equals(v2)) {
break;
}
}
}
private boolean fetchNextKey() {
while (true) {
if (state == 3) {
return false;
}
if (state == 1) {
// read from root1
pos1 = fetchNext(pos1);
if (pos1 == null) {
// reached the end of pos1
state = 2;
pos2 = null;
continue;
}
pos2 = find(root2, current);
if (pos2 == null) {
// not found in root2
return true;
}
if (!pos1.page.equals(pos2.page)) {
// the page is different,
// so the entry has possibly changed
return true;
}
while (true) {
pos1 = pos1.parent;
if (pos1 == null) {
// reached end of pos1
state = 2;
pos2 = null;
break;
}
pos2 = pos2.parent;
if (pos2 == null || !pos1.page.equals(pos2.page)) {
if (pos1.index + 1 < map.getChildPageCount(pos1.page)) {
pos1 = new CursorPos(pos1.page.getChildPage(++pos1.index), 0, pos1);
pos1 = min(pos1);
break;
}
}
}
}
if (state == 2) {
if (pos2 == null) {
// init reading from root2
pos2 = new CursorPos(root2, 0, null);
pos2 = min(pos2);
}
// read from root2
pos2 = fetchNext(pos2);
if (pos2 == null) {
// reached the end of pos2
state = 3;
current = null;
continue;
}
pos1 = find(root1, current);
if (pos1 != null) {
// found a corresponding record
// so it was not deleted
// but now we may need to skip pages
if (!pos1.page.equals(pos2.page)) {
// the page is different
pos1 = null;
continue;
}
while (true) {
pos2 = pos2.parent;
if (pos2 == null) {
// reached end of pos1
state = 3;
current = null;
pos1 = null;
break;
}
pos1 = pos1.parent;
if (pos1 == null || !pos2.page.equals(pos1.page)) {
if (pos2.index + 1 < map.getChildPageCount(pos2.page)) {
pos2 = new CursorPos(pos2.page.getChildPage(++pos2.index), 0, pos2);
pos2 = min(pos2);
break;
}
}
}
pos1 = null;
continue;
}
// found no corresponding record
// so it was deleted
return true;
}
}
}
private CursorPos find(Page p, K key) {
// TODO combine with RangeCursor.min
// possibly move to MVMap
CursorPos pos = null;
while (true) {
if (p.isLeaf()) {
int x = key == null ? 0 : p.binarySearch(key);
if (x < 0) {
return null;
}
return new CursorPos(p, x, pos);
}
int x = key == null ? -1 : p.binarySearch(key);
if (x < 0) {
x = -x - 1;
} else {
x++;
}
pos = new CursorPos(p, x, pos);
p = p.getChildPage(x);
}
}
@SuppressWarnings("unchecked")
private CursorPos fetchNext(CursorPos p) {
while (p != null) {
if (p.index < p.page.getKeyCount()) {
current = (K) p.page.getKey(p.index++);
return p;
}
p = p.parent;
if (p == null) {
break;
}
if (p.index + 1 < map.getChildPageCount(p.page)) {
p = new CursorPos(p.page.getChildPage(++p.index), 0, p);
p = min(p);
}
}
current = null;
return p;
}
private static CursorPos min(CursorPos p) {
while (true) {
if (p.page.isLeaf()) {
return p;
}
Page c = p.page.getChildPage(0);
p = new CursorPos(c, 0, p);
}
}
}
h2/src/main/org/h2/mvstore/Chunk.java
浏览文件 @ 09769f17
......@@ -42,9 +42,14 @@ public class Chunk {
int length;
/**
* The number of pages.
* The total number of pages in this chunk.
*/
int pageCount;
/**
* The number of pages still alive.
*/
int pageCountLive;
/**
* The sum of the max length of all pages.
......@@ -101,6 +106,7 @@ public class Chunk {
Chunk c = new Chunk(chunkId);
c.length = length;
c.pageCount = pageCount;
c.pageCountLive = pageCount;
c.start = start;
c.metaRootPos = metaRootPos;
c.maxLength = maxLength;
......@@ -136,6 +142,7 @@ public class Chunk {
c.start = Long.parseLong(map.get("start"));
c.length = Integer.parseInt(map.get("length"));
c.pageCount = Integer.parseInt(map.get("pageCount"));
c.pageCountLive = Integer.parseInt(map.get("pageCountLive"));
c.maxLength = Long.parseLong(map.get("maxLength"));
c.maxLengthLive = Long.parseLong(map.get("maxLengthLive"));
c.metaRootPos = Long.parseLong(map.get("metaRoot"));
......@@ -169,6 +176,7 @@ public class Chunk {
"maxLengthLive:" + maxLengthLive + "," +
"metaRoot:" + metaRootPos + "," +
"pageCount:" + pageCount + "," +
"pageCountLive:" + pageCountLive + "," +
"start:" + start + "," +
"time:" + time + "," +
"version:" + version;
......
h2/src/main/org/h2/mvstore/FreeSpaceList.java
浏览文件 @ 09769f17
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.mvstore;
import java.util.ArrayList;
import java.util.List;
/**
* A list that maintains ranges of free space (in pages) in a file.
*/
public class FreeSpaceList {
/** the first 2 pages are occupied by the file header. */
/**
* The first 2 pages are occupied by the file header.
*/
private static final int FIRST_FREE_PAGE = 2;
/** I use max_value/2 to avoid overflow errors during arithmetic. */
private static final int MAX_NO_PAGES = Integer.MAX_VALUE / 2;
/**
* The maximum number of pages. Smaller than than MAX_VALUE to avoid
* overflow errors during arithmetic operations.
*/
private static final int MAX_PAGE_COUNT = Integer.MAX_VALUE / 2;
private List<PageRange> freeSpaceList = new ArrayList<PageRange>();
public FreeSpaceList() {
freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_NO_PAGES));
clear();
}
private static final class PageRange {
/** the start point, in pages */
public int start;
/** the length, in pages */
public int length;
public PageRange(int start, int length) {
this.start = start;
this.length = length;
}
@Override
public String toString() {
return "start:" + start + " length:" + length;
}
/**
* Reset the list.
*/
public synchronized void clear() {
freeSpaceList.clear();
freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_PAGE_COUNT));
}
/**
* @return position in pages
* Allocate a number of pages.
*
* @param length the number of bytes to allocate
* @return the position in pages
*/
public synchronized int allocatePages(long length) {
int required = (int) (length / MVStore.BLOCK_SIZE) + 1;
......@@ -43,9 +52,15 @@ public class FreeSpaceList {
return pr.start;
}
}
throw DataUtils.newIllegalStateException("could not find a free page to allocate");
throw DataUtils.newIllegalStateException(
"Could not find a free page to allocate");
}
/**
* Mark a chunk as used.
*
* @param c the chunk
*/
public synchronized void markUsed(Chunk c) {
int chunkStart = (int) (c.start / MVStore.BLOCK_SIZE);
int required = (int) ((c.start + c.length) / MVStore.BLOCK_SIZE) + 2 - chunkStart;
......@@ -59,11 +74,12 @@ public class FreeSpaceList {
i++;
}
if (found == null) {
throw DataUtils.newIllegalStateException("cannot find spot to mark chunk as used in free list, "
+ c.toString());
throw DataUtils.newIllegalStateException(
"Cannot find spot to mark chunk as used in free list: {0}", c);
}
if (chunkStart + required > found.start + found.length) {
throw DataUtils.newIllegalStateException("chunk runs over edge of free space, " + c.toString());
throw DataUtils.newIllegalStateException(
"Chunk runs over edge of free space: {0}", c);
}
if (found.start == chunkStart) {
// if the used-chunk is at the beginning of a free-space-range
......@@ -92,6 +108,11 @@ public class FreeSpaceList {
}
}
/**
* Mark the chunk as free.
*
* @param c the chunk
*/
public synchronized void markFree(Chunk c) {
int chunkStart = (int) (c.start / MVStore.BLOCK_SIZE);
int required = (c.length / MVStore.BLOCK_SIZE) + 1;
......@@ -105,15 +126,15 @@ public class FreeSpaceList {
i++;
}
if (found == null) {
throw DataUtils.newIllegalStateException("cannot find spot to mark chunk as unused in free list, "
+ c.toString());
throw DataUtils.newIllegalStateException(
"Cannot find spot to mark chunk as unused in free list: {0}", c);
}
if (chunkStart + required + 1 == found.start) {
// if the used-chunk is adjacent to the beginning of a
// free-space-range
found.start = chunkStart;
found.length += required;
// compactify-free-list: merge the previous entry into this one if
// compact: merge the previous entry into this one if
// they are now adjacent
if (i > 0) {
PageRange previous = freeSpaceList.get(i - 1);
......@@ -129,7 +150,7 @@ public class FreeSpaceList {
PageRange previous = freeSpaceList.get(i - 1);
if (previous.start + previous.length + 1 == chunkStart) {
previous.length += required;
// compactify-free-list: merge the next entry into this one if
// compact: merge the next entry into this one if
// they are now adjacent
if (previous.start + previous.length + 1 == found.start) {
previous.length += found.length;
......@@ -144,23 +165,44 @@ public class FreeSpaceList {
freeSpaceList.add(i, newRange);
}
public synchronized void clear() {
freeSpaceList.clear();
freeSpaceList.add(new PageRange(FIRST_FREE_PAGE, MAX_NO_PAGES));
}
/** debug method */
public void dumpFreeList() {
StringBuilder buf = new StringBuilder("free list : ");
public String toString() {
StringBuilder buff = new StringBuilder();
boolean first = true;
for (PageRange pr : freeSpaceList) {
for (PageRange r : freeSpaceList) {
if (first) {
first = false;
} else {
buf.append(", ");
buff.append(", ");
}
buf.append(pr.start + "-" + (pr.start + pr.length - 1));
buff.append(r.start + "-" + (r.start + r.length - 1));
}
System.out.println(buf.toString());
return buff.toString();
}
/**
* A range of free pages.
*/
private static final class PageRange {
/**
* The starting point, in pages.
*/
public int start;
/**
* The length, in pages.
*/
public int length;
public PageRange(int start, int length) {
this.start = start;
this.length = length;
}
@Override
public String toString() {
return "start:" + start + " length:" + length;
}
}
}
h2/src/main/org/h2/mvstore/MVMap.java
浏览文件 @ 09769f17
......@@ -773,18 +773,6 @@ public class MVMap<K, V> extends AbstractMap<K, V>
return new Cursor<K>(this, root, from);
}
/**
* Iterate over all keys in changed pages.
*
* @param version the old version
* @return the iterator
*/
public Iterator<K> changeIterator(long version) {
checkOpen();
MVMap<K, V> old = openVersion(version);
return new ChangeCursor<K, V>(this, root, old.root);
}
public Set<Map.Entry<K, V>> entrySet() {
HashMap<K, V> map = new HashMap<K, V>();
for (K k : keySet()) {
......
h2/src/main/org/h2/mvstore/Page.java
浏览文件 @ 09769f17
......@@ -814,11 +814,15 @@ public class Page {
^ DataUtils.getCheckValue(start)
^ DataUtils.getCheckValue(pageLength);
buff.putShort(start + 4, (short) check);
this.pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
if (pos != 0) {
throw DataUtils.newIllegalStateException("Page already stored");
}
pos = DataUtils.getPagePos(chunkId, start, pageLength, type);
long max = DataUtils.getPageMaxLength(pos);
chunk.maxLength += max;
chunk.maxLengthLive += max;
chunk.pageCount++;
chunk.pageCountLive++;
return buff;
}
......
h2/src/main/org/h2/mvstore/TransactionStore.java
浏览文件 @ 09769f17
......@@ -90,6 +90,7 @@ public class TransactionStore {
MVMapConcurrent.Builder<long[], Object[]> builder =
new MVMapConcurrent.Builder<long[], Object[]>().
valueType(valueType);
// TODO escape other map names, to avoid conflicts
undoLog = store.openMap("undoLog", builder);
init();
}
......@@ -220,8 +221,9 @@ public class TransactionStore {
void commit(Transaction t, long maxLogId) {
store.incrementVersion();
for (long logId = 0; logId < maxLogId; logId++) {
Object[] op = undoLog.get(new long[] {
t.getId(), logId });
long[] undoKey = new long[] {
t.getId(), logId };
Object[] op = undoLog.get(undoKey);
int opType = (Integer) op[0];
if (opType == Transaction.OP_REMOVE) {
int mapId = (Integer) op[1];
......@@ -238,7 +240,7 @@ public class TransactionStore {
map.remove(key);
}
}
undoLog.remove(logId);
undoLog.remove(undoKey);
}
openTransactions.remove(t.getId());
openTransactionMap.remove(t.getId());
......@@ -858,6 +860,7 @@ public class TransactionStore {
}
public void removeMap() {
// TODO remove in a transaction
mapWrite.removeMap();
}
......
h2/src/main/org/h2/mvstore/db/MVTableEngine.java
浏览文件 @ 09769f17
......@@ -116,7 +116,8 @@ public class MVTableEngine implements TableEngine {
* @param store the store
*/
static void store(MVStore store) {
store.compact(50);
int test;
// store.compact(50);
store.store();
}
......
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ 09769f17
......@@ -203,6 +203,9 @@ java org.pitest.mutationtest.MutationCoverageReport
--targetTests org.h2.test.store.TestStreamStore
--sourceDirs src/test,src/tools
Dump heap on out of memory:
-XX:+HeapDumpOnOutOfMemoryError
Random test:
java15
cd h2database/h2/bin
......@@ -536,6 +539,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
ssl = false;
traceLevelFile = 0;
test();
testUnit();
big = false;
cipher = "AES";
......
h2/src/test/org/h2/test/TestBase.java
浏览文件 @ 09769f17
......@@ -600,9 +600,9 @@ public abstract class TestBase {
*/
public void assertEquals(java.util.Date expected, java.util.Date actual) {
if (expected != actual && !expected.equals(actual)) {
final DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
final SimpleTimeZone GMT_TIMEZONE = new SimpleTimeZone(0, "Z");
df.setTimeZone(GMT_TIMEZONE);
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
SimpleTimeZone gmt = new SimpleTimeZone(0, "Z");
df.setTimeZone(gmt);
fail("Expected: " + df.format(expected) + " actual: " + df.format(actual));
}
}
......
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ 09769f17
......@@ -8,7 +8,6 @@ package org.h2.test.store;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
......@@ -22,7 +21,6 @@ import org.h2.mvstore.type.StringDataType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.New;
/**
* Tests the MVStore.
......@@ -35,7 +33,10 @@ public class TestMVStore extends TestBase {
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
TestBase test = TestBase.createCaller().init();
test.config.traceTest = true;
test.config.big = true;
test.test();
}
public void test() throws Exception {
......@@ -61,7 +62,6 @@ public class TestMVStore extends TestBase {
testIterateOldVersion();
testObjects();
testExample();
testIterateOverChanges();
testOpenStoreCloseLoop();
testVersion();
testTruncateFile();
......@@ -79,10 +79,10 @@ public class TestMVStore extends TestBase {
testIterate();
testCloseTwice();
testSimple();
// this test will run out of memory on a 32-bit VM before it gets to the part we need it to test
if ("64".equals(System.getProperty("sun.arch.data.model"))) {
testLargerThan2G();
}
// longer running tests
testLargerThan2G();
}
private void testAtomicOperations() {
......@@ -705,39 +705,6 @@ public class TestMVStore extends TestBase {
}
}
private void testIterateOverChanges() {
String fileName = getBaseDir() + "/testIterate.h3";
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
s.setPageSize(6);
MVMap<Integer, String> m = s.openMap("data");
for (int i = 0; i < 100; i++) {
m.put(i, "Hi");
}
s.incrementVersion();
s.store();
for (int i = 20; i < 40; i++) {
assertEquals("Hi", m.put(i, "Hello"));
}
long old = s.getCurrentVersion();
s.incrementVersion();
for (int i = 10; i < 15; i++) {
m.put(i, "Hallo");
}
m.put(50, "Hallo");
for (int i = 90; i < 93; i++) {
assertEquals("Hi", m.remove(i));
}
assertEquals(null, m.put(100, "Hallo"));
Iterator<Integer> it = m.changeIterator(old);
ArrayList<Integer> list = New.arrayList();
while (it.hasNext()) {
list.add(it.next());
}
assertEquals("[10, 11, 12, 13, 14, 50, 100, 90, 91, 92]", list.toString());
s.close();
}
private void testOldVersion() {
MVStore s;
for (int op = 0; op <= 1; op++) {
......@@ -1031,8 +998,8 @@ public class TestMVStore extends TestBase {
assertEquals("name:data", m.get("map." + id));
assertTrue(m.get("root.1").length() > 0);
assertTrue(m.containsKey("chunk.1"));
assertEquals("id:1,length:246,maxLength:224,maxLengthLive:0," +
"metaRoot:274877910922,pageCount:2," +
assertEquals("id:1,length:263,maxLength:288,maxLengthLive:288," +
"metaRoot:274877910924,pageCount:2,pageCountLive:2," +
"start:8192,time:0,version:1", m.get("chunk.1"));
assertTrue(m.containsKey("chunk.2"));
......@@ -1397,30 +1364,36 @@ public class TestMVStore extends TestBase {
s.close();
}
private void testLargerThan2G() {
private void testLargerThan2G() throws IOException {
if (!config.big) {
return;
}
String fileName = getBaseDir() + "/testLargerThan2G.h3";
FileUtils.delete(fileName);
MVStore store = new MVStore.Builder().cacheSize(800).fileName(fileName).writeBufferSize(0).writeDelay(-1)
.open();
MVMap<String, String> map = store.openMap("test");
int totalWrite = 15000000;
int lineStored = 0;
while (lineStored <= totalWrite) {
lineStored++;
String actualKey = lineStored + " just for length length length " + lineStored;
String value = "Just a a string that is kinda long long long but not too much much much much much much much much much "
+ lineStored;
map.put(actualKey, value);
if (lineStored % 1000000 == 0) {
store.store();
MVStore store = new MVStore.Builder().cacheSize(16).
fileName(fileName).open();
MVMap<Integer, String> map = store.openMap("test");
long last = System.currentTimeMillis();
String data = new String(new char[2500]).replace((char) 0, 'x');
for (int i = 0;; i++) {
map.put(i, data);
if (i % 10000 == 0) {
long version = store.commit();
store.setRetainVersion(version);
long time = System.currentTimeMillis();
if (time - last > 2000) {
long mb = store.getFile().size() / 1024 / 1024;
trace(mb + "/4500");
if (mb > 4500) {
break;
}
last = time;
}
}
}
store.store();
store.commit();
store.close();
FileUtils.delete(fileName);
}
/**
......
h2/src/test/org/h2/test/store/TestMVTableEngine.java
浏览文件 @ 09769f17
......@@ -36,7 +36,7 @@ public class TestMVTableEngine extends TestBase {
public void test() throws Exception {
testEncryption();
testReadOnly();
// testReuseDiskSpace();
testReuseDiskSpace();
testDataTypes();
testLocking();
testSimple();
......@@ -87,8 +87,8 @@ public class TestMVTableEngine extends TestBase {
";DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
Connection conn;
Statement stat;
long maxSizeHalf = 0;
for (int i = 0; i < 10; i++) {
long maxSize = 0;
for (int i = 0; i < 20; i++) {
conn = getConnection(dbName);
for (MVTableEngine.Store s : MVTableEngine.getStores()) {
s.getStore().setRetentionTime(0);
......@@ -100,10 +100,10 @@ public class TestMVTableEngine extends TestBase {
conn.close();
long size = FileUtils.size(getBaseDir() + "/mvstore"
+ Constants.SUFFIX_MV_FILE);
if (i < 6) {
maxSizeHalf = Math.max(size, maxSizeHalf);
} else if (size > maxSizeHalf) {
fail(i + " size: " + size + " max: " + maxSizeHalf);
if (i < 10) {
maxSize = (int) (Math.max(size, maxSize) * 1.1);
} else if (size > maxSize) {
fail(i + " size: " + size + " max: " + maxSize);
}
}
}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论