Skip to content

H
h2database
提交 53fd89a8 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVTableEngine

上级 f26792c2
全部展开 显示空白字符变更
内嵌 并排
正在显示 包含 828 行增加 和 35 行删除
h2/src/main/org/h2/engine/Database.java
浏览文件 @ 53fd89a8
......@@ -6,6 +6,7 @@
*/
package org.h2.engine;
import java.beans.ExceptionListener;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
......@@ -273,6 +274,15 @@ public class Database implements DataHandler {
public void setMvStore(MVTableEngine.Store mvStore) {
this.mvStore = mvStore;
mvStore.getStore().setBackgroundExceptionListener(new ExceptionListener() {
@Override
public void exceptionThrown(Exception e) {
setBackgroundException(DbException.convert(e));
}
});
}
/**
......@@ -1086,6 +1096,7 @@ public class Database implements DataHandler {
if (closing) {
return;
}
throwLastBackgroundException();
if (fileLockMethod == FileLock.LOCK_SERIALIZED && !reconnectChangePending) {
// another connection may have written something - don't write
try {
......@@ -1799,6 +1810,17 @@ public class Database implements DataHandler {
* @param session the session
*/
synchronized void commit(Session session) {
throwLastBackgroundException();
if (readOnly) {
return;
}
if (pageStore != null) {
pageStore.commit(session);
}
session.setAllCommitted();
}
private void throwLastBackgroundException() {
if (backgroundException != null) {
// we don't care too much about concurrency here,
// we just want to make sure the exception is _normally_
......@@ -1809,13 +1831,16 @@ public class Database implements DataHandler {
throw b;
}
}
if (readOnly) {
return;
}
if (pageStore != null) {
pageStore.commit(session);
public void setBackgroundException(DbException e) {
if (backgroundException == null) {
backgroundException = e;
TraceSystem t = getTraceSystem();
if (t != null) {
t.getTrace(Trace.DATABASE).error(e, "flush");
}
}
session.setAllCommitted();
}
/**
......
h2/src/main/org/h2/mvstore/Chunk.java
浏览文件 @ 53fd89a8
......@@ -116,7 +116,7 @@ public class Chunk {
}
/**
* Write the header.
* Write the chunk header.
*
* @param buff the target buffer
*/
......
h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java 0 → 100644
浏览文件 @ 53fd89a8
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.mvstore;
import java.util.BitSet;
import org.h2.util.MathUtils;
/**
* A free space bit set.
*/
public class FreeSpaceBitSet {
/**
* The first usable block.
*/
private final int firstFreeBlock;
/**
* The block size in bytes.
*/
private final int blockSize;
/**
* The bit set.
*/
private final BitSet set = new BitSet();
/**
* Create a new free space map.
*
* @param firstFreeBlock the first free block
* @param blockSize the block size
*/
public FreeSpaceBitSet(int firstFreeBlock, int blockSize) {
this.firstFreeBlock = firstFreeBlock;
this.blockSize = blockSize;
clear();
}
/**
* Reset the list.
*/
public synchronized void clear() {
set.clear();
set.set(0, firstFreeBlock);
}
/**
* Check whether one of the blocks is in use.
*
* @param pos the position in bytes
* @param length the number of bytes
* @return true if a block is in use
*/
public synchronized boolean isUsed(long pos, int length) {
int start = getBlock(pos);
int blocks = getBlockCount(length);
for (int i = start; i < start + blocks; i++) {
if (!set.get(i)) {
return false;
}
}
return true;
}
/**
* Check whether one of the blocks is free.
*
* @param pos the position in bytes
* @param length the number of bytes
* @return true if a block is free
*/
public synchronized boolean isFree(long pos, int length) {
int start = getBlock(pos);
int blocks = getBlockCount(length);
for (int i = start; i < start + blocks; i++) {
if (set.get(i)) {
return false;
}
}
return true;
}
/**
* Allocate a number of blocks and mark them as used.
*
* @param length the number of bytes to allocate
* @return the start position in bytes
*/
public synchronized long allocate(int length) {
int blocks = getBlockCount(length);
for (int i = 0;;) {
int start = set.nextClearBit(i);
int end = set.nextSetBit(start + 1);
if (end < 0 || end - start >= blocks) {
set.set(start, start + blocks);
return getPos(start);
}
i = end;
}
}
/**
* Mark the space as in use.
*
* @param pos the position in bytes
* @param length the number of bytes
*/
public synchronized void markUsed(long pos, int length) {
int start = getBlock(pos);
int blocks = getBlockCount(length);
set.set(start, start + blocks);
}
/**
* Mark the space as free.
*
* @param pos the position in bytes
* @param length the number of bytes
*/
public synchronized void free(long pos, int length) {
int start = getBlock(pos);
int blocks = getBlockCount(length);
set.clear(start, start + blocks);
}
private long getPos(int block) {
return (long) block * (long) blockSize;
}
private int getBlock(long pos) {
return (int) (pos / blockSize);
}
private int getBlockCount(int length) {
return MathUtils.roundUpInt(length, blockSize) / blockSize;
}
@Override
public String toString() {
StringBuilder buff = new StringBuilder("[");
for (int i = 0;;) {
if (i > 0) {
buff.append(", ");
}
int start = set.nextClearBit(i);
buff.append(start).append('-');
int end = set.nextSetBit(start + 1);
if (end < 0) {
break;
}
buff.append(end - 1);
i = end + 1;
}
return buff.append(']').toString();
}
}
\ No newline at end of file
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ 53fd89a8
......@@ -6,6 +6,7 @@
*/
package org.h2.mvstore;
import java.beans.ExceptionListener;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
......@@ -104,7 +105,6 @@ MVStore:
- maybe rename 'rollback' to 'revert' to distinguish from transactions
- support other compression algorithms (deflate, LZ4,...)
- only retain the last version, unless explicitly set (setRetainVersion)
- unit test for the FreeSpaceList; maybe find a simpler implementation
- support opening (existing) maps by id
- more consistent null handling (keys/values sometimes may be null)
- logging mechanism, specially for operations in a background thread
......@@ -161,11 +161,12 @@ public class MVStore {
*/
private final ConcurrentHashMap<Integer, Chunk> chunks =
new ConcurrentHashMap<Integer, Chunk>();
/**
* The list of free spaces between the chunks.
* The free spaces between the chunks. The first block to use is block 2
* (the first two blocks are the file header).
*/
private FreeSpaceList freeSpaceList = new FreeSpaceList();
private FreeSpaceBitSet freeSpace = new FreeSpaceBitSet(2, BLOCK_SIZE);
/**
* The map of temporarily removed pages. The key is the unsaved version, the
......@@ -235,6 +236,8 @@ public class MVStore {
*/
private int writeDelay = 1000;
private ExceptionListener backgroundExceptionListener;
MVStore(HashMap<String, Object> config) {
String f = (String) config.get("fileName");
if (f != null && f.indexOf(':') < 0) {
......@@ -598,12 +601,13 @@ public class MVStore {
}
}
// rebuild the free space list
freeSpaceList.clear();
freeSpace.clear();
for (Chunk c : chunks.values()) {
if (c.start == Long.MAX_VALUE) {
continue;
}
freeSpaceList.markUsed(c);
int len = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
freeSpace.markUsed(c.start, len);
}
}
......@@ -710,6 +714,7 @@ public class MVStore {
/**
* Close the file and the store, without writing anything.
* This will stop the background thread.
*/
public void closeImmediately() {
closeFile(false);
......@@ -743,7 +748,7 @@ public class MVStore {
}
meta = null;
chunks.clear();
freeSpaceList.clear();
freeSpace.clear();
cache.clear();
maps.clear();
} catch (Exception e) {
......@@ -927,6 +932,8 @@ public class MVStore {
int chunkLength = buff.position();
// round to the next block,
// and one additional block for the file header
int length = MathUtils.roundUpInt(chunkLength, BLOCK_SIZE) + BLOCK_SIZE;
if (length > buff.capacity()) {
buff = DataUtils.ensureCapacity(buff, length - buff.capacity());
......@@ -934,17 +941,22 @@ public class MVStore {
buff.limit(length);
long fileSizeUsed = getFileSizeUsed();
long filePos = reuseSpace ? allocateChunk(length) : fileSizeUsed;
long filePos;
if (reuseSpace) {
filePos = freeSpace.allocate(length);
} else {
filePos = fileSizeUsed;
freeSpace.markUsed(fileSizeUsed, length);
}
boolean storeAtEndOfFile = filePos + length >= fileSizeUsed;
// free up the space of unused chunks now
for (Chunk x : removedChunks) {
freeSpaceList.markFree(x);
freeSpace.free(x.start, x.length);
}
c.start = filePos;
c.length = chunkLength;
freeSpaceList.markUsed(c);
c.metaRootPos = meta.getRoot().getPos();
buff.position(0);
c.writeHeader(buff);
......@@ -1107,10 +1119,6 @@ public class MVStore {
return size;
}
private long allocateChunk(long length) {
return ((long) freeSpaceList.allocatePages(length)) * BLOCK_SIZE;
}
/**
* Check whether there are any unsaved changes.
*
......@@ -1450,6 +1458,20 @@ public class MVStore {
return v;
}
/**
* Set the listener to be used for exceptions that occur in the background thread.
*
* @param backgroundExceptionListener the listener
*/
public void setBackgroundExceptionListener(
ExceptionListener backgroundExceptionListener) {
this.backgroundExceptionListener = backgroundExceptionListener;
}
public ExceptionListener getBackgroundExceptionListener() {
return backgroundExceptionListener;
}
/**
* Check whether all data can be read from this version. This requires that
* all chunks referenced by this version are still available (not
......@@ -1561,7 +1583,7 @@ public class MVStore {
}
meta.clear();
chunks.clear();
freeSpaceList.clear();
freeSpace.clear();
maps.clear();
synchronized (freedPages) {
freedPages.clear();
......@@ -1594,7 +1616,7 @@ public class MVStore {
loadFromFile = true;
do {
last = chunks.remove(lastChunkId);
freeSpaceList.markFree(last);
freeSpace.free(last.start, last.length);
lastChunkId--;
} while (last.version > version && chunks.size() > 0);
rootChunkStart = last.start;
......@@ -1771,7 +1793,13 @@ public class MVStore {
if (time <= lastStoreTime + writeDelay) {
return;
}
try {
store(true);
} catch (Exception e) {
if (backgroundExceptionListener != null) {
backgroundExceptionListener.exceptionThrown(e);
}
}
}
/**
......@@ -1780,8 +1808,10 @@ public class MVStore {
* @param mb the cache size in MB.
*/
public void setCacheSize(long mb) {
if (cache != null) {
cache.setMaxMemory(mb * 1024 * 1024);
}
}
public boolean isReadOnly() {
return readOnly;
......@@ -1848,13 +1878,7 @@ public class MVStore {
// ignore
}
}
try {
store.storeInBackground();
} catch (Exception e) {
int todo;
// TODO throw the exception in the main thread
// at some point, or log the problem
}
}
}
......
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ 53fd89a8
......@@ -110,6 +110,7 @@ import org.h2.test.store.TestCacheLIRS;
import org.h2.test.store.TestCacheLongKeyLIRS;
import org.h2.test.store.TestConcurrent;
import org.h2.test.store.TestDataUtils;
import org.h2.test.store.TestFreeSpace;
import org.h2.test.store.TestMVRTree;
import org.h2.test.store.TestMVStore;
import org.h2.test.store.TestMVTableEngine;
......@@ -448,10 +449,17 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
} else {
test.runTests();
Profiler prof = new Profiler();
prof.depth = 4;
prof.depth = 8;
prof.interval = 1;
prof.startCollecting();
if (test.mvStore) {
TestPerformance.main("-init", "-db", "9", "-size", "1000");
TestPerformance.main("-init", "-db", "1", "-size", "1000");
TestPerformance.main("-init", "-db", "9", "-size", "1000");
TestPerformance.main("-init", "-db", "1", "-size", "1000");
} else {
TestPerformance.main("-init", "-db", "1");
}
prof.stopCollecting();
System.out.println(prof.getTop(3));
// Recover.execute("data", null);
......@@ -694,6 +702,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
new TestCacheLongKeyLIRS().runTest(this);
new TestConcurrent().runTest(this);
new TestDataUtils().runTest(this);
new TestFreeSpace().runTest(this);
new TestMVRTree().runTest(this);
new TestMVStore().runTest(this);
new TestMVTableEngine().runTest(this);
......
h2/src/test/org/h2/test/bench/test.properties
浏览文件 @ 53fd89a8
db1 = H2, org.h2.Driver, jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, sa
db9 = H2 (MVStore), org.h2.Driver, jdbc:h2:data/test;MV_STORE=TRUE;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, sa
#xdb1 = H2, org.h2.Driver, jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine, sa, sa
......
h2/src/test/org/h2/test/store/FreeSpaceTree.java 0 → 100644
浏览文件 @ 53fd89a8
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.util.TreeSet;
import org.h2.mvstore.DataUtils;
import org.h2.util.MathUtils;
/**
* A list that maintains ranges of free space (in blocks) in a file.
*/
public class FreeSpaceTree {
/**
* The first usable block.
*/
private final int firstFreeBlock;
/**
* The block size in bytes.
*/
private final int blockSize;
/**
* The list of free space.
*/
private TreeSet<BlockRange> freeSpace = new TreeSet<BlockRange>();
public FreeSpaceTree(int firstFreeBlock, int blockSize) {
this.firstFreeBlock = firstFreeBlock;
if (Integer.bitCount(blockSize) != 1) {
throw DataUtils.newIllegalArgumentException("Block size is not a power of 2");
}
this.blockSize = blockSize;
clear();
}
/**
* Reset the list.
*/
public synchronized void clear() {
freeSpace.clear();
freeSpace.add(new BlockRange(firstFreeBlock,
Integer.MAX_VALUE - firstFreeBlock));
}
/**
* Allocate a number of blocks and mark them as used.
*
* @param length the number of bytes to allocate
* @return the start position in bytes
*/
public synchronized long allocate(int length) {
int blocks = getBlockCount(length);
BlockRange x = null;
for (BlockRange b : freeSpace) {
if (b.blocks >= blocks) {
x = b;
break;
}
}
long pos = getPos(x.start);
if (x.blocks == blocks) {
freeSpace.remove(x);
} else {
x.start += blocks;
x.blocks -= blocks;
}
return pos;
}
public synchronized void markUsed(long pos, int length) {
int start = getBlock(pos);
int blocks = getBlockCount(length);
BlockRange x = new BlockRange(start, blocks);
BlockRange prev = freeSpace.floor(x);
if (prev == null) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_INTERNAL, "Free space already marked");
}
if (prev.start == start) {
if (prev.blocks == blocks) {
// match
freeSpace.remove(prev);
} else {
// cut the front
prev.start += blocks;
prev.blocks -= blocks;
}
} else if (prev.start + prev.blocks == start + blocks) {
// cut the end
prev.blocks -= blocks;
} else {
// insert an entry
x.start = start + blocks;
x.blocks = prev.start + prev.blocks - x.start;
freeSpace.add(x);
prev.blocks = start - prev.start;
}
}
public synchronized void free(long pos, int length) {
int start = getBlock(pos);
int blocks = getBlockCount(length);
BlockRange x = new BlockRange(start, blocks);
BlockRange next = freeSpace.ceiling(x);
if (next == null) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_INTERNAL, "Free space sentinel is missing");
}
BlockRange prev = freeSpace.lower(x);
if (prev != null) {
if (prev.start + prev.blocks == start) {
// extend the previous entry
prev.blocks += blocks;
if (prev.start + prev.blocks == next.start) {
// merge with the next entry
prev.blocks += next.blocks;
freeSpace.remove(next);
}
return;
}
}
if (start + blocks == next.start) {
// extend the next entry
next.start -= blocks;
next.blocks += blocks;
return;
}
freeSpace.add(x);
}
private long getPos(int block) {
return (long) block * (long) blockSize;
}
private int getBlock(long pos) {
return (int) (pos / blockSize);
}
private int getBlockCount(int length) {
if (length <= 0) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_INTERNAL, "Free space invalid length");
}
return MathUtils.roundUpInt(length, blockSize) / blockSize;
}
@Override
public String toString() {
return freeSpace.toString();
}
/**
* A range of free blocks.
*/
private static final class BlockRange implements Comparable<BlockRange> {
/**
* The starting point (the block number).
*/
public int start;
/**
* The length, in blocks.
*/
public int blocks;
public BlockRange(int start, int blocks) {
this.start = start;
this.blocks = blocks;
}
@Override
public int compareTo(BlockRange o) {
return start < o.start ? -1 : start > o.start ? 1 : 0;
}
@Override
public String toString() {
if (blocks + start == Integer.MAX_VALUE) {
return start + "-";
}
return start + "-" + (start + blocks - 1);
}
}
}
h2/src/test/org/h2/test/store/TestFreeSpace.java 0 → 100644
浏览文件 @ 53fd89a8
/*
* Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License, Version
* 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html). Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.util.Random;
import org.h2.mvstore.FreeSpaceBitSet;
import org.h2.test.TestBase;
import org.h2.util.Utils;
/**
* Tests the free space list.
*/
public class TestFreeSpace extends TestBase {
/**
* Run just this test.
*
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
testMemoryUsage();
testPerformance();
}
@Override
public void test() throws Exception {
testSimple();
testRandomized();
}
private static void testPerformance() {
for (int i = 0; i < 10; i++) {
long t = System.currentTimeMillis();
FreeSpaceBitSet f = new FreeSpaceBitSet(0, 4096);
// 75 ms
// FreeSpaceList f = new FreeSpaceList(0, 4096);
// 13868 ms
// FreeSpaceTree f = new FreeSpaceTree(0, 4096);
// 56 ms
for (int j = 0; j < 100000; j++) {
f.markUsed(j * 2 * 4096, 4096);
}
for (int j = 0; j < 100000; j++) {
f.free(j * 2 * 4096, 4096);
}
for (int j = 0; j < 100000; j++) {
f.allocate(4096 * 2);
}
System.out.println(System.currentTimeMillis() - t);
}
}
private static void testMemoryUsage() {
// 16 GB file size
long size = 16L * 1024 * 1024 * 1024;
System.gc();
System.gc();
long first = Utils.getMemoryUsed();
FreeSpaceBitSet f = new FreeSpaceBitSet(0, 4096);
// 512 KB
// FreeSpaceTree f = new FreeSpaceTree(0, 4096);
// 64 MB
// FreeSpaceList f = new FreeSpaceList(0, 4096);
// too slow
for (long j = size; j > 0; j -= 4 * 4096) {
f.markUsed(j, 4096);
}
System.gc();
System.gc();
long mem = Utils.getMemoryUsed() - first;
System.out.println("Memory used: " + mem);
System.out.println("f: " + f.toString().length());
}
private void testSimple() {
FreeSpaceBitSet f1 = new FreeSpaceBitSet(2, 1024);
FreeSpaceList f2 = new FreeSpaceList(2, 1024);
FreeSpaceTree f3 = new FreeSpaceTree(2, 1024);
assertEquals(f1.toString(), f2.toString());
assertEquals(f1.toString(), f3.toString());
assertEquals(2 * 1024, f1.allocate(10240));
assertEquals(2 * 1024, f2.allocate(10240));
assertEquals(2 * 1024, f3.allocate(10240));
assertEquals(f1.toString(), f2.toString());
assertEquals(f1.toString(), f3.toString());
f1.markUsed(20480, 1024);
f2.markUsed(20480, 1024);
f3.markUsed(20480, 1024);
assertEquals(f1.toString(), f2.toString());
assertEquals(f1.toString(), f3.toString());
}
private void testRandomized() {
FreeSpaceBitSet f1 = new FreeSpaceBitSet(2, 8);
FreeSpaceList f2 = new FreeSpaceList(2, 8);
Random r = new Random(1);
StringBuilder log = new StringBuilder();
for (int i = 0; i < 100000; i++) {
long pos = r.nextInt(1024);
int length = 1 + r.nextInt(8 * 128);
switch (r.nextInt(3)) {
case 0: {
log.append("allocate(" + length + ");\n");
long a = f1.allocate(length);
long b = f2.allocate(length);
assertEquals(a, b);
break;
}
case 1:
if (f1.isUsed(pos, length)) {
log.append("free(" + pos + ", " + length + ");\n");
f1.free(pos, length);
f2.free(pos, length);
}
break;
case 2:
if (f1.isFree(pos, length)) {
log.append("markUsed(" + pos + ", " + length + ");\n");
f1.markUsed(pos, length);
f2.markUsed(pos, length);
}
break;
}
assertEquals(f1.toString(), f2.toString());
}
}
}
h2/src/test/org/h2/test/store/TestMVTableEngine.java
浏览文件 @ 53fd89a8
......@@ -46,6 +46,7 @@ public class TestMVTableEngine extends TestBase {
@Override
public void test() throws Exception {
// testShrinkDatabaseFile();
testTwoPhaseCommit();
testRecover();
testSeparateKey();
testRollback();
......@@ -88,6 +89,33 @@ public class TestMVTableEngine extends TestBase {
}
}
private void testTwoPhaseCommit() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
Connection conn;
Statement stat;
String url = "mvstore;MV_STORE=TRUE";
url = getURL(url, true);
conn = getConnection(url);
stat = conn.createStatement();
stat.execute("create table test(id int primary key, name varchar)");
stat.execute("set write_delay 0");
conn.setAutoCommit(false);
stat.execute("insert into test values(1, 'Hello')");
stat.execute("prepare commit test_tx");
stat.execute("shutdown immediately");
JdbcUtils.closeSilently(conn);
conn = getConnection(url);
stat = conn.createStatement();
ResultSet rs;
rs = stat.executeQuery("select * from information_schema.in_doubt");
assertTrue(rs.next());
stat.execute("commit transaction test_tx");
rs = stat.executeQuery("select * from test");
assertTrue(rs.next());
conn.close();
}
private void testRecover() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
Connection conn;
......
h2/src/test/org/h2/test/store/TestTransactionStore.java
浏览文件 @ 53fd89a8
......@@ -45,7 +45,7 @@ public class TestTransactionStore extends TestBase {
@Override
public void test() throws Exception {
FileUtils.createDirectories(getBaseDir());
testStopWhileCommitting();
// testStopWhileCommitting();
testGetModifiedMaps();
testKeyIterator();
testMultiStatement();
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论