Skip to content

H
h2database
提交 a633f7f3 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVStore: compact (including MVTableEngine)

上级 1bb9cb16
隐藏空白字符变更
内嵌 并排
正在显示 包含 196 行增加 和 75 行删除
h2/src/main/org/h2/engine/Database.java
浏览文件 @ a633f7f3
......@@ -17,7 +17,6 @@ import java.util.Set;
import java.util.StringTokenizer;
import org.h2.api.DatabaseEventListener;
import org.h2.api.JavaObjectSerializer;
import org.h2.command.CommandInterface;
import org.h2.command.ddl.CreateTableData;
import org.h2.command.dml.SetTypes;
import org.h2.constant.DbSettings;
......@@ -1255,7 +1254,7 @@ public class Database implements DataHandler {
if (mvStore != null) {
if (!readOnly) {
if (compactMode != 0) {
mvStore.compact(compactMode == CommandInterface.SHUTDOWN_DEFRAG);
mvStore.compact();
}
}
mvStore.close();
......@@ -1774,11 +1773,11 @@ public class Database implements DataHandler {
mvStore.setWriteDelay(value);
}
}
public int getRetentionTime() {
return retentionTime;
}
public void setRetentionTime(int value) {
retentionTime = value;
if (mvStore != null) {
......@@ -2093,18 +2092,17 @@ public class Database implements DataHandler {
}
public QueryStatisticsData getQueryStatisticsData() {
if (queryStatistics) {
if (queryStatisticsData == null) {
synchronized (this) {
if (queryStatisticsData == null) {
queryStatisticsData = new QueryStatisticsData();
}
if (!queryStatistics) {
return null;
}
if (queryStatisticsData == null) {
synchronized (this) {
if (queryStatisticsData == null) {
queryStatisticsData = new QueryStatisticsData();
}
}
return queryStatisticsData;
} else {
return null;
}
return queryStatisticsData;
}
/**
......
h2/src/main/org/h2/mvstore/FreeSpaceBitSet.java
浏览文件 @ a633f7f3
......@@ -144,7 +144,7 @@ public class FreeSpaceBitSet {
/**
* Get the fill rate of the space in percent. The value 0 means the space is
* completely free, and 100 means it is completely full.
*
*
* @return the fill rate (0 - 100)
*/
public int getFillRate() {
......@@ -159,10 +159,10 @@ public class FreeSpaceBitSet {
}
return Math.max(1, (int) (100L * count / total));
}
/**
* Get the position of the first free space.
*
*
* @return the position.
*/
public long getFirstFree() {
......
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ a633f7f3
......@@ -123,6 +123,10 @@ MVStore:
- temporary file storage
- simple rollback method (rollback to last committed version)
- MVMap to implement SortedMap, then NavigableMap
- add abstraction ChunkStore,
with free space handling, retention time / flush,
possibly one file per chunk to support SSD trim on file system level,
and free up memory for off-heap storage)
*/
......@@ -850,6 +854,9 @@ public class MVStore {
* @return the new version (incremented if there were changes)
*/
public long store() {
;
new Exception().printStackTrace(System.out);
checkOpen();
return store(false);
}
......@@ -1036,11 +1043,11 @@ public class MVStore {
}
return version;
}
/**
* Get a buffer for writing. This caller must synchronize on the store
* before calling the method and until after using the buffer.
*
*
* @return the buffer
*/
private ByteBuffer getWriteBuffer() {
......@@ -1057,7 +1064,7 @@ public class MVStore {
/**
* Release a buffer for writing. This caller must synchronize on the store
* before calling the method and until after using the buffer.
*
*
* @param buff the buffer than can be re-used
*/
private void releaseWriteBuffer(ByteBuffer buff) {
......@@ -1101,6 +1108,10 @@ public class MVStore {
Map<Integer, Chunk> freed = freedPages.get(v);
for (Chunk f : freed.values()) {
Chunk c = chunks.get(f.id);
if (c == null) {
// already removed
continue;
}
c.maxLengthLive += f.maxLengthLive;
c.pageCountLive += f.pageCountLive;
if (c.pageCountLive < 0) {
......@@ -1213,10 +1224,12 @@ public class MVStore {
buff.rewind();
return Chunk.fromHeader(buff, start);
}
/**
* Compact the store by moving all chunks next to each other, if there is
* free space between chunks. This might temporarily double the file size.
* Chunks are overwritten irrespective of the current retention time. Before
* overwriting chunks and before resizing the file, syncFile() is called.
*
* @return if anything was written
*/
......@@ -1226,6 +1239,23 @@ public class MVStore {
// nothing to do
return false;
}
int oldRetentionTime = retentionTime;
retentionTime = 0;
long time = getTime();
ArrayList<Chunk> free = New.arrayList();
for (Chunk c : chunks.values()) {
if (c.maxLengthLive == 0) {
if (canOverwriteChunk(c, time)) {
free.add(c);
}
}
}
for (Chunk c : free) {
chunks.remove(c.id);
meta.remove("chunk." + c.id);
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
freeSpace.free(c.start, length);
}
if (freeSpace.getFillRate() == 100) {
return false;
}
......@@ -1266,12 +1296,15 @@ public class MVStore {
reuseSpace = false;
store();
syncFile();
// now re-use the empty space
reuseSpace = true;
for (Chunk c : move) {
ByteBuffer buff = getWriteBuffer();
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
buff = DataUtils.ensureCapacity(buff, length);
buff.limit(length);
DataUtils.readFully(file, c.start, buff);
long pos = freeSpace.allocate(length);
freeSpace.free(c.start, length);
......@@ -1283,7 +1316,6 @@ public class MVStore {
buff.put(header);
// fill the header with zeroes
buff.put(new byte[BLOCK_SIZE - header.length]);
buff.limit(length);
buff.position(0);
fileWriteCount++;
DataUtils.writeFully(file, pos, buff);
......@@ -1291,12 +1323,31 @@ public class MVStore {
releaseWriteBuffer(buff);
meta.put("chunk." + c.id, c.asString());
}
// update the metadata (within the file)
store();
syncFile();
shrinkFileIfPossible(0);
reuseSpace = oldReuse;
retentionTime = oldRetentionTime;
return true;
}
/**
* Force all changes to be written to the file. The default implementation
* calls FileChannel.force(true).
*/
public void syncFile() {
try {
file.force(true);
} catch (IOException e) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_WRITING_FAILED,
"Could not sync file {0}", fileName, e);
}
}
/**
* Try to reduce the file size by re-writing partially full chunks. Chunks
......@@ -1306,7 +1357,7 @@ public class MVStore {
* Only data of open maps can be moved. For maps that are not open, the old
* chunk is still referenced. Therefore, it is recommended to open all maps
* before calling this method.
*
*
* @param fillRate the minimum percentage of live entries
* @return if anything was written
*/
......@@ -1327,7 +1378,7 @@ public class MVStore {
}
// the fill rate of all chunks combined
int totalChunkFillRate = (int) (100 * maxLengthLiveSum / maxLengthSum);
if (totalChunkFillRate > fillRate) {
return false;
}
......
h2/src/main/org/h2/mvstore/db/MVTableEngine.java
浏览文件 @ a633f7f3
......@@ -13,7 +13,6 @@ import java.util.ArrayList;
import java.util.List;
import org.h2.api.TableEngine;
import org.h2.command.CommandInterface;
import org.h2.command.ddl.CreateTableData;
import org.h2.constant.ErrorCode;
import org.h2.engine.Constants;
......@@ -169,11 +168,13 @@ public class MVTableEngine implements TableEngine {
* Store all pending changes.
*/
public void store() {
if (!store.isReadOnly()) {
store.commit();
store.compact(50);
store.store();
if (store.isReadOnly()) {
return;
}
int todo;
store.commit();
store.compact(50);
store.store();
}
/**
......@@ -248,7 +249,7 @@ public class MVTableEngine implements TableEngine {
public InputStream getInputStream() {
return new FileChannelInputStream(store.getFile(), false);
}
/**
* Force the changes to disk.
*/
......@@ -261,14 +262,11 @@ public class MVTableEngine implements TableEngine {
}
}
public void compact(boolean defrag) {
sync();
store.setRetentionTime(0);
public void compact() {
while (store.compact(90)) {
System.out.println("compact");
sync();
// repeat
}
System.out.println("compact done");
store.compactMoveChunks();
}
}
......
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ a633f7f3
......@@ -248,6 +248,11 @@ java org.h2.test.TestAll timer
* Whether to use the MVStore.
*/
public boolean mvStore;
/**
* Whether the test is running with code coverage.
*/
public boolean coverage;
/**
* If code coverage is enabled.
......@@ -505,6 +510,8 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
* Run the tests with a number of different settings.
*/
private void runTests() throws SQLException {
coverage = isCoverage();
{}
mvStore = false;
......@@ -569,6 +576,20 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
memory = true;
test();
}
/**
* Check whether this method is running with "Emma" code coverage turned on.
*
* @return true if the stack trace contains ".emma."
*/
private static boolean isCoverage() {
for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
if (e.toString().indexOf(".emma.") >= 0) {
return true;
}
}
return false;
}
/**
* Run all tests with the current settings.
......
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ a633f7f3
......@@ -47,7 +47,7 @@ public class TestMVStore extends TestBase {
public void test() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
FileUtils.createDirectories(getBaseDir());
testCompactFully();
testBackgroundExceptionListener();
testOldVersion();
testAtomicOperations();
......@@ -92,6 +92,30 @@ public class TestMVStore extends TestBase {
testLargerThan2G();
}
private void testCompactFully() throws Exception {
String fileName = getBaseDir() + "/testCompactFully.h3";
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).
open();
MVMap<Integer, String> m;
for (int i = 0; i < 10; i++) {
m = s.openMap("data" + i);
m.put(0, "Hello World");
s.store();
}
for (int i = 0; i < 10; i += 2) {
m = s.openMap("data" + i);
m.removeMap();
s.store();
}
long sizeOld = s.getFile().size();
s.compactMoveChunks();
long sizeNew = s.getFile().size();
assertTrue("old: " + sizeOld + " new: " + sizeNew, sizeNew < sizeOld);
s.close();
}
private void testBackgroundExceptionListener() throws Exception {
String fileName = getBaseDir() + "/testBackgroundExceptionListener.h3";
FileUtils.delete(fileName);
......@@ -1221,7 +1245,7 @@ public class TestMVStore extends TestBase {
}
s.close();
}
private void testCompactMapNotOpen() {
String fileName = getBaseDir() + "/testCompactNotOpen.h3";
FileUtils.delete(fileName);
......@@ -1235,10 +1259,10 @@ public class TestMVStore extends TestBase {
s.store();
}
s.close();
s = openStore(fileName);
s.setRetentionTime(0);
Map<String, String> meta = s.getMetaMap();
int chunkCount1 = 0;
for (String k : meta.keySet()) {
......
h2/src/test/org/h2/test/store/TestMVStoreBenchmark.java
浏览文件 @ a633f7f3
......@@ -39,17 +39,24 @@ public class TestMVStoreBenchmark extends TestBase {
if (!config.big) {
return;
}
if (config.coverage || config.codeCoverage) {
// run only when _not_ using a code coverage tool,
// because the tool might instrument our code but not
// java.util.*
return;
}
testPerformanceComparison();
testMemoryUsageComparison();
}
private void testMemoryUsageComparison() {
long[] mem;
long hash, tree, mv;
String msg;
mem = getMemoryUsed(10000, 10);
hash = mem[0];
hash = mem[0];
tree = mem[1];
mv = mem[2];
msg = Arrays.toString(mem);
......@@ -65,12 +72,12 @@ public class TestMVStoreBenchmark extends TestBase {
assertTrue(msg, mv < tree);
}
private static long[] getMemoryUsed(int count, int size) {
long hash, tree, mv;
ArrayList<Map<Integer, String>> mapList;
long mem;
mapList = New.arrayList();
mem = getMemory();
for (int i = 0; i < count; i++) {
......@@ -79,7 +86,7 @@ public class TestMVStoreBenchmark extends TestBase {
addEntries(mapList, size);
hash = getMemory() - mem;
mapList.size();
mapList = New.arrayList();
mem = getMemory();
for (int i = 0; i < count; i++) {
......@@ -91,7 +98,7 @@ public class TestMVStoreBenchmark extends TestBase {
mapList = New.arrayList();
mem = getMemory();
MVStore store = MVStore.open(null);
MVStore store = MVStore.open(null);
for (int i = 0; i < count; i++) {
Map<Integer, String> map = store.openMap("t" + i);
mapList.add(map);
......@@ -99,10 +106,10 @@ public class TestMVStoreBenchmark extends TestBase {
addEntries(mapList, size);
mv = getMemory() - mem;
mapList.size();
return new long[]{hash, tree, mv};
}
private static void addEntries(List<Map<Integer, String>> mapList, int size) {
for (Map<Integer, String> map : mapList) {
for (int i = 0; i < size; i++) {
......@@ -110,7 +117,7 @@ public class TestMVStoreBenchmark extends TestBase {
}
}
}
static long getMemory() {
try {
LinkedList<byte[]> list = new LinkedList<byte[]>();
......@@ -130,7 +137,7 @@ public class TestMVStoreBenchmark extends TestBase {
}
return getMemoryUsedBytes();
}
private void testPerformanceComparison() {
if (!config.big) {
return;
......@@ -144,14 +151,12 @@ public class TestMVStoreBenchmark extends TestBase {
MVStore store = MVStore.open(null);
map = store.openMap("test");
long mv = testPerformance(map, size);
String msg = "mv " + mv + " tree " + tree + " hash " + hash;
String msg = "mv " + mv + " tree " + tree + " hash " + hash;
assertTrue(msg, hash < tree);
assertTrue(msg, hash < mv);
int todo;
// check only when _not_ using a code coverage tool
// assertTrue(msg, mv < tree);
assertTrue(msg, mv < tree);
}
private long testPerformance(Map<Integer, String> map, int size) {
System.gc();
long time = 0;
......
h2/src/test/org/h2/test/store/TestMVTableEngine.java
浏览文件 @ a633f7f3
......@@ -8,7 +8,6 @@ package org.h2.test.store;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DriverManager;
......@@ -22,7 +21,8 @@ import org.h2.constant.ErrorCode;
import org.h2.engine.Constants;
import org.h2.engine.Database;
import org.h2.jdbc.JdbcConnection;
import org.h2.mvstore.MVStoreTool;
import org.h2.mvstore.MVStore;
import org.h2.mvstore.db.TransactionStore;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.tools.DeleteDbFiles;
......@@ -47,8 +47,9 @@ public class TestMVTableEngine extends TestBase {
@Override
public void test() throws Exception {
;;
// testTransactionLogUsuallyNotStored();
testShrinkDatabaseFile();
testTransactionLogUsuallyNotStored();
testTwoPhaseCommit();
testRecover();
testSeparateKey();
......@@ -67,6 +68,35 @@ public class TestMVTableEngine extends TestBase {
testLocking();
testSimple();
}
private void testTransactionLogUsuallyNotStored() throws Exception {
int todo;
FileUtils.deleteRecursive(getBaseDir(), true);
Connection conn;
Statement stat;
String url = "mvstore;MV_STORE=TRUE";
url = getURL(url, true);
conn = getConnection(url);
stat = conn.createStatement();
stat.execute("create table test(id identity, name varchar)");
conn.setAutoCommit(false);
for (int j = 0; j < 100; j++) {
for (int i = 0; i < 100; i++) {
stat.execute("insert into test(name) values('Hello World')");
}
conn.commit();
}
stat.execute("shutdown immediately");
JdbcUtils.closeSilently(conn);
String file = getBaseDir() + "/mvstore" + Constants.SUFFIX_MV_FILE;
MVStore store = MVStore.open(file);
TransactionStore t = new TransactionStore(store);
assertEquals(0, t.getOpenTransactions().size());
store.close();
}
private void testShrinkDatabaseFile() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
......@@ -88,7 +118,7 @@ public class TestMVTableEngine extends TestBase {
retentionTime = 0;
}
ResultSet rs = stat.executeQuery(
"select value from information_schema.settings " +
"select value from information_schema.settings " +
"where name='RETENTION_TIME'");
assertTrue(rs.next());
assertEquals(retentionTime, rs.getInt(1));
......@@ -104,21 +134,15 @@ public class TestMVTableEngine extends TestBase {
fail(i + " size: " + size + " max: " + maxSize);
}
}
int todo;
// conn = getConnection(dbName);
// stat = conn.createStatement();
// stat.execute("shutdown compact");
// conn.close();
//
// MVStoreTool.dump(getBaseDir() + "/mvstore.mv.db", new PrintWriter(System.out));
//
// long size = FileUtils.size(getBaseDir() + "/mvstore"
// + Constants.SUFFIX_MV_FILE);
// assertTrue(size < 16 * 1024);
}
private void testTransactionLogUsuallyNotStored() {
int todo;
long sizeOld = FileUtils.size(getBaseDir() + "/mvstore"
+ Constants.SUFFIX_MV_FILE);
conn = getConnection(dbName);
stat = conn.createStatement();
stat.execute("shutdown compact");
conn.close();
long sizeNew = FileUtils.size(getBaseDir() + "/mvstore"
+ Constants.SUFFIX_MV_FILE);
assertTrue(sizeNew < sizeOld);
}
private void testTwoPhaseCommit() throws Exception {
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论