Skip to content

H
h2database
提交 a633f7f3 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVStore: compact (including MVTableEngine)

上级 1bb9cb16
显示空白字符变更
内嵌 并排
正在显示 包含 196 行增加 和 75 行删除
h2/src/main/org/h2/engine/Database.java
浏览文件 @ a633f7f3
......@@ -17,7 +17,6 @@ import java.util.Set;
import java.util.StringTokenizer;
import org.h2.api.DatabaseEventListener;
import org.h2.api.JavaObjectSerializer;
import org.h2.command.CommandInterface;
import org.h2.command.ddl.CreateTableData;
import org.h2.command.dml.SetTypes;
import org.h2.constant.DbSettings;
......@@ -1255,7 +1254,7 @@ public class Database implements DataHandler {
if (mvStore != null) {
if (!readOnly) {
if (compactMode != 0) {
mvStore.compact(compactMode == CommandInterface.SHUTDOWN_DEFRAG);
mvStore.compact();
}
}
mvStore.close();
......@@ -2093,7 +2092,9 @@ public class Database implements DataHandler {
}
public QueryStatisticsData getQueryStatisticsData() {
if (queryStatistics) {
if (!queryStatistics) {
return null;
}
if (queryStatisticsData == null) {
synchronized (this) {
if (queryStatisticsData == null) {
......@@ -2102,9 +2103,6 @@ public class Database implements DataHandler {
}
}
return queryStatisticsData;
} else {
return null;
}
}
/**
......
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ a633f7f3
......@@ -123,6 +123,10 @@ MVStore:
- temporary file storage
- simple rollback method (rollback to last committed version)
- MVMap to implement SortedMap, then NavigableMap
- add abstraction ChunkStore,
with free space handling, retention time / flush,
possibly one file per chunk to support SSD trim on file system level,
and free up memory for off-heap storage)
*/
......@@ -850,6 +854,9 @@ public class MVStore {
* @return the new version (incremented if there were changes)
*/
public long store() {
;
new Exception().printStackTrace(System.out);
checkOpen();
return store(false);
}
......@@ -1101,6 +1108,10 @@ public class MVStore {
Map<Integer, Chunk> freed = freedPages.get(v);
for (Chunk f : freed.values()) {
Chunk c = chunks.get(f.id);
if (c == null) {
// already removed
continue;
}
c.maxLengthLive += f.maxLengthLive;
c.pageCountLive += f.pageCountLive;
if (c.pageCountLive < 0) {
......@@ -1217,6 +1228,8 @@ public class MVStore {
/**
* Compact the store by moving all chunks next to each other, if there is
* free space between chunks. This might temporarily double the file size.
* Chunks are overwritten irrespective of the current retention time. Before
* overwriting chunks and before resizing the file, syncFile() is called.
*
* @return if anything was written
*/
......@@ -1226,6 +1239,23 @@ public class MVStore {
// nothing to do
return false;
}
int oldRetentionTime = retentionTime;
retentionTime = 0;
long time = getTime();
ArrayList<Chunk> free = New.arrayList();
for (Chunk c : chunks.values()) {
if (c.maxLengthLive == 0) {
if (canOverwriteChunk(c, time)) {
free.add(c);
}
}
}
for (Chunk c : free) {
chunks.remove(c.id);
meta.remove("chunk." + c.id);
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
freeSpace.free(c.start, length);
}
if (freeSpace.getFillRate() == 100) {
return false;
}
......@@ -1266,12 +1296,15 @@ public class MVStore {
reuseSpace = false;
store();
syncFile();
// now re-use the empty space
reuseSpace = true;
for (Chunk c : move) {
ByteBuffer buff = getWriteBuffer();
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE;
buff = DataUtils.ensureCapacity(buff, length);
buff.limit(length);
DataUtils.readFully(file, c.start, buff);
long pos = freeSpace.allocate(length);
freeSpace.free(c.start, length);
......@@ -1283,7 +1316,6 @@ public class MVStore {
buff.put(header);
// fill the header with zeroes
buff.put(new byte[BLOCK_SIZE - header.length]);
buff.limit(length);
buff.position(0);
fileWriteCount++;
DataUtils.writeFully(file, pos, buff);
......@@ -1294,10 +1326,29 @@ public class MVStore {
// update the metadata (within the file)
store();
syncFile();
shrinkFileIfPossible(0);
reuseSpace = oldReuse;
retentionTime = oldRetentionTime;
return true;
}
/**
* Force all changes to be written to the file. The default implementation
* calls FileChannel.force(true).
*/
public void syncFile() {
try {
file.force(true);
} catch (IOException e) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_WRITING_FAILED,
"Could not sync file {0}", fileName, e);
}
}
/**
* Try to reduce the file size by re-writing partially full chunks. Chunks
* with a low number of live items are re-written. If the current fill rate
......
h2/src/main/org/h2/mvstore/db/MVTableEngine.java
浏览文件 @ a633f7f3
......@@ -13,7 +13,6 @@ import java.util.ArrayList;
import java.util.List;
import org.h2.api.TableEngine;
import org.h2.command.CommandInterface;
import org.h2.command.ddl.CreateTableData;
import org.h2.constant.ErrorCode;
import org.h2.engine.Constants;
......@@ -169,12 +168,14 @@ public class MVTableEngine implements TableEngine {
* Store all pending changes.
*/
public void store() {
if (!store.isReadOnly()) {
if (store.isReadOnly()) {
return;
}
int todo;
store.commit();
store.compact(50);
store.store();
}
}
/**
* Close the store, without persisting changes.
......@@ -261,14 +262,11 @@ public class MVTableEngine implements TableEngine {
}
}
public void compact(boolean defrag) {
sync();
store.setRetentionTime(0);
public void compact() {
while (store.compact(90)) {
System.out.println("compact");
sync();
// repeat
}
System.out.println("compact done");
store.compactMoveChunks();
}
}
......
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ a633f7f3
......@@ -249,6 +249,11 @@ java org.h2.test.TestAll timer
*/
public boolean mvStore;
/**
* Whether the test is running with code coverage.
*/
public boolean coverage;
/**
* If code coverage is enabled.
*/
......@@ -506,6 +511,8 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
*/
private void runTests() throws SQLException {
coverage = isCoverage();
{}
mvStore = false;
......@@ -570,6 +577,20 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
test();
}
/**
* Check whether this method is running with "Emma" code coverage turned on.
*
* @return true if the stack trace contains ".emma."
*/
private static boolean isCoverage() {
for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
if (e.toString().indexOf(".emma.") >= 0) {
return true;
}
}
return false;
}
/**
* Run all tests with the current settings.
*/
......
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ a633f7f3
......@@ -47,7 +47,7 @@ public class TestMVStore extends TestBase {
public void test() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
FileUtils.createDirectories(getBaseDir());
testCompactFully();
testBackgroundExceptionListener();
testOldVersion();
testAtomicOperations();
......@@ -92,6 +92,30 @@ public class TestMVStore extends TestBase {
testLargerThan2G();
}
private void testCompactFully() throws Exception {
String fileName = getBaseDir() + "/testCompactFully.h3";
FileUtils.delete(fileName);
MVStore s = new MVStore.Builder().
fileName(fileName).
open();
MVMap<Integer, String> m;
for (int i = 0; i < 10; i++) {
m = s.openMap("data" + i);
m.put(0, "Hello World");
s.store();
}
for (int i = 0; i < 10; i += 2) {
m = s.openMap("data" + i);
m.removeMap();
s.store();
}
long sizeOld = s.getFile().size();
s.compactMoveChunks();
long sizeNew = s.getFile().size();
assertTrue("old: " + sizeOld + " new: " + sizeNew, sizeNew < sizeOld);
s.close();
}
private void testBackgroundExceptionListener() throws Exception {
String fileName = getBaseDir() + "/testBackgroundExceptionListener.h3";
FileUtils.delete(fileName);
......
h2/src/test/org/h2/test/store/TestMVStoreBenchmark.java
浏览文件 @ a633f7f3
......@@ -39,6 +39,13 @@ public class TestMVStoreBenchmark extends TestBase {
if (!config.big) {
return;
}
if (config.coverage || config.codeCoverage) {
// run only when _not_ using a code coverage tool,
// because the tool might instrument our code but not
// java.util.*
return;
}
testPerformanceComparison();
testMemoryUsageComparison();
}
......@@ -147,9 +154,7 @@ public class TestMVStoreBenchmark extends TestBase {
String msg = "mv " + mv + " tree " + tree + " hash " + hash;
assertTrue(msg, hash < tree);
assertTrue(msg, hash < mv);
int todo;
// check only when _not_ using a code coverage tool
// assertTrue(msg, mv < tree);
assertTrue(msg, mv < tree);
}
private long testPerformance(Map<Integer, String> map, int size) {
......
h2/src/test/org/h2/test/store/TestMVTableEngine.java
浏览文件 @ a633f7f3
......@@ -8,7 +8,6 @@ package org.h2.test.store;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DriverManager;
......@@ -22,7 +21,8 @@ import org.h2.constant.ErrorCode;
import org.h2.engine.Constants;
import org.h2.engine.Database;
import org.h2.jdbc.JdbcConnection;
import org.h2.mvstore.MVStoreTool;
import org.h2.mvstore.MVStore;
import org.h2.mvstore.db.TransactionStore;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.tools.DeleteDbFiles;
......@@ -47,8 +47,9 @@ public class TestMVTableEngine extends TestBase {
@Override
public void test() throws Exception {
;;
// testTransactionLogUsuallyNotStored();
testShrinkDatabaseFile();
testTransactionLogUsuallyNotStored();
testTwoPhaseCommit();
testRecover();
testSeparateKey();
......@@ -68,6 +69,35 @@ public class TestMVTableEngine extends TestBase {
testSimple();
}
private void testTransactionLogUsuallyNotStored() throws Exception {
int todo;
FileUtils.deleteRecursive(getBaseDir(), true);
Connection conn;
Statement stat;
String url = "mvstore;MV_STORE=TRUE";
url = getURL(url, true);
conn = getConnection(url);
stat = conn.createStatement();
stat.execute("create table test(id identity, name varchar)");
conn.setAutoCommit(false);
for (int j = 0; j < 100; j++) {
for (int i = 0; i < 100; i++) {
stat.execute("insert into test(name) values('Hello World')");
}
conn.commit();
}
stat.execute("shutdown immediately");
JdbcUtils.closeSilently(conn);
String file = getBaseDir() + "/mvstore" + Constants.SUFFIX_MV_FILE;
MVStore store = MVStore.open(file);
TransactionStore t = new TransactionStore(store);
assertEquals(0, t.getOpenTransactions().size());
store.close();
}
private void testShrinkDatabaseFile() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
String dbName = "mvstore" +
......@@ -104,21 +134,15 @@ public class TestMVTableEngine extends TestBase {
fail(i + " size: " + size + " max: " + maxSize);
}
}
int todo;
// conn = getConnection(dbName);
// stat = conn.createStatement();
// stat.execute("shutdown compact");
// conn.close();
//
// MVStoreTool.dump(getBaseDir() + "/mvstore.mv.db", new PrintWriter(System.out));
//
// long size = FileUtils.size(getBaseDir() + "/mvstore"
// + Constants.SUFFIX_MV_FILE);
// assertTrue(size < 16 * 1024);
}
private void testTransactionLogUsuallyNotStored() {
int todo;
long sizeOld = FileUtils.size(getBaseDir() + "/mvstore"
+ Constants.SUFFIX_MV_FILE);
conn = getConnection(dbName);
stat = conn.createStatement();
stat.execute("shutdown compact");
conn.close();
long sizeNew = FileUtils.size(getBaseDir() + "/mvstore"
+ Constants.SUFFIX_MV_FILE);
assertTrue(sizeNew < sizeOld);
}
private void testTwoPhaseCommit() throws Exception {
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论