Skip to content

H
h2database
提交 cbab3234 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVStore: tool to quickly compact a store.

上级 5da4e45c
隐藏空白字符变更
内嵌 并排
正在显示 包含 120 行增加 和 157 行删除
h2/src/main/org/h2/mvstore/CursorPos.java
浏览文件 @ cbab3234
......@@ -13,7 +13,7 @@ public class CursorPos {
/**
* The current page.
*/
public final Page page;
public Page page;
/**
* The current index.
......
h2/src/main/org/h2/mvstore/MVMap.java
浏览文件 @ cbab3234
......@@ -995,7 +995,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
return;
}
Page last = oldRoots.peekLast();
// TODO why is this?
// TODO why is this? (maybe not needed)
;
oldest--;
while (true) {
......@@ -1257,23 +1257,32 @@ public class MVMap<K, V> extends AbstractMap<K, V>
* @param sourceMap the source map
*/
void copyFrom(MVMap<K, V> sourceMap) {
; // TODO work in progress
root = copy(sourceMap.root, null);
beforeWrite();
newRoot(copy(sourceMap.root, null));
afterWrite();
}
private Page copy(Page source, CursorPos parent) {
Page target = Page.create(this, writeVersion, source);
for (CursorPos p = parent; p != null; p = p.parent) {
p.page.setChild(p.index, target);
}
if (!target.isLeaf()) {
if (target.isLeaf()) {
Page child = target;
for (CursorPos p = parent; p != null; p = p.parent) {
p.page.setChild(p.index, child);
p.page = copyOnWrite(p.page, writeVersion);
child = p.page;
if (p.parent == null) {
newRoot(p.page);
afterWrite();
beforeWrite();
}
}
} else {
CursorPos pos = new CursorPos(target, 0, parent);
target = copyOnWrite(target, writeVersion);
for (int i = 0; i < target.getChildPageCount(); i++) {
Page sourceChild = source.getChildPage(i);
pos.index = i;
copy(sourceChild, pos);
copy(source.getChildPage(i), pos);
}
target = pos.page;
}
return target;
}
......
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ cbab3234
......@@ -238,8 +238,8 @@ public class MVStore {
*/
private int autoCommitDelay;
private int autoCompactFillRate = 10;
private int autoCompactSize = 2 * 1024 * 1024;
private int autoCompactFillRate;
private long autoCompactLastFileOpCount;
/**
* Create and open the store.
......@@ -291,6 +291,10 @@ public class MVStore {
int kb = o == null ? 512 : (Integer) o;
// 19 KB memory is about 1 KB storage
autoCommitMemory = kb * 1024 * 19;
o = config.get("autoCompactFillRate");
autoCompactFillRate = o == null ? 80 : (Integer) o;
char[] encryptionKey = (char[]) config.get("encryptionKey");
try {
fileStore.open(fileName, readOnly, encryptionKey);
......@@ -1349,19 +1353,20 @@ public class MVStore {
* @return if anything was written
*/
public synchronized boolean compactMoveChunks() {
return compactMoveChunks(Long.MAX_VALUE);
return compactMoveChunks(100, Long.MAX_VALUE);
}
/**
* Compact the store by moving all chunks next to each other, if there is
* free space between chunks. This might temporarily double the file size.
* free space between chunks. This might temporarily increase the file size.
* Chunks are overwritten irrespective of the current retention time. Before
* overwriting chunks and before resizing the file, syncFile() is called.
*
* @param targetFillRate do nothing if the file store fill rate is higher than this
* @param moveSize the number of bytes to move
* @return if anything was written
*/
public synchronized boolean compactMoveChunks(long moveSize) {
public synchronized boolean compactMoveChunks(int targetFillRate, long moveSize) {
checkOpen();
if (lastChunk == null) {
// nothing to do
......@@ -1372,12 +1377,11 @@ public class MVStore {
try {
retentionTime = 0;
compactFreeUnusedChunks();
if (fileStore.getFillRate() == 100) {
if (fileStore.getFillRate() > targetFillRate) {
return false;
}
ArrayList<Chunk> move;
long start = fileStore.getFirstFree() / BLOCK_SIZE;
move = compactGetMoveBlocks(start, moveSize);
ArrayList<Chunk> move = compactGetMoveBlocks(start, moveSize);
compactMoveChunks(move);
} finally {
reuseSpace = oldReuse;
......@@ -1523,7 +1527,6 @@ public class MVStore {
* with a low number of live items are re-written.
* <p>
* If the current fill rate is higher than the target fill rate, nothing is
* done. If not at least a minimum amount of space can be saved, nothing is
* done.
* <p>
* Please note this method will not necessarily reduce the file size, as
......@@ -1534,7 +1537,7 @@ public class MVStore {
* before calling this method.
*
* @param targetFillRate the minimum percentage of live entries
* @param write the number of bytes to write
* @param write the minimum number of bytes to write
* @return if a chunk was re-written
*/
public synchronized boolean compact(int targetFillRate, int write) {
......@@ -1642,118 +1645,6 @@ public class MVStore {
return true;
}
/**
* Try to increase the fill rate by re-writing partially full chunks. Chunks
* with a low number of live items are re-written.
* <p>
* If the current fill rate is higher than the target fill rate, nothing is
* done. If not at least a minimum amount of space can be saved, nothing is
* done.
* <p>
* Please note this method will not necessarily reduce the file size, as
* empty chunks are not overwritten.
* <p>
* Only data of open maps can be moved. For maps that are not open, the old
* chunk is still referenced. Therefore, it is recommended to open all maps
* before calling this method.
*
* @param targetFillRate the minimum percentage of live entries
* @param minSaving the amount of saved space,
* which is also the size of the new chunk
* @return if a chunk was re-written
*/
public synchronized boolean compactOld(int targetFillRate, int minSaving) {
checkOpen();
if (lastChunk == null) {
// nothing to do
return false;
}
// calculate the fill rate
long maxLengthSum = 0;
long maxLengthLiveSum = 0;
for (Chunk c : chunks.values()) {
maxLengthSum += c.maxLen;
maxLengthLiveSum += c.maxLenLive;
}
// the fill rate of all chunks combined
if (maxLengthSum <= 0) {
// avoid division by 0
maxLengthSum = 1;
}
int fillRate = (int) (100 * maxLengthLiveSum / maxLengthSum);
if (fillRate >= targetFillRate) {
return false;
}
long time = getTime();
// the 'old' list contains the chunks we want to free up
ArrayList<Chunk> old = New.arrayList();
Chunk last = chunks.get(lastChunk.id);
for (Chunk c : chunks.values()) {
if (canOverwriteChunk(c, time)) {
long age = last.version - c.version + 1;
c.collectPriority = (int) (c.getFillRate() / age);
old.add(c);
}
}
if (old.size() == 0) {
return false;
}
// sort the list, so the first entry should be collected first
Collections.sort(old, new Comparator<Chunk>() {
@Override
public int compare(Chunk o1, Chunk o2) {
int comp = new Integer(o1.collectPriority).
compareTo(o2.collectPriority);
if (comp == 0) {
comp = new Long(o1.maxLenLive).
compareTo(o2.maxLenLive);
}
return comp;
}
});
// find out up to were in the old list we need to move
long saved = 0;
long totalSize = 0;
Chunk move = null;
for (Chunk c : old) {
long size = c.maxLen - c.maxLenLive;
totalSize += c.maxLenLive;
if (move != null) {
if (saved > minSaving && totalSize > minSaving) {
break;
}
}
saved += size;
move = c;
}
if (saved < minSaving) {
return false;
}
// remove the chunks we want to keep from this list
boolean remove = false;
for (Iterator<Chunk> it = old.iterator(); it.hasNext();) {
Chunk c = it.next();
if (move == c) {
remove = true;
} else if (remove) {
it.remove();
}
}
// iterate over all the pages in the old pages
for (Chunk c : old) {
copyLive(c);
}
commitAndSave();
return true;
}
private void copyLive(Chunk chunk) {
if (chunk.pageCountLive == 0) {
// remove this chunk in the next save operation
......@@ -2355,10 +2246,11 @@ public class MVStore {
}
/**
* Commit and save all changes, if there are any.
* Commit and save all changes, if there are any, and compact the store if
* needed.
*/
void commitInBackground() {
if (unsavedMemory == 0 || closed) {
void writeInBackground() {
if (closed) {
return;
}
......@@ -2379,9 +2271,26 @@ public class MVStore {
}
}
}
if (autoCompactSize > 0) {
if (autoCompactFillRate > 0) {
try {
compact(autoCompactFillRate, autoCompactSize);
// whether there were file read or write operations since
// the last time
boolean fileOps;
long fileOpCount = fileStore.getWriteCount() + fileStore.getReadCount();
if (autoCompactLastFileOpCount != fileOpCount) {
fileOps = true;
} else {
fileOps = false;
}
// use a lower fill rate if there were any file operations
int fillRate = fileOps ? autoCompactFillRate / 4 : autoCompactFillRate;
compact(fillRate, autoCommitMemory);
if (!fileOps) {
// if there were no file operations at all,
// compact the file by moving chunks
compactMoveChunks(autoCompactFillRate, autoCommitMemory);
}
autoCompactLastFileOpCount = fileStore.getWriteCount() + fileStore.getReadCount();
} catch (Exception e) {
if (backgroundExceptionHandler != null) {
backgroundExceptionHandler.uncaughtException(null, e);
......@@ -2570,7 +2479,7 @@ public class MVStore {
continue;
}
}
store.commitInBackground();
store.writeInBackground();
}
}
......@@ -2619,6 +2528,24 @@ public class MVStore {
public Builder autoCommitBufferSize(int kb) {
return set("autoCommitBufferSize", kb);
}
/**
* Set the auto-compact target fill rate. If the average fill rate (the
* percentage of the storage space that contains active data) of the
* chunks is lower, then the chunks with a low fill rate are re-written.
* Also, if the percentage of empty space between chunks is higher than
* this value, then chunks at the end of the file are moved. Compaction
* stops if the target fill rate is reached.
* <p>
* The default value is 80 (80%). The value 0 disables auto-compacting.
* <p>
*
* @param percent the target fill rate
* @return this
*/
public Builder autoCompactFillRate(int percent) {
return set("autoCompactFillRate", percent);
}
/**
* Use the following file name. If the file does not exist, it is
......
h2/src/main/org/h2/mvstore/MVStoreTool.java
浏览文件 @ cbab3234
......@@ -45,6 +45,12 @@ public class MVStoreTool {
} else if ("-info".equals(args[i])) {
String fileName = args[++i];
info(fileName, new PrintWriter(System.out));
} else if ("-compact".equals(args[i])) {
String fileName = args[++i];
compact(fileName, false);
} else if ("-compress".equals(args[i])) {
String fileName = args[++i];
compact(fileName, true);
}
}
}
......@@ -301,9 +307,10 @@ public class MVStoreTool {
* there.
*
* @param fileName the file name
* @param compress whether to compress the data
*/
public static void compress(String fileName) {
compress(fileName, fileName + ".new");
public static void compact(String fileName, boolean compress) {
compact(fileName, fileName + ".new", compress);
FileUtils.moveTo(fileName, fileName + ".old");
FileUtils.moveTo(fileName, fileName);
FileUtils.delete(fileName + ".old");
......@@ -315,12 +322,18 @@ public class MVStoreTool {
* @param sourceFileName the name of the source store
* @param targetFileName the name of the target store
*/
public static void compress(String sourceFileName, String targetFileName) {
public static void compact(String sourceFileName, String targetFileName, boolean compress) {
MVStore source = new MVStore.Builder().
fileName(sourceFileName).readOnly().open();
fileName(sourceFileName).
readOnly().
open();
FileUtils.delete(targetFileName);
MVStore target = new MVStore.Builder().
fileName(targetFileName).open();
MVStore.Builder b = new MVStore.Builder().
fileName(targetFileName);
if (compress) {
b.compress();
}
MVStore target = b.open();
MVMap<String, String> sourceMeta = source.getMetaMap();
MVMap<String, String> targetMeta = target.getMetaMap();
for (Entry<String, String> m : sourceMeta.entrySet()) {
......@@ -345,7 +358,6 @@ public class MVStoreTool {
MVMap<Object, Object> sourceMap = source.openMap(mapName, mp);
MVMap<Object, Object> targetMap = target.openMap(mapName, mp);
targetMap.copyFrom(sourceMap);
target.commit();
}
target.close();
source.close();
......@@ -364,7 +376,7 @@ public class MVStoreTool {
@Override
public int getMemory(Object obj) {
return obj == null ? 0 : ((byte[]) obj).length;
return obj == null ? 0 : ((byte[]) obj).length * 8;
}
@Override
......
h2/src/main/org/h2/mvstore/WriteBuffer.java
浏览文件 @ cbab3234
......@@ -304,7 +304,11 @@ public class WriteBuffer {
// grow at least 50% of the current size
grow = Math.max(temp.capacity() / 2, grow);
int newCapacity = temp.capacity() + grow;
buff = ByteBuffer.allocate(newCapacity);
try {
buff = ByteBuffer.allocate(newCapacity);
} catch (OutOfMemoryError e) {
throw new OutOfMemoryError("Capacity: " + newCapacity);
}
temp.flip();
buff.put(temp);
if (newCapacity <= MAX_REUSE_CAPACITY) {
......
h2/src/main/org/h2/mvstore/db/MVTableEngine.java
浏览文件 @ cbab3234
......@@ -317,9 +317,9 @@ public class MVTableEngine implements TableEngine {
store.compactMoveChunks();
} else {
long start = System.currentTimeMillis();
while (store.compact(99, 16 * 1024 * 1024)) {
while (store.compact(95, 16 * 1024 * 1024)) {
store.sync();
store.compactMoveChunks(16 * 1024 * 1024);
store.compactMoveChunks(95, 16 * 1024 * 1024);
long time = System.currentTimeMillis() - start;
if (time > maxCompactTime) {
break;
......@@ -341,8 +341,8 @@ public class MVTableEngine implements TableEngine {
if (!store.getFileStore().isReadOnly()) {
transactionStore.close();
if (maxCompactTime > 0) {
store.compact(99, 1 * 1024 * 1024);
store.compactMoveChunks(1 * 1024 * 1024);
store.compact(95, 1024 * 1024);
store.compactMoveChunks(95, 1024 * 1024);
}
}
store.close();
......
h2/src/test/org/h2/test/store/TestMVStoreTool.java
浏览文件 @ cbab3234
......@@ -32,12 +32,12 @@ public class TestMVStoreTool extends TestBase {
@Override
public void test() throws Exception {
; // TODO work in progress
// testCompress();
testCompress();
}
private void testCompress() {
String fileName = getBaseDir() + "/testCompress.h3";
FileUtils.createDirectory(getBaseDir());
FileUtils.delete(fileName);
// store with a very small page size, to make sure
// there are many leaf pages
......@@ -51,15 +51,26 @@ public class TestMVStoreTool extends TestBase {
s.commit();
}
}
for (int i = 0; i < 10; i++) {
map = s.openMap("data" + i);
for (int j = 0; j < i * i; j++) {
map.put(j, j * 10);
}
s.commit();
}
s.close();
// MVStoreTool.dump(fileName);
// MVStoreTool.dump(fileName + ".new");
MVStoreTool.compress(fileName, fileName + ".new");
MVStoreTool.compact(fileName, fileName + ".new", false);
MVStoreTool.compact(fileName, fileName + ".new.compress", true);
MVStore s1 = new MVStore.Builder().
fileName(fileName).readOnly().open();
MVStore s2 = new MVStore.Builder().
fileName(fileName + ".new").readOnly().open();
MVStore s3 = new MVStore.Builder().
fileName(fileName + ".new.compress").readOnly().open();
assertEquals(s1, s2);
assertEquals(s1, s3);
assertTrue(FileUtils.size(fileName + ".new") < FileUtils.size(fileName));
assertTrue(FileUtils.size(fileName + ".new.compress") < FileUtils.size(fileName + ".new"));
}
private void assertEquals(MVStore a, MVStore b) {
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论