提交 5301473f authored 作者: Thomas Mueller's avatar Thomas Mueller

MVStore: new utility to compress a store. (Work in progress.)

上级 e20b12b4
......@@ -1029,6 +1029,12 @@ public class MVMap<K, V> extends AbstractMap<K, V>
throw DataUtils.newConcurrentModificationException(
getName());
}
;
// TODO work in progress
if(oldRoots.size() - list.size() > 1) {
// System.out.println("reduced! from " + oldRoots.size() + " to " + list.size() +" " + getClass());
}
oldRoots = list;
}
......@@ -1292,21 +1298,23 @@ public class MVMap<K, V> extends AbstractMap<K, V>
this.writeVersion = writeVersion;
}
public void copyFrom(MVMap<K, V> sourceMap) {
void copyFrom(MVMap<K, V> sourceMap) {
; // TODO work in progress
Page sourceRoot = sourceMap.root;
root = Page.create(this, writeVersion, sourceRoot);
root = copy(root, sourceRoot);
root = copy(sourceMap.root, null);
}
private Page copy(Page target, Page source) {
target = copyOnWrite(target, writeVersion);
private Page copy(Page source, CursorPos parent) {
Page target = Page.create(this, writeVersion, source);
for (CursorPos p = parent; p != null; p = p.parent) {
p.page.setChild(p.index, target);
}
if (!target.isLeaf()) {
CursorPos pos = new CursorPos(target, 0, parent);
target = copyOnWrite(target, writeVersion);
for (int i = 0; i < target.getChildPageCount(); i++) {
Page sourceChild = source.getChildPage(i);
Page targetChild = Page.create(this, writeVersion, sourceChild);
targetChild = copy(targetChild, sourceChild);
target.setChild(i, targetChild);
pos.index = i;
copy(sourceChild, pos);
}
}
return target;
......
......@@ -15,6 +15,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import org.h2.mvstore.type.DataType;
import org.h2.mvstore.type.StringDataType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
......@@ -79,7 +80,8 @@ public class MVStoreTool {
pw.println("File not found: " + fileName);
return;
}
pw.printf("File %s, length %d\n", fileName, FileUtils.size(fileName));
long size = FileUtils.size(fileName);
pw.printf("File %s, %d bytes, %d MB\n", fileName, size, size / 1024 / 1024);
FileChannel file = null;
int blockSize = MVStore.BLOCK_SIZE;
try {
......@@ -214,6 +216,7 @@ public class MVStoreTool {
Chunk.FOOTER_LENGTH, DataUtils.LATIN).trim());
}
pw.printf("%n%0" + len + "x eof%n", fileSize);
pw.printf("\n");
} catch (IOException e) {
pw.println("ERROR: " + e);
e.printStackTrace(pw);
......@@ -278,6 +281,7 @@ public class MVStoreTool {
c.len
);
}
pw.printf("\n");
} catch (Exception e) {
pw.println("ERROR: " + e);
e.printStackTrace(pw);
......@@ -292,4 +296,109 @@ public class MVStoreTool {
return x.substring(0, 19);
}
/**
* Compress the store by creating a new file and copying the live pages there.
*
* @param fileName the file name
*/
public static void compress(String fileName) {
compress(fileName, fileName + ".new");
FileUtils.moveTo(fileName, fileName + ".old");
FileUtils.moveTo(fileName, fileName);
FileUtils.delete(fileName + ".old");
}
/**
* Copy all live pages from the source store to the target store.
*
* @param sourceFileName the name of the source store
* @param targetFileName the name of the target store
*/
public static void compress(String sourceFileName, String targetFileName) {
MVStore source = new MVStore.Builder().
fileName(sourceFileName).readOnly().open();
FileUtils.delete(targetFileName);
MVStore target = new MVStore.Builder().
fileName(targetFileName).open();
MVMap<String, String> sourceMeta = source.getMetaMap();
MVMap<String, String> targetMeta = target.getMetaMap();
for (Entry<String, String> m : sourceMeta.entrySet()) {
String key = m.getKey();
if (key.startsWith("chunk.")) {
// ignore
} else if (key.startsWith("map.")) {
// ignore
} else if (key.startsWith("name.")) {
// ignore
} else if (key.startsWith("root.")) {
// ignore
} else {
targetMeta.put(key, m.getValue());
}
}
for (String mapName : source.getMapNames()) {
MVMap.Builder<Object, Object> mp =
new MVMap.Builder<Object, Object>().
keyType(new GenericDataType()).
valueType(new GenericDataType());
MVMap<Object, Object> sourceMap = source.openMap(mapName, mp);
MVMap<Object, Object> targetMap = target.openMap(mapName, mp);
targetMap.copyFrom(sourceMap);
target.commit();
}
target.close();
source.close();
}
/**
* A data type that can read any data that is persisted, and converts it to
* a byte array.
*/
static class GenericDataType implements DataType {
@Override
public int compare(Object a, Object b) {
throw DataUtils.newUnsupportedOperationException("Can not compare");
}
@Override
public int getMemory(Object obj) {
return obj == null ? 0 : ((byte[]) obj).length;
}
@Override
public void write(WriteBuffer buff, Object obj) {
if (obj != null) {
buff.put((byte[]) obj);
}
}
@Override
public void write(WriteBuffer buff, Object[] obj, int len, boolean key) {
for (Object o : obj) {
write(buff, o);
}
}
@Override
public Object read(ByteBuffer buff) {
int len = buff.remaining();
if (len == 0) {
return null;
}
byte[] data = new byte[len];
buff.get(data);
return data;
}
@Override
public void read(ByteBuffer buff, Object[] obj, int len, boolean key) {
for (int i = 0; i < obj.length; i++) {
obj[i] = read(buff);
}
}
}
}
/*
* Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
* and the EPL 1.0 (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.util.Map.Entry;
import org.h2.mvstore.MVMap;
import org.h2.mvstore.MVStore;
import org.h2.mvstore.MVStoreTool;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
/**
* Tests the MVStoreTool class.
*/
public class TestMVStoreTool extends TestBase {
/**
* Run just this test.
*
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase test = TestBase.createCaller().init();
test.config.traceTest = true;
test.config.big = true;
test.test();
}
@Override
public void test() throws Exception {
testCompress();
}
private void testCompress() {
String fileName = getBaseDir() + "/testCompress.h3";
FileUtils.delete(fileName);
// store with a very small page size, to make sure
// there are many leaf pages
MVStore s = new MVStore.Builder().
pageSplitSize(100).
fileName(fileName).autoCommitDisabled().open();
MVMap<Integer, Integer> map = s.openMap("data");
for (int i = 0; i < 10; i++) {
map.put(i, i * 10);
if (i % 3 == 0) {
s.commit();
}
}
s.close();
MVStoreTool.dump(fileName);
MVStoreTool.dump(fileName + ".new");
MVStoreTool.compress(fileName, fileName + ".new");
MVStore s1 = new MVStore.Builder().
fileName(fileName).readOnly().open();
MVStore s2 = new MVStore.Builder().
fileName(fileName + ".new").readOnly().open();
assertEquals(s1, s2);
}
private void assertEquals(MVStore a, MVStore b) {
assertEquals(a.getMapNames().size(), b.getMapNames().size());
for (String mapName : a.getMapNames()) {
MVMap<?, ?> ma = a.openMap(mapName);
MVMap<?, ?> mb = a.openMap(mapName);
assertEquals(ma.sizeAsLong(), mb.sizeAsLong());
for (Entry<?, ?> e : ma.entrySet()) {
Object x = mb.get(e.getKey());
assertEquals(e.getValue().toString(), x.toString());
}
}
}
}
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论