Skip to content

H
h2database
提交 27be225f authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

A persistent multi-version map (work in progress) - allow reading old versions… 

A persistent multi-version map (work in progress) - allow reading old versions after storing; in-memory mode
上级 7b72a44d
隐藏空白字符变更
内嵌 并排
正在显示 包含 195 行增加 和 107 行删除
h2/src/test/org/h2/test/store/RtreeMap.java
浏览文件 @ 27be225f
......@@ -104,7 +104,7 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
for (int i = 0; i < p.getKeyCount(); i++) {
if (contains(p, i, key)) {
Page c = p.getChildPage(i);
long oldSize = c.getTotalSize();
long oldSize = c.getTotalCount();
Page c2 = remove(c, writeVersion, key);
if (c2 == null) {
// this child was deleted
......@@ -114,7 +114,7 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
}
p = p.copyOnWrite(writeVersion);
p.remove(i);
} else if (oldSize != c2.getTotalSize()) {
} else if (oldSize != c2.getTotalCount()) {
p = p.copyOnWrite(writeVersion);
Object oldBounds = p.getKey(i);
if (!keyType.isInside(key, oldBounds)) {
......@@ -197,21 +197,22 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
Object[] keys = { key };
Object[] values = { value };
p = Page.create(this, writeVersion, 1,
keys, values, null, null, 1, 0);
keys, values, null, null, null, 1, 0);
return p;
}
if (p.getKeyCount() > maxPageSize) {
// only possible if this is the root, else we would have split earlier
// (this requires maxPageSize is fixed)
p = p.copyOnWrite(writeVersion);
long totalSize = p.getTotalSize();
long totalCount = p.getTotalCount();
Page split = split(p, writeVersion);
Object[] keys = { getBounds(p), getBounds(split) };
long[] children = { p.getPos(), split.getPos(), 0 };
long[] childrenSize = { p.getTotalSize(), split.getTotalSize(), 0 };
Page[] childrenPages = { p, split, null };
long[] counts = { p.getTotalCount(), split.getTotalCount(), 0 };
p = Page.create(this, writeVersion, 2,
keys, null, children, childrenSize,
totalSize, 0);
keys, null, children, childrenPages, counts,
totalCount, 0);
// now p is a node; continues
} else if (p.isLeaf()) {
p = p.copyOnWrite(writeVersion);
......@@ -246,7 +247,7 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
p = p.copyOnWrite(writeVersion);
p.setKey(index, getBounds(c));
p.setChild(index, c);
p.insertNode(index, getBounds(split), split.getPos(), split.getTotalSize());
p.insertNode(index, getBounds(split), split);
// now we are not sure where to add
return add(p, writeVersion, key, value, maxPageSize);
}
......@@ -360,8 +361,9 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
private Page newPage(boolean leaf, long writeVersion) {
Object[] values = leaf ? new Object[4] : null;
long[] c = leaf ? null : new long[1];
Page[] cp = leaf ? null : new Page[1];
return Page.create(this, writeVersion, 0,
new Object[4], values, c, c, 0, 0);
new Object[4], values, c, cp, c, 0, 0);
}
private static void move(Page source, Page target, int sourceIndex) {
......@@ -370,9 +372,8 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
Object v = source.getValue(sourceIndex);
target.insertLeaf(0, k, v);
} else {
long c = source.getChildPage(sourceIndex).getPos();
long count = source.getCounts(sourceIndex);
target.insertNode(0, k, c, count);
Page c = source.getChildPage(sourceIndex);
target.insertNode(0, k, c);
}
source.remove(sourceIndex);
}
......
h2/src/test/org/h2/test/store/TestBtreeMapStore.java
浏览文件 @ 27be225f
......@@ -53,6 +53,7 @@ public class TestBtreeMapStore extends TestBase {
testRollbackInMemory();
testRollbackStored();
testMeta();
testInMemory();
testLargeImport();
testBtreeStore();
testDefragment();
......@@ -95,9 +96,9 @@ public class TestBtreeMapStore extends TestBase {
s.setRetainChunk(0);
long old2 = s.store();
// TODO keep old version after storing
// assertEquals("Hello", mOld.get("1"));
// assertEquals("World", mOld.get("2"));
// the old version is still available
assertEquals("Hello", mOld.get("1"));
assertEquals("World", mOld.get("2"));
m.put("1", "Hi");
m.remove("2");
......@@ -528,6 +529,36 @@ public class TestBtreeMapStore extends TestBase {
s.close();
}
private void testInMemory() {
for (int j = 0; j < 1; j++) {
BtreeMapStore s = openStore(null);
// s.setMaxPageSize(10);
// long t;
int len = 100;
// TreeMap<Integer, String> m = new TreeMap<Integer, String>();
// HashMap<Integer, String> m = New.hashMap();
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
// t = System.currentTimeMillis();
for (int i = 0; i < len; i++) {
m.put(i, "Hello World");
}
// System.out.println("put: " + (System.currentTimeMillis() - t));
// t = System.currentTimeMillis();
for (int i = 0; i < len; i++) {
assertEquals("Hello World", m.get(i));
}
// System.out.println("get: " + (System.currentTimeMillis() - t));
// t = System.currentTimeMillis();
for (int i = 0; i < len; i++) {
m.remove(i);
}
// System.out.println("remove: " + (System.currentTimeMillis() - t));
// System.out.println();
assertEquals(0, m.size());
s.close();
}
}
private void testLargeImport() {
String fileName = getBaseDir() + "/testImport.h3";
int len = 1000;
......
h2/src/tools/org/h2/dev/store/btree/BtreeMap.java
浏览文件 @ 27be225f
......@@ -75,7 +75,7 @@ public class BtreeMap<K, V> {
Object[] keys = { key };
Object[] values = { value };
p = Page.create(this, writeVersion, 1,
keys, values, null, null, 1, 0);
keys, values, null, null, null, 1, 0);
return p;
}
if (p.getKeyCount() > maxPageSize) {
......@@ -83,14 +83,15 @@ public class BtreeMap<K, V> {
// (this requires maxPageSize is fixed)
p = p.copyOnWrite(writeVersion);
int at = p.getKeyCount() / 2;
long totalSize = p.getTotalSize();
long totalCount = p.getTotalCount();
Object k = p.getKey(at);
Page split = p.split(at);
Object[] keys = { k };
long[] children = { p.getPos(), split.getPos() };
long[] childrenSize = { p.getTotalSize(), split.getTotalSize() };
Page[] childrenPages = { p, split };
long[] counts = { p.getTotalCount(), split.getTotalCount() };
p = Page.create(this, writeVersion, 1,
keys, null, children, childrenSize, totalSize, 0);
keys, null, children, childrenPages, counts, totalCount, 0);
// now p is a node; insert continues
} else if (p.isLeaf()) {
int index = p.binarySearch(key);
......@@ -119,13 +120,13 @@ public class BtreeMap<K, V> {
Page split = c.split(at);
p = p.copyOnWrite(writeVersion);
p.setChild(index, split);
p.insertNode(index, k, c.getPos(), c.getTotalSize());
p.insertNode(index, k, c);
// now we are not sure where to add
return put(p, writeVersion, key, value, maxPageSize);
}
long oldSize = c.getTotalSize();
long oldSize = c.getTotalCount();
Page c2 = put(c, writeVersion, key, value, maxPageSize);
if (c != c2 || oldSize != c2.getTotalSize()) {
if (c != c2 || oldSize != c2.getTotalCount()) {
p = p.copyOnWrite(writeVersion);
p.setChild(index, c2);
}
......@@ -354,7 +355,7 @@ public class BtreeMap<K, V> {
index++;
}
Page c = p.getChildPage(index);
long oldSize = c.getTotalSize();
long oldCount = c.getTotalCount();
Page c2 = remove(c, writeVersion, key);
if (c2 == null) {
// this child was deleted
......@@ -364,7 +365,7 @@ public class BtreeMap<K, V> {
removePage(p);
p = p.getChildPage(0);
}
} else if (oldSize != c2.getTotalSize()) {
} else if (oldCount != c2.getTotalCount()) {
p = p.copyOnWrite(writeVersion);
p.setChild(index, c2);
}
......@@ -562,7 +563,7 @@ public class BtreeMap<K, V> {
}
public long getSize() {
return root == null ? 0 : root.getTotalSize();
return root == null ? 0 : root.getTotalCount();
}
public boolean equals(Object o) {
......
h2/src/tools/org/h2/dev/store/btree/BtreeMapStore.java
浏览文件 @ 27be225f
......@@ -11,7 +11,6 @@ import java.io.StringReader;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
......@@ -53,7 +52,6 @@ TODO:
- support large binaries
- support stores that span multiple files (chunks stored in other files)
- triggers
- compare with newest version of IndexedDb
- support database version / schema version
- implement more counted b-tree (skip, get positions)
- merge pages if small
......@@ -83,10 +81,7 @@ public class BtreeMapStore {
private int blockSize = 4 * 1024;
private long rootChunkStart;
private int tempPageId;
private Map<Long, Page> cache = CacheLIRS.newInstance(readCacheSize, 2048);
private HashMap<Long, Page> temp = New.hashMap();
private Page[] tempCache = new Page[64];
private int lastChunkId;
private HashMap<Integer, Chunk> chunks = New.hashMap();
......@@ -134,7 +129,7 @@ public class BtreeMapStore {
/**
* Open a tree store.
*
* @param fileName the file name
* @param fileName the file name (null for in-memory)
* @param mapFactory the map factory
* @return the store
*/
......@@ -306,6 +301,9 @@ public class BtreeMapStore {
private void open() {
meta = new BtreeMap<String, String>(this, 0, "meta", STRING_TYPE, STRING_TYPE, 0);
if (fileName == null) {
return;
}
FileUtils.createDirectories(FileUtils.getParent(fileName));
try {
log("file open");
......@@ -397,8 +395,6 @@ public class BtreeMapStore {
for (BtreeMap<?, ?> m : New.arrayList(maps.values())) {
m.close();
}
temp.clear();
Arrays.fill(tempCache, null);
meta = null;
compressor = null;
chunks.clear();
......@@ -623,21 +619,6 @@ public class BtreeMapStore {
return set.size() * blockSize;
}
/**
* Register a page and get the next temporary page id.
*
* @param p the new page
* @return the page id
*/
long registerTempPage(Page p) {
long pos = --tempPageId;
// use -pos so the Long cache can be used
temp.put(-pos, p);
int index = (int) pos & (tempCache.length - 1);
tempCache[index] = p;
return pos;
}
/**
* Check whether there are any unsaved changes.
*
......@@ -833,15 +814,6 @@ public class BtreeMapStore {
* @return the page
*/
Page readPage(BtreeMap<?, ?> map, long pos) {
if (pos < 0) {
int index = (int) pos & (tempCache.length - 1);
Page p = tempCache[index];
if (p == null || p.getPos() != pos) {
p = temp.get(-pos);
tempCache[index] = p;
}
return p;
}
Page p = cache.get(pos);
if (p == null) {
long filePos = getFilePosition(pos);
......@@ -1033,9 +1005,6 @@ public class BtreeMapStore {
m.revertTemp();
}
mapsChanged.clear();
temp.clear();
Arrays.fill(tempCache, null);
tempPageId = 0;
}
/**
......
h2/src/tools/org/h2/dev/store/btree/Page.java
浏览文件 @ 27be225f
......@@ -46,6 +46,7 @@ public class Page {
private Object[] keys;
private Object[] values;
private long[] children;
private Page[] childrenPages;
private long[] counts;
private Page(BtreeMap<?, ?> map, long version) {
......@@ -65,14 +66,15 @@ public class Page {
*/
public static Page create(BtreeMap<?, ?> map, long version,
int keyCount, Object[] keys,
Object[] values, long[] children, long[] counts,
Object[] values, long[] children, Page[] childrenPages, long[] counts,
long totalCount, int sharedFlags) {
Page p = new Page(map, version);
p.pos = map.getStore().registerTempPage(p);
// the position is 0
p.keys = keys;
p.keyCount = keyCount;
p.values = values;
p.children = children;
p.childrenPages = childrenPages;
p.counts = counts;
p.totalCount = totalCount;
p.sharedFlags = sharedFlags;
......@@ -117,7 +119,8 @@ public class Page {
}
public Page getChildPage(int index) {
return map.readPage(children[index]);
Page p = childrenPages[index];
return p != null ? p : map.readPage(children[index]);
}
public Object getValue(int x) {
......@@ -168,7 +171,7 @@ public class Page {
}
map.getStore().removePage(pos);
Page newPage = create(map, writeVersion,
keyCount, keys, values, children,
keyCount, keys, values, children, childrenPages,
counts, totalCount,
SHARED_KEYS | SHARED_VALUES | SHARED_CHILDREN | SHARED_COUNTS);
newPage.cachedCompare = cachedCompare;
......@@ -243,29 +246,39 @@ public class Page {
sharedFlags &= ~(SHARED_KEYS | SHARED_VALUES);
totalCount = a;
Page newPage = create(map, version, b,
bKeys, bValues, null, null,
bKeys, bValues, null, null, null,
bKeys.length, 0);
return newPage;
}
private Page splitNode(int at) {
int a = at, b = keyCount - a;
Object[] aKeys = new Object[a];
Object[] bKeys = new Object[b - 1];
System.arraycopy(keys, 0, aKeys, 0, a);
System.arraycopy(keys, a + 1, bKeys, 0, b - 1);
keys = aKeys;
keyCount = a;
long[] aChildren = new long[a + 1];
long[] bChildren = new long[b];
System.arraycopy(children, 0, aChildren, 0, a + 1);
System.arraycopy(children, a + 1, bChildren, 0, b);
children = aChildren;
Page[] aChildrenPages = new Page[a + 1];
Page[] bChildrenPages = new Page[b];
System.arraycopy(childrenPages, 0, aChildrenPages, 0, a + 1);
System.arraycopy(childrenPages, a + 1, bChildrenPages, 0, b);
childrenPages = aChildrenPages;
long[] aCounts = new long[a + 1];
long[] bCounts = new long[b];
System.arraycopy(counts, 0, aCounts, 0, a + 1);
System.arraycopy(counts, a + 1, bCounts, 0, b);
counts = aCounts;
sharedFlags &= ~(SHARED_KEYS | SHARED_CHILDREN | SHARED_COUNTS);
long t = 0;
for (long x : aCounts) {
......@@ -277,12 +290,12 @@ public class Page {
t += x;
}
Page newPage = create(map, version, b - 1,
bKeys, null, bChildren,
bKeys, null, bChildren, bChildrenPages,
bCounts, t, 0);
return newPage;
}
public long getTotalSize() {
public long getTotalCount() {
if (BtreeMapStore.ASSERT) {
long check = 0;
if (isLeaf()) {
......@@ -304,17 +317,19 @@ public class Page {
if (c.getPos() != children[index]) {
if ((sharedFlags & SHARED_CHILDREN) != 0) {
children = Arrays.copyOf(children, children.length);
childrenPages = Arrays.copyOf(childrenPages, childrenPages.length);
sharedFlags &= ~SHARED_CHILDREN;
}
children[index] = c.getPos();
childrenPages[index] = c;
}
if (c.getTotalSize() != counts[index]) {
if (c.getTotalCount() != counts[index]) {
if ((sharedFlags & SHARED_COUNTS) != 0) {
counts = Arrays.copyOf(counts, counts.length);
sharedFlags &= ~SHARED_COUNTS;
}
long oldCount = counts[index];
counts[index] = c.getTotalSize();
counts[index] = c.getTotalCount();
totalCount += counts[index] - oldCount;
}
}
......@@ -340,12 +355,18 @@ public class Page {
*/
void removeAllRecursive() {
if (children != null) {
for (long c : children) {
int type = DataUtils.getPageType(c);
if (type == DataUtils.PAGE_TYPE_LEAF) {
map.getStore().removePage(c);
for (int i = 0, size = children.length; i < size; i++) {
Page p = childrenPages[i];
if (p != null) {
p.removeAllRecursive();
} else {
map.readPage(c).removeAllRecursive();
long c = children[i];
int type = DataUtils.getPageType(c);
if (type == DataUtils.PAGE_TYPE_LEAF) {
map.getStore().removePage(c);
} else {
map.readPage(c).removeAllRecursive();
}
}
}
}
......@@ -374,52 +395,118 @@ public class Page {
totalCount++;
}
public void insertNode(int index, Object key, long child, long count) {
public void insertNode(int index, Object key, Page childPage) {
Object[] newKeys = new Object[keyCount + 1];
DataUtils.copyWithGap(keys, newKeys, keyCount, index);
newKeys[index] = key;
keys = newKeys;
keyCount++;
long[] newChildren = new long[children.length + 1];
DataUtils.copyWithGap(children, newChildren, children.length, index);
newChildren[index] = child;
newChildren[index] = childPage.getPos();
children = newChildren;
Page[] newChildrenPages = new Page[childrenPages.length + 1];
DataUtils.copyWithGap(childrenPages, newChildrenPages, childrenPages.length, index);
newChildrenPages[index] = childPage;
childrenPages = newChildrenPages;
long[] newCounts = new long[counts.length + 1];
DataUtils.copyWithGap(counts, newCounts, counts.length, index);
newCounts[index] = count;
newCounts[index] = childPage.getTotalCount();
counts = newCounts;
sharedFlags &= ~(SHARED_KEYS | SHARED_CHILDREN | SHARED_COUNTS);
totalCount += count;
totalCount += childPage.getTotalCount();
}
public void remove(int index) {
Object[] newKeys = new Object[keyCount - 1];
int keyIndex = index >= keyCount ? index - 1 : index;
DataUtils.copyExcept(keys, newKeys, keyCount, keyIndex);
keys = newKeys;
sharedFlags &= ~SHARED_KEYS;
if ((sharedFlags & SHARED_KEYS) == 0 && keys.length > keyCount - 4) {
if (keyIndex < keyCount - 1) {
System.arraycopy(keys, keyIndex + 1, keys, keyIndex, keyCount - keyIndex - 1);
}
keys[keyCount - 1] = null;
} else {
Object[] newKeys = new Object[keyCount - 1];
DataUtils.copyExcept(keys, newKeys, keyCount, keyIndex);
keys = newKeys;
sharedFlags &= ~SHARED_KEYS;
}
if (values != null) {
Object[] newValues = new Object[keyCount - 1];
DataUtils.copyExcept(values, newValues, keyCount, index);
values = newValues;
sharedFlags &= ~SHARED_VALUES;
if ((sharedFlags & SHARED_VALUES) == 0 && values.length > keyCount - 4) {
if (index < keyCount - 1) {
System.arraycopy(values, index + 1, values, index, keyCount - index - 1);
}
values[keyCount - 1] = null;
} else {
Object[] newValues = new Object[keyCount - 1];
DataUtils.copyExcept(values, newValues, keyCount, index);
values = newValues;
sharedFlags &= ~SHARED_VALUES;
}
totalCount--;
}
keyCount--;
if (children != null) {
long countOffset = counts[index];
long[] newChildren = new long[children.length - 1];
DataUtils.copyExcept(children, newChildren, children.length, index);
children = newChildren;
Page[] newChildrenPages = new Page[childrenPages.length - 1];
DataUtils.copyExcept(childrenPages, newChildrenPages,
childrenPages.length, index);
childrenPages = newChildrenPages;
long[] newCounts = new long[counts.length - 1];
DataUtils.copyExcept(counts, newCounts,
counts.length, index);
DataUtils.copyExcept(counts, newCounts, counts.length, index);
counts = newCounts;
sharedFlags &= ~(SHARED_CHILDREN | SHARED_COUNTS);
totalCount -= countOffset;
}
}
// public void remove(int index) {
// Object[] newKeys = new Object[keyCount - 1];
// int keyIndex = index >= keyCount ? index - 1 : index;
// DataUtils.copyExcept(keys, newKeys, keyCount, keyIndex);
// keys = newKeys;
// sharedFlags &= ~SHARED_KEYS;
// if (values != null) {
// Object[] newValues = new Object[keyCount - 1];
// DataUtils.copyExcept(values, newValues, keyCount, index);
// values = newValues;
// sharedFlags &= ~SHARED_VALUES;
// totalCount--;
// }
// keyCount--;
// if (children != null) {
// long countOffset = counts[index];
//
// long[] newChildren = new long[children.length - 1];
// DataUtils.copyExcept(children, newChildren, children.length, index);
// children = newChildren;
//
// Page[] newChildrenPages = new Page[childrenPages.length - 1];
// DataUtils.copyExcept(childrenPages, newChildrenPages, childrenPages.length, index);
// childrenPages = newChildrenPages;
//
// long[] newCounts = new long[counts.length - 1];
// DataUtils.copyExcept(counts, newCounts,
// counts.length, index);
// counts = newCounts;
//
// sharedFlags &= ~(SHARED_CHILDREN | SHARED_COUNTS);
// totalCount -= countOffset;
// }
// }
private void read(ByteBuffer buff, int chunkId, int offset, int maxLength) {
int start = buff.position();
int pageLength = buff.getInt();
......@@ -464,6 +551,7 @@ public class Page {
for (int i = 0; i <= len; i++) {
children[i] = buff.getLong();
}
childrenPages = new Page[len + 1];
counts = new long[len + 1];
long total = 0;
for (int i = 0; i <= len; i++) {
......@@ -550,19 +638,19 @@ public class Page {
for (int i = 0; i < len; i++) {
maxLength += map.getKeyType().getMaxLength(keys[i]);
}
if (children != null) {
if (isLeaf()) {
for (int i = 0; i < len; i++) {
maxLength += map.getValueType().getMaxLength(values[i]);
}
} else {
maxLength += 8 * len;
maxLength += DataUtils.MAX_VAR_LONG_LEN * len;
for (int i = 0; i <= len; i++) {
long c = children[i];
if (c < 0) {
maxLength += map.readPage(c).getMaxLengthTempRecursive();
Page p = childrenPages[i];
if (p != null) {
maxLength += p.getMaxLengthTempRecursive();
}
}
} else {
for (int i = 0; i < len; i++) {
maxLength += map.getValueType().getMaxLength(values[i]);
}
}
return maxLength;
}
......@@ -576,13 +664,13 @@ public class Page {
* @return the page id
*/
long writeTempRecursive(ByteBuffer buff, int chunkId) {
if (children != null) {
if (!isLeaf()) {
int len = children.length;
for (int i = 0; i < len; i++) {
long c = children[i];
if (c < 0) {
children[i] = map.readPage(c).writeTempRecursive(buff,
chunkId);
Page p = childrenPages[i];
if (p != null) {
children[i] = p.writeTempRecursive(buff, chunkId);
childrenPages[i] = null;
}
}
}
......@@ -597,12 +685,10 @@ public class Page {
*/
int countTempRecursive() {
int count = 1;
if (children != null) {
int size = children.length;
for (int i = 0; i < size; i++) {
long c = children[i];
if (c < 0) {
count += map.readPage(c).countTempRecursive();
if (!isLeaf()) {
for (Page p : childrenPages) {
if (p != null) {
count += p.countTempRecursive();
}
}
}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论