Skip to content

H
h2database
提交 d5cc1876 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

A persistent multi-version map (work in progress) - allow reading just the changed pages

上级 27be225f
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 446 行增加 和 279 行删除
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ d5cc1876
......@@ -106,6 +106,7 @@ import org.h2.test.server.TestInit;
import org.h2.test.store.TestCacheLIRS;
import org.h2.test.store.TestDataUtils;
import org.h2.test.store.TestBtreeMapStore;
import org.h2.test.store.TestRtree;
import org.h2.test.synth.TestBtreeIndex;
import org.h2.test.synth.TestCrashAPI;
import org.h2.test.synth.TestDiskFull;
......@@ -667,6 +668,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
new TestBtreeMapStore().runTest(this);
new TestCacheLIRS().runTest(this);
new TestDataUtils().runTest(this);
new TestRtree().runTest(this);
// unit
new TestAutoReconnect().runTest(this);
......
h2/src/test/org/h2/test/store/RtreeMap.java
浏览文件 @ d5cc1876
......@@ -392,7 +392,7 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
* Go to the first element for the given key.
*
* @param p the current page
* @param parents the stack of parent page positions
* @param cursor the cursor
* @param key the key
*/
protected CursorPos min(Page p, Cursor<K, V> cursor, Object key) {
......@@ -413,11 +413,12 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
/**
* Get the next key.
*
* @param parents the stack of parent page positions
* @param p the cursor position
* @param cursor the cursor
* @return the next key
*/
protected Object nextKey(CursorPos p, Cursor<K, V> cursor) {
while (true) {
while (p != null) {
int index = p.index++;
Page x = p.page;
if (index < x.getKeyCount()) {
......@@ -426,19 +427,22 @@ public class RtreeMap<K, V> extends BtreeMap<K, V> {
while (true) {
p = cursor.pop();
if (p == null) {
return null;
break;
}
index = ++p.index;
x = p.page;
// this is different from a b-tree:
// we have one less child
if (index < x.getKeyCount()) {
cursor.push(p);
x = x.getChildPage(index);
p = min(x, cursor, null);
cursor.setCurrentPos(p);
break;
p = cursor.visitChild(x, index);
if (p != null) {
break;
}
}
}
}
return null;
}
public boolean isQuadraticSplit() {
......
h2/src/test/org/h2/test/store/TestRtree.java 0 → 100644
浏览文件 @ d5cc1876
package org.h2.test.store;
import java.awt.AlphaComposite;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.RenderingHints;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Random;
import javax.imageio.ImageIO;
import javax.imageio.ImageWriter;
import javax.imageio.stream.FileImageOutputStream;
import org.h2.dev.store.btree.BtreeMapStore;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.New;
/**
* Tests the r-tree.
*/
public class TestRtree extends TestBtreeMapStore {
/**
* Run just this test.
*
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
}
public void test() {
testRtreeMany();
testRtree();
testRandomRtree();
testCustomMapType();
}
private void testRtreeMany() {
String fileName = getBaseDir() + "/testRtree.h3";
FileUtils.delete(fileName);
BtreeMapStore s;
s = openStore(fileName);
// s.setMaxPageSize(50);
RtreeMap<SpatialKey, String> r = s.openMap("data", "r", "s2", "");
// r.setQuadraticSplit(true);
Random rand = new Random(1);
int len = 1000;
// long t = System.currentTimeMillis();
// Profiler prof = new Profiler();
// prof.startCollecting();
for (int i = 0; i < len; i++) {
float x = rand.nextFloat(), y = rand.nextFloat();
float p = (float) (rand.nextFloat() * 0.000001);
SpatialKey k = new SpatialKey(i, x - p, x + p, y - p, y + p);
r.add(k, "" + i);
if (i > 0 && (i % len / 10) == 0) {
s.store();
}
if (i > 0 && (i % 10000) == 0) {
render(r, getBaseDir() + "/test.png");
}
}
// System.out.println(prof.getTop(5));
// System.out.println("add: " + (System.currentTimeMillis() - t));
s.store();
s.close();
s = openStore(fileName);
r = s.openMap("data", "r", "s2", "");
// t = System.currentTimeMillis();
rand = new Random(1);
for (int i = 0; i < len; i++) {
float x = rand.nextFloat(), y = rand.nextFloat();
float p = (float) (rand.nextFloat() * 0.000001);
SpatialKey k = new SpatialKey(i, x - p, x + p, y - p, y + p);
assertEquals("" + i, r.get(k));
}
// System.out.println("query: " + (System.currentTimeMillis() - t));
assertEquals(len, r.size());
int count = 0;
for (SpatialKey k : r.keySet()) {
assertTrue(r.get(k) != null);
count++;
}
assertEquals(len, count);
// t = System.currentTimeMillis();
// Profiler prof = new Profiler();
// prof.startCollecting();
rand = new Random(1);
for (int i = 0; i < len; i++) {
float x = rand.nextFloat(), y = rand.nextFloat();
float p = (float) (rand.nextFloat() * 0.000001);
SpatialKey k = new SpatialKey(i, x - p, x + p, y - p, y + p);
r.remove(k);
}
assertEquals(0, r.size());
s.close();
// System.out.println(prof.getTop(5));
// System.out.println("remove: " + (System.currentTimeMillis() - t));
}
private void testRtree() {
String fileName = getBaseDir() + "/testRtree.h3";
FileUtils.delete(fileName);
BtreeMapStore s;
s = openStore(fileName);
RtreeMap<SpatialKey, String> r = s.openMap("data", "r", "s2", "");
add(r, "Bern", 46.57, 7.27, 124381);
add(r, "Basel", 47.34, 7.36, 170903);
add(r, "Zurich", 47.22, 8.33, 376008);
add(r, "Lucerne", 47.03, 8.18, 77491);
add(r, "Geneva", 46.12, 6.09, 191803);
add(r, "Lausanne", 46.31, 6.38, 127821);
add(r, "Winterthur", 47.30, 8.45, 102966);
add(r, "St. Gallen", 47.25, 9.22, 73500);
add(r, "Biel/Bienne", 47.08, 7.15, 51203);
add(r, "Lugano", 46.00, 8.57, 54667);
add(r, "Thun", 46.46, 7.38, 42623);
add(r, "Bellinzona", 46.12, 9.01, 17373);
add(r, "Chur", 46.51, 9.32, 33756);
ArrayList<String> list = New.arrayList();
for (SpatialKey x : r.keySet()) {
list.add(r.get(x));
}
Collections.sort(list);
assertEquals("[Basel, Bellinzona, Bern, Biel/Bienne, Chur, Geneva, " +
"Lausanne, Lucerne, Lugano, St. Gallen, Thun, Winterthur, Zurich]",
list.toString());
// render(r, getBaseDir() + "/test.png");
s.close();
}
private static void add(RtreeMap<SpatialKey, String> r, String name, double y, double x, int population) {
int id = r.size();
float a = (float) ((int) x + (x - (int) x) * 5 / 3);
float b = 50 - (float) ((int) y + (y - (int) y) * 5 / 3);
float s = (float) Math.sqrt(population / 10000000.);
SpatialKey k = new SpatialKey(id, a - s, a + s, b - s, b + s);
r.put(k, name);
}
private static void render(RtreeMap<SpatialKey, String> r, String fileName) {
int width = 1000, height = 500;
BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
Graphics2D g2d = (Graphics2D) img.getGraphics();
g2d.setBackground(Color.WHITE);
g2d.setColor(Color.WHITE);
g2d.fillRect(0, 0, width, height);
g2d.setComposite(AlphaComposite.SrcOver.derive(0.5f));
g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
RenderingHints.VALUE_ANTIALIAS_ON);
g2d.setColor(Color.BLACK);
SpatialKey b = new SpatialKey(0, Float.MAX_VALUE, Float.MIN_VALUE,
Float.MAX_VALUE, Float.MIN_VALUE);
for (SpatialKey x : r.keySet()) {
b.setMin(0, Math.min(b.min(0), x.min(0)));
b.setMin(1, Math.min(b.min(1), x.min(1)));
b.setMax(0, Math.max(b.max(0), x.max(0)));
b.setMax(1, Math.max(b.max(1), x.max(1)));
}
// System.out.println(b);
for (SpatialKey x : r.keySet()) {
int[] rect = scale(b, x, width, height);
g2d.drawRect(rect[0], rect[1], rect[2] - rect[0], rect[3] - rect[1]);
String s = r.get(x);
g2d.drawChars(s.toCharArray(), 0, s.length(), rect[0], rect[1] - 4);
}
g2d.setColor(Color.red);
ArrayList<SpatialKey> list = New.arrayList();
r.addNodeKeys(list, r.getRoot());
for (SpatialKey x : list) {
int[] rect = scale(b, x, width, height);
g2d.drawRect(rect[0], rect[1], rect[2] - rect[0], rect[3] - rect[1]);
}
ImageWriter out = ImageIO.getImageWritersByFormatName("png").next();
try {
out.setOutput(new FileImageOutputStream(new File(fileName)));
out.write(img);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static int[] scale(SpatialKey b, SpatialKey x, int width, int height) {
int[] rect = {
(int) ((x.min(0) - b.min(0)) * (width * 0.9) / (b.max(0) - b.min(0)) + width * 0.05),
(int) ((x.min(1) - b.min(1)) * (height * 0.9) / (b.max(1) - b.min(1)) + height * 0.05),
(int) ((x.max(0) - b.min(0)) * (width * 0.9) / (b.max(0) - b.min(0)) + width * 0.05),
(int) ((x.max(1) - b.min(1)) * (height * 0.9) / (b.max(1) - b.min(1)) + height * 0.05),
};
return rect;
}
private void testRandomRtree() {
String fileName = getBaseDir() + "/testRtreeRandom.h3";
FileUtils.delete(fileName);
BtreeMapStore s = openStore(fileName);
RtreeMap<SpatialKey, String> m = s.openMap("data", "r", "s2", "");
HashMap<SpatialKey, String> map = new HashMap<SpatialKey, String>();
Random rand = new Random(1);
int operationCount = 1000;
int maxValue = 30;
for (int i = 0; i < operationCount; i++) {
int key = rand.nextInt(maxValue);
Random rk = new Random(key);
float x = rk.nextFloat(), y = rk.nextFloat();
float p = (float) (rk.nextFloat() * 0.000001);
SpatialKey k = new SpatialKey(key, x - p, x + p, y - p, y + p);
String v = "" + rand.nextInt();
switch (rand.nextInt(3)) {
case 0:
log(i + ": put " + k + " = " + v + " " + m.size());
m.put(k, v);
map.put(k, v);
break;
case 1:
log(i + ": remove " + k + " " + m.size());
m.remove(k);
map.remove(k);
break;
default:
String a = map.get(k);
String b = m.get(k);
if (a == null || b == null) {
assertTrue(a == b);
} else {
assertEquals(a, b);
}
break;
}
assertEquals(map.size(), m.size());
}
s.close();
}
private void testCustomMapType() {
String fileName = getBaseDir() + "/testMapType.h3";
FileUtils.delete(fileName);
BtreeMapStore s;
s = openStore(fileName);
SequenceMap<Integer, String> seq = s.openMap("data", "s", "i", "");
StringBuilder buff = new StringBuilder();
for (int x : seq.keySet()) {
buff.append(x).append(';');
}
assertEquals("1;2;3;4;5;6;7;8;9;10;", buff.toString());
s.close();
}
}
h2/src/tools/org/h2/dev/store/btree/BtreeMap.java
浏览文件 @ d5cc1876
......@@ -152,24 +152,12 @@ public class BtreeMap<K, V> {
* Go to the first element for the given key.
*
* @param p the current page
* @param parents the stack of parent page positions
* @param cursor the cursor
* @param key the key
*/
protected CursorPos min(Page p, Cursor<K, V> cursor, Object key) {
while (p != null) {
if (!p.isLeaf()) {
int x = key == null ? -1 : p.binarySearch(key);
if (x < 0) {
x = -x - 1;
} else {
x++;
}
CursorPos c = new CursorPos();
c.page = p;
c.index = x;
cursor.push(c);
p = p.getChildPage(x);
} else {
if (p.isLeaf()) {
int x = key == null ? 0 : p.binarySearch(key);
if (x < 0) {
x = -x - 1;
......@@ -179,6 +167,17 @@ public class BtreeMap<K, V> {
c.index = x;
return c;
}
int x = key == null ? -1 : p.binarySearch(key);
if (x < 0) {
x = -x - 1;
} else {
x++;
}
CursorPos c = new CursorPos();
c.page = p;
c.index = x;
cursor.push(c);
p = p.getChildPage(x);
}
return null;
}
......@@ -186,31 +185,34 @@ public class BtreeMap<K, V> {
/**
* Get the next key.
*
* @param parents the stack of parent page positions
* @param p the cursor position
* @param cursor the cursor
* @return the next key
*/
protected Object nextKey(CursorPos p, Cursor<K, V> cursor) {
while (true) {
while (p != null) {
int index = p.index++;
if (index < p.page.getKeyCount()) {
return p.page.getKey(index);
Page x = p.page;
if (index < x.getKeyCount()) {
return x.getKey(index);
}
while (true) {
p = cursor.pop();
if (p == null) {
return null;
break;
}
index = ++p.index;
if (index <= p.page.getKeyCount()) {
x = p.page;
if (index <= x.getKeyCount()) {
cursor.push(p);
Page x = p.page;
x = x.getChildPage(index);
p = min(x, cursor, null);
cursor.setCurrentPos(p);
break;
p = cursor.visitChild(x, index);
if (p != null) {
break;
}
}
}
}
return null;
}
/**
......@@ -442,7 +444,22 @@ public class BtreeMap<K, V> {
*/
public Iterator<K> keyIterator(K from) {
checkOpen();
return new Cursor<K, V>(this, root, from);
Cursor<K, V> c = new Cursor<K, V>(this);
c.start(root, from);
return c;
}
/**
* Iterate over all keys in changed pages.
*
* @param minVersion the minimum version
* @return the iterator
*/
public Iterator<K> changeIterator(long minVersion) {
checkOpen();
Cursor<K, V> c = new ChangeCursor<K, V>(this, minVersion);
c.start(root, null);
return c;
}
public Set<K> keySet() {
......@@ -452,7 +469,9 @@ public class BtreeMap<K, V> {
@Override
public Iterator<K> iterator() {
return new Cursor<K, V>(BtreeMap.this, root, null);
Cursor<K, V> c = new Cursor<K, V>(BtreeMap.this);
c.start(root, null);
return c;
}
@Override
......
h2/src/tools/org/h2/dev/store/btree/BtreeMapStore.java
浏览文件 @ d5cc1876
......@@ -38,10 +38,9 @@ header:
blockSize=4096
TODO:
- ability to diff / merge versions
- limited support for writing to old versions (branches)
- implement complete java.util.Map interface
- maybe rename to MVStore, MVMap, TestMVStore
- limited support for writing to old versions (branches)
- atomic test-and-set (when supporting concurrent writes)
- support background writes (store old version)
- file header could be a regular chunk, end of file the second
......@@ -408,7 +407,7 @@ public class BtreeMapStore {
}
}
private Chunk getChunk(long pos) {
Chunk getChunk(long pos) {
return chunks.get(DataUtils.getPageChunkId(pos));
}
......
h2/src/tools/org/h2/dev/store/btree/ChangeCursor.java 0 → 100644
浏览文件 @ d5cc1876
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store.btree;
/**
* A cursor to iterate over all keys in new pages.
*
* @param <K> the key type
* @param <V> the value type
*/
public class ChangeCursor<K, V> extends Cursor<K, V> {
private final long minVersion;
ChangeCursor(BtreeMap<K, V> map, long minVersion) {
super(map);
this.minVersion = minVersion;
}
public CursorPos min(Page p, K from) {
while (p != null && p.getVersion() >= minVersion) {
if (p.isLeaf()) {
CursorPos c = new CursorPos();
c.page = p;
c.index = 0;
return c;
}
for (int i = 0; i < p.getChildPageCount(); i++) {
if (isChildOld(p, i)) {
continue;
}
CursorPos c = new CursorPos();
c.page = p;
c.index = i;
push(c);
p = p.getChildPage(i);
break;
}
}
return null;
}
public CursorPos visitChild(Page p, int childIndex) {
if (isChildOld(p, childIndex)) {
return null;
}
return super.visitChild(p, childIndex);
}
private boolean isChildOld(Page p, int childIndex) {
long pos = p.getChildPagePos(childIndex);
if (pos == 0) {
Page c = p.getChildPage(childIndex);
if (c.getVersion() < minVersion) {
return true;
}
} else if (map.getStore().getChunk(pos).version < minVersion) {
return true;
}
return false;
}
}
h2/src/tools/org/h2/dev/store/btree/Cursor.java
浏览文件 @ d5cc1876
......@@ -17,14 +17,17 @@ import java.util.Iterator;
*/
public class Cursor<K, V> implements Iterator<K> {
private final BtreeMap<K, V> map;
private final ArrayList<CursorPos> parents = new ArrayList<CursorPos>();
private CursorPos currentPos;
private K current;
protected final BtreeMap<K, V> map;
protected final ArrayList<CursorPos> parents = new ArrayList<CursorPos>();
protected CursorPos currentPos;
protected K current;
Cursor(BtreeMap<K, V> map, Page root, K from) {
Cursor(BtreeMap<K, V> map) {
this.map = map;
currentPos = map.min(root, this, from);
}
void start(Page root, K from) {
currentPos = min(root, from);
if (currentPos != null) {
fetchNext();
}
......@@ -39,7 +42,7 @@ public class Cursor<K, V> implements Iterator<K> {
}
@SuppressWarnings("unchecked")
private void fetchNext() {
protected void fetchNext() {
current = (K) map.nextKey(currentPos, this);
}
......@@ -51,10 +54,6 @@ public class Cursor<K, V> implements Iterator<K> {
throw new UnsupportedOperationException();
}
public void setCurrentPos(CursorPos p) {
currentPos = p;
}
public void push(CursorPos p) {
parents.add(p);
}
......@@ -64,5 +63,15 @@ public class Cursor<K, V> implements Iterator<K> {
return size == 0 ? null : parents.remove(size - 1);
}
public CursorPos min(Page p, K from) {
return map.min(p, this, from);
}
public CursorPos visitChild(Page p, int childIndex) {
p = p.getChildPage(childIndex);
currentPos = min(p, null);
return currentPos;
}
}
h2/src/tools/org/h2/dev/store/btree/Page.java
浏览文件 @ d5cc1876
......@@ -123,6 +123,10 @@ public class Page {
return p != null ? p : map.readPage(children[index]);
}
long getChildPagePos(int index) {
return children[index];
}
public Object getValue(int x) {
return values[x];
}
......@@ -314,7 +318,7 @@ public class Page {
}
public void setChild(int index, Page c) {
if (c.getPos() != children[index]) {
if (c != childrenPages[index] || c.getPos() != children[index]) {
if ((sharedFlags & SHARED_CHILDREN) != 0) {
children = Arrays.copyOf(children, children.length);
childrenPages = Arrays.copyOf(childrenPages, childrenPages.length);
......@@ -699,4 +703,12 @@ public class Page {
return counts[index];
}
long getVersion() {
return version;
}
int getChildPageCount() {
return children.length;
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论