Skip to content

H
h2database
提交 a7d2ed06 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

A persistent tree map (work in progress).

上级 3ff388a0
隐藏空白字符变更
内嵌 并排
正在显示 包含 893 行增加 和 409 行删除
h2/src/tools/org/h2/dev/store/btree/IntegerType.java → h2/src/test/org/h2/test/store/IntegerType.java
浏览文件 @ a7d2ed06
......@@ -4,9 +4,11 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store.btree;
package org.h2.test.store;
import java.nio.ByteBuffer;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataUtils;
/**
* An integer type.
......@@ -21,6 +23,14 @@ class IntegerType implements DataType {
return DataUtils.getVarIntLen((Integer) obj);
}
public int getMaxLength(Object obj) {
return DataUtils.MAX_VAR_INT_LEN;
}
public int getMemory(Object obj) {
return 20;
}
public Integer read(ByteBuffer buff) {
return DataUtils.readVarInt(buff);
}
......
h2/src/test/org/h2/test/store/RowType.java 0 → 100644
浏览文件 @ a7d2ed06
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.nio.ByteBuffer;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataTypeFactory;
import org.h2.dev.store.btree.DataUtils;
import org.h2.util.StringUtils;
/**
* A row type.
*/
public class RowType implements DataType {
private final DataType[] types;
private RowType(DataType[] types) {
this.types = types;
}
public int compare(Object a, Object b) {
if (a == b) {
return 0;
}
Object[] ax = (Object[]) a;
Object[] bx = (Object[]) b;
int al = ax.length;
int bl = bx.length;
int len = Math.min(al, bl);
for (int i = 0; i < len; i++) {
int comp = types[i].compare(ax[i], bx[i]);
if (comp != 0) {
return comp;
}
}
if (len < al) {
return -1;
} else if (len < bl) {
return 1;
}
return 0;
}
public int length(Object obj) {
Object[] x = (Object[]) obj;
int len = x.length;
int result = DataUtils.getVarIntLen(len);
for (int i = 0; i < len; i++) {
result += types[i].length(x[i]);
}
return result;
}
public int getMaxLength(Object obj) {
Object[] x = (Object[]) obj;
int len = x.length;
int result = DataUtils.MAX_VAR_INT_LEN;
for (int i = 0; i < len; i++) {
result += types[i].getMaxLength(x[i]);
}
return result;
}
public int getMemory(Object obj) {
Object[] x = (Object[]) obj;
int len = x.length;
int memory = 0;
for (int i = 0; i < len; i++) {
memory += types[i].getMemory(x[i]);
}
return memory;
}
public Object[] read(ByteBuffer buff) {
int len = DataUtils.readVarInt(buff);
Object[] x = new Object[len];
for (int i = 0; i < len; i++) {
x[i] = types[i].read(buff);
}
return x;
}
public void write(ByteBuffer buff, Object obj) {
Object[] x = (Object[]) obj;
int len = x.length;
DataUtils.writeVarInt(buff, len);
for (int i = 0; i < len; i++) {
types[i].write(buff, x[i]);
}
}
public String asString() {
StringBuilder buff = new StringBuilder();
buff.append('r');
buff.append('(');
for (int i = 0; i < types.length; i++) {
if (i > 0) {
buff.append(',');
}
buff.append(types[i].asString());
}
buff.append(')');
return buff.toString();
}
static RowType fromString(String t, DataTypeFactory factory) {
if (!t.startsWith("r(") || !t.endsWith(")")) {
throw new RuntimeException("Unknown type: " + t);
}
t = t.substring(2, t.length() - 1);
String[] array = StringUtils.arraySplit(t, ',', false);
DataType[] types = new DataType[array.length];
for (int i = 0; i < array.length; i++) {
types[i] = factory.fromString(array[i]);
}
return new RowType(types);
}
}
h2/src/test/org/h2/test/store/TestTreeMapStore.java
浏览文件 @ a7d2ed06
......@@ -29,6 +29,7 @@ public class TestTreeMapStore extends TestBase {
}
public void test() throws Exception {
testLargeImport();
testBtreeStore();
testDefragment();
testReuseSpace();
......@@ -38,10 +39,39 @@ public class TestTreeMapStore extends TestBase {
testSimple();
}
private void testLargeImport() throws Exception {
String fileName = getBaseDir() + "/testCsv.h3";
int len = 10000;
for (int j = 0; j < 5; j++) {
FileUtils.delete(fileName);
BtreeMapStore s = openStore(fileName);
s.setMaxPageSize(40);
RowType rowType = RowType.fromString("r(i,,)", new TestTypeFactory());
BtreeMap<Integer, Object[]> m = s.openMap("data", new IntegerType(), rowType);
int i = 0;
// long t = System.currentTimeMillis();
for (; i < len;) {
Object[] o = new Object[3];
o[0] = i;
o[1] = "Hello";
o[2] = "World";
m.put(i, o);
i++;
if (i % 10000 == 0) {
s.store();
}
}
s.store();
s.close();
// System.out.println("store time " + (System.currentTimeMillis() - t));
// System.out.println("store size " + FileUtils.size(fileName));
}
}
private void testBtreeStore() {
String fileName = getBaseDir() + "/testBtreeStore.h3";
FileUtils.delete(fileName);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
int count = 2000;
// Profiler p = new Profiler();
......@@ -64,8 +94,9 @@ public class TestTreeMapStore extends TestBase {
for (int i = 1; i < count; i++) {
assertEquals("hello " + i, m.get(i));
}
s.store();
s.close();
s = BtreeMapStore.open(fileName);
s = openStore(fileName);
m = s.openMap("data", Integer.class, String.class);
assertNull(m.get(0));
for (int i = 1; i < count; i++) {
......@@ -87,7 +118,7 @@ public class TestTreeMapStore extends TestBase {
FileUtils.delete(fileName);
long initialLength = 0;
for (int j = 0; j < 20; j++) {
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
for (int i = 0; i < 10; i++) {
m.put(j + i, "Hello " + j);
......@@ -103,9 +134,9 @@ public class TestTreeMapStore extends TestBase {
assertTrue("initial: " + initialLength + " len: " + len, len <= initialLength * 3);
}
}
long len = FileUtils.size(fileName);
// long len = FileUtils.size(fileName);
// System.out.println("len0: " + len);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
for (int i = 0; i < 100; i++) {
m.remove(i);
......@@ -113,13 +144,13 @@ public class TestTreeMapStore extends TestBase {
s.store();
s.compact();
s.close();
len = FileUtils.size(fileName);
// len = FileUtils.size(fileName);
// System.out.println("len1: " + len);
s = BtreeMapStore.open(fileName);
s = openStore(fileName);
m = s.openMap("data", Integer.class, String.class);
s.compact();
s.close();
len = FileUtils.size(fileName);
// len = FileUtils.size(fileName);
// System.out.println("len2: " + len);
}
......@@ -127,14 +158,15 @@ public class TestTreeMapStore extends TestBase {
String fileName = getBaseDir() + "/testReuseSpace.h3";
FileUtils.delete(fileName);
long initialLength = 0;
for (int j = 0; j < 10; j++) {
BtreeMapStore s = BtreeMapStore.open(fileName);
for (int j = 0; j < 20; j++) {
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
for (int i = 0; i < 10; i++) {
m.put(i, "Hello");
}
s.store();
for (int i = 0; i < 10; i++) {
assertEquals("Hello", m.get(i));
m.remove(i);
}
s.store();
......@@ -151,23 +183,25 @@ public class TestTreeMapStore extends TestBase {
private void testRandom() {
String fileName = getBaseDir() + "/testRandom.h3";
FileUtils.delete(fileName);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, Integer> m = s.openMap("data", Integer.class, Integer.class);
TreeMap<Integer, Integer> map = new TreeMap<Integer, Integer>();
Random r = new Random(1);
int operationCount = 1000;
int maxValue = 20;
int operationCount = 10000;
int maxValue = 30;
for (int i = 0; i < operationCount; i++) {
int k = r.nextInt(maxValue);
int v = r.nextInt();
boolean compareAll;
switch (r.nextInt(3)) {
case 0:
log(i + ": put " + k + " = " + v);
m.put(k, v);
map.put(k, v);
compareAll = true;
break;
case 1:
log(i + ": remove " + k);
m.remove(k);
map.remove(k);
compareAll = true;
......@@ -185,10 +219,12 @@ public class TestTreeMapStore extends TestBase {
}
if (compareAll) {
Iterator<Integer> it = m.keyIterator(null);
Iterator<Integer> it2 = map.keySet().iterator();
while (it2.hasNext()) {
Iterator<Integer> itExpected = map.keySet().iterator();
while (itExpected.hasNext()) {
assertTrue(it.hasNext());
assertEquals(it2.next(), it.next());
Integer expected = itExpected.next();
Integer got = it.next();
assertEquals(expected, got);
}
assertFalse(it.hasNext());
}
......@@ -199,7 +235,7 @@ public class TestTreeMapStore extends TestBase {
private void testKeyValueClasses() {
String fileName = getBaseDir() + "/testKeyValueClasses.h3";
FileUtils.delete(fileName);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
is.put(1, "Hello");
BtreeMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
......@@ -220,8 +256,9 @@ public class TestTreeMapStore extends TestBase {
} catch (RuntimeException e) {
// expected
}
s.store();
s.close();
s = BtreeMapStore.open(fileName);
s = openStore(fileName);
is = s.openMap("intString", Integer.class, String.class);
assertEquals("Hello", is.get(1));
ii = s.openMap("intInt", Integer.class, Integer.class);
......@@ -236,7 +273,7 @@ public class TestTreeMapStore extends TestBase {
private void testIterate() {
String fileName = getBaseDir() + "/testIterate.h3";
FileUtils.delete(fileName);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
Iterator<Integer> it = m.keyIterator(null);
assertFalse(it.hasNext());
......@@ -269,7 +306,7 @@ public class TestTreeMapStore extends TestBase {
private void testSimple() {
String fileName = getBaseDir() + "/testSimple.h3";
FileUtils.delete(fileName);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMapStore s = openStore(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
for (int i = 0; i < 3; i++) {
m.put(i, "hello " + i);
......@@ -281,10 +318,10 @@ public class TestTreeMapStore extends TestBase {
for (int i = 1; i < 3; i++) {
assertEquals("hello " + i, m.get(i));
}
s.store();
s.close();
s = BtreeMapStore.open(fileName);
s = openStore(fileName);
m = s.openMap("data", Integer.class, String.class);
assertNull(m.get(0));
for (int i = 1; i < 3; i++) {
......@@ -293,4 +330,19 @@ public class TestTreeMapStore extends TestBase {
s.close();
}
private static BtreeMapStore openStore(String fileName) {
BtreeMapStore store = BtreeMapStore.open(fileName, new TestTypeFactory());
store.setMaxPageSize(10);
return store;
}
/**
* Log the message.
*
* @param msg the message
*/
private static void log(String msg) {
// System.out.println(msg);
}
}
h2/src/test/org/h2/test/store/TestTypeFactory.java 0 → 100644
浏览文件 @ a7d2ed06
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License, Version
* 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html). Initial Developer: H2 Group
*/
package org.h2.test.store;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataTypeFactory;
import org.h2.dev.store.btree.StringType;
/**
* A data type factory.
*/
public class TestTypeFactory implements DataTypeFactory {
public DataType fromString(String s) {
if (s.length() == 0) {
return new StringType();
}
switch (s.charAt(0)) {
case 'i':
return new IntegerType();
case 'r':
return RowType.fromString(s, this);
}
throw new RuntimeException("Unknown data type " + s);
}
public DataType getDataType(Class<?> objectClass) {
if (objectClass == Integer.class) {
return new IntegerType();
}
throw new RuntimeException("Unsupported object class " + objectClass.toString());
}
}
h2/src/test/org/h2/test/store/package.html 0 → 100644
浏览文件 @ a7d2ed06
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<!--
Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License, Version 1.0,
and under the Eclipse Public License, Version 1.0
(http://h2database.com/html/license.html).
Initial Developer: H2 Group
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /><title>
Javadoc package documentation
</title></head><body style="font: 9pt/130% Tahoma, Arial, Helvetica, sans-serif; font-weight: normal;"><p>
This package contains tests for the map store.
</p></body></html>
\ No newline at end of file
h2/src/tools/org/h2/dev/store/btree/BtreeMap.java
浏览文件 @ a7d2ed06
......@@ -17,13 +17,15 @@ import java.util.Iterator;
public class BtreeMap<K, V> {
private final BtreeMapStore store;
private final long id;
private final String name;
private final DataType keyType;
private final DataType valueType;
private Page root;
private BtreeMap(BtreeMapStore store, String name, DataType keyType, DataType valueType) {
private BtreeMap(BtreeMapStore store, long id, String name, DataType keyType, DataType valueType) {
this.store = store;
this.id = id;
this.name = name;
this.keyType = keyType;
this.valueType = valueType;
......@@ -40,8 +42,8 @@ public class BtreeMap<K, V> {
* @param valueClass the value class
* @return the map
*/
static <K, V> BtreeMap<K, V> open(BtreeMapStore store, String name, DataType keyType, DataType valueType) {
return new BtreeMap<K, V>(store, name, keyType, valueType);
static <K, V> BtreeMap<K, V> open(BtreeMapStore store, long id, String name, DataType keyType, DataType valueType) {
return new BtreeMap<K, V>(store, id, name, keyType, valueType);
}
/**
......@@ -51,9 +53,7 @@ public class BtreeMap<K, V> {
* @param data the value
*/
public void put(K key, V data) {
if (!isChanged()) {
store.markChanged(name, this);
}
markChanged();
root = Page.put(this, root, key, data);
}
......@@ -77,23 +77,32 @@ public class BtreeMap<K, V> {
* @param key the key
* @return the value, or null if not found
*/
public Page getPage(K key) {
Page getPage(K key) {
if (root == null) {
return null;
}
return root.findPage(key);
}
/**
* Remove all entries.
*/
public void clear() {
if (root != null) {
markChanged();
root.removeAllRecursive();
root = null;
}
}
/**
* Remove a key-value pair.
*
* @param key the key
*/
public void remove(K key) {
if (!isChanged()) {
store.markChanged(name, this);
}
if (root != null) {
markChanged();
root = Page.remove(root, key);
}
}
......@@ -107,6 +116,12 @@ public class BtreeMap<K, V> {
return root != null && root.getId() < 0;
}
private void markChanged() {
if (!isChanged()) {
store.markChanged(name, this);
}
}
/**
* Compare two keys.
*
......@@ -206,4 +221,12 @@ public class BtreeMap<K, V> {
return name;
}
int getMaxPageSize() {
return store.getMaxPageSize();
}
long getId() {
return id;
}
}
h2/src/tools/org/h2/dev/store/btree/BtreeMapStore.java
浏览文件 @ a7d2ed06
......@@ -6,8 +6,8 @@
*/
package org.h2.dev.store.btree;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringReader;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
......@@ -17,7 +17,6 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Properties;
import java.util.TreeMap;
import org.h2.dev.store.FilePathCache;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
......@@ -35,21 +34,21 @@ header
header:
# H3 store #
pageSize=4096
r=1
blockSize=4096
chunk:
'd' [id] [metaRootPos] data ...
1 byte: 'c'
4 bytes: length
4 bytes: chunk id (an incrementing number)
4 bytes: metaRootPos (relative to the chunk start)
data ...
todo:
- garbage collection
- use page checksums
- compress chunks
- encode length in pos (1=32, 2=128, 3=512,...)
- possibly encode the length in pos (1=32, 2=128, 3=512,...)
- floating header (avoid duplicate header)
for each chunk, store chunk (a counter)
......@@ -66,27 +65,35 @@ public class BtreeMapStore {
private static final StringType STRING_TYPE = new StringType();
private final String fileName;
private final DataTypeFactory typeFactory;
private FileChannel file;
private int pageSize = 4 * 1024;
private long rootBlockStart;
private HashMap<Long, Page> cache = SmallLRUCache.newInstance(50000);
private ArrayList<Page> temp = New.arrayList();
private TreeMap<Integer, Block> blocks = new TreeMap<Integer, Block>();
private BtreeMap<String, String> meta;
private int blockSize = 4 * 1024;
private long rootChunkPos;
private HashMap<Long, Page> cache = SmallLRUCache.newInstance(5000);
private HashMap<Long, Page> temp = New.hashMap();
private HashMap<Integer, Chunk> chunks = New.hashMap();
private HashMap<String, BtreeMap<?, ?>> maps = New.hashMap();
private HashMap<String, BtreeMap<?, ?>> mapsChanged = New.hashMap();
private BtreeMap<String, String> meta;
// TODO use an int instead? (with rollover to 0)
private long transaction;
private long tempPageId;
private int lastChunkId;
private int lastBlockId;
// TODO use bit set, and integer instead of long
private long lastMapId;
// TODO support quota (per map, per storage)
private int maxPageSize = 30;
// TODO support reading metadata to support quota (per map, per storage)
// TODO support r-tree
// TODO support triggers and events (possibly on a different layer)
private BtreeMapStore(String fileName) {
private BtreeMapStore(String fileName, DataTypeFactory typeFactory) {
this.fileName = fileName;
this.typeFactory = typeFactory;
}
/**
......@@ -96,7 +103,18 @@ public class BtreeMapStore {
* @return the store
*/
public static BtreeMapStore open(String fileName) {
BtreeMapStore s = new BtreeMapStore(fileName);
return open(fileName, null);
}
/**
* Open a tree store.
*
* @param fileName the file name
* @param typeFactory the type factory
* @return the store
*/
public static BtreeMapStore open(String fileName, DataTypeFactory typeFactory) {
BtreeMapStore s = new BtreeMapStore(fileName, typeFactory);
s.open();
return s;
}
......@@ -107,43 +125,74 @@ public class BtreeMapStore {
* @param <K> the key type
* @param <V> the value type
* @param name the name of the map
* @param keyClass the key class
* @param valueClass the value class
* @param keyType the key type
* @param valueType the value type
* @return the map
*/
public <K, V> BtreeMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
public <K, V> BtreeMap<K, V> openMap(String name, DataType keyType, DataType valueType) {
@SuppressWarnings("unchecked")
BtreeMap<K, V> m = (BtreeMap<K, V>) maps.get(name);
if (m == null) {
String root = meta.get("root." + name);
DataType keyType = getDataType(keyClass);
DataType valueType = getDataType(valueClass);
m = BtreeMap.open(this, name, keyType, valueType);
maps.put(name, m);
if (root == null) {
String info = m.getKeyType().asString() + "/" + m.getValueType().asString();
meta.put("map." + name, info);
String identifier = meta.get("map." + name);
long id;
String root;
if (identifier == null) {
id = ++lastMapId;
String types = id + "/" + keyType.asString() + "/" + valueType.asString();
meta.put("map." + name, types);
root = null;
} else {
if (!root.equals("0")) {
m.setRoot(Long.parseLong(root));
}
String types = meta.get("map." + name);
String[] idTypeList = StringUtils.arraySplit(types, '/', false);
id = Long.parseLong(idTypeList[0]);
keyType = getDataType(idTypeList[1]);
valueType = getDataType(idTypeList[2]);
root = meta.get("root." + id);
}
m = BtreeMap.open(this, id, name, keyType, valueType);
maps.put(name, m);
if (root != null && !"0".equals(root)) {
m.setRoot(Long.parseLong(root));
}
}
return m;
}
/**
* Open a map.
*
* @param <K> the key type
* @param <V> the value type
* @param name the name of the map
* @param keyClass the key class
* @param valueClass the value class
* @return the map
*/
public <K, V> BtreeMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
DataType keyType = getDataType(keyClass);
DataType valueType = getDataType(valueClass);
return openMap(name, keyType, valueType);
}
private DataType getDataType(Class<?> clazz) {
if (clazz == String.class) {
return STRING_TYPE;
}
return DataTypeFactory.getDataType(clazz);
return getTypeFactory().getDataType(clazz);
}
private DataType getDataType(String s) {
if (s.equals("")) {
return STRING_TYPE;
}
return DataTypeFactory.fromString(s);
return getTypeFactory().fromString(s);
}
private DataTypeFactory getTypeFactory() {
if (typeFactory == null) {
throw new RuntimeException("No data type factory set");
}
return typeFactory;
}
/**
......@@ -159,7 +208,7 @@ public class BtreeMapStore {
}
private void open() {
meta = BtreeMap.open(this, "meta", STRING_TYPE, STRING_TYPE);
meta = BtreeMap.open(this, 0, "meta", STRING_TYPE, STRING_TYPE);
FileUtils.createDirectories(FileUtils.getParent(fileName));
try {
log("file open");
......@@ -176,21 +225,24 @@ public class BtreeMapStore {
}
private void readMeta() {
long rootId = readMetaRootId(rootBlockStart);
lastBlockId = getBlockId(rootId);
Block b = new Block(lastBlockId);
b.start = rootBlockStart;
blocks.put(b.id, b);
meta.setRoot(rootId);
Iterator<String> it = meta.keyIterator("block.");
Chunk header = readChunkHeader(rootChunkPos);
lastChunkId = header.id;
chunks.put(header.id, header);
meta.setRoot(getId(header.id, header.metaRootOffset));
Iterator<String> it = meta.keyIterator("chunk.");
while (it.hasNext()) {
String s = it.next();
if (!s.startsWith("block.")) {
if (!s.startsWith("chunk.")) {
break;
}
b = Block.fromString(meta.get(s));
lastBlockId = Math.max(b.id, lastBlockId);
blocks.put(b.id, b);
Chunk c = Chunk.fromString(meta.get(s));
if (c.id == header.id) {
c.start = header.start;
c.length = header.length;
c.metaRootOffset = header.metaRootOffset;
}
lastChunkId = Math.max(c.id, lastChunkId);
chunks.put(c.id, c);
}
}
......@@ -198,13 +250,15 @@ public class BtreeMapStore {
try {
ByteBuffer header = ByteBuffer.wrap((
"# H2 1.5\n" +
"read-version: 1\n" +
"write-version: 1\n" +
"rootBlock: " + rootBlockStart + "\n" +
"transaction: " + transaction + "\n").getBytes());
"versionRead:1\n" +
"versionWrite:1\n" +
"blockSize:" + blockSize + "\n" +
"rootChunk:" + rootChunkPos + "\n" +
"lastMapId:" + lastMapId + "\n" +
"transaction:" + transaction + "\n").getBytes("UTF-8"));
file.position(0);
file.write(header);
file.position(pageSize);
file.position(blockSize);
file.write(header);
} catch (Exception e) {
throw convert(e);
......@@ -214,13 +268,14 @@ public class BtreeMapStore {
private void readHeader() {
try {
file.position(0);
byte[] header = new byte[pageSize];
byte[] header = new byte[blockSize];
// TODO read fully; read both headers
file.read(ByteBuffer.wrap(header));
Properties prop = new Properties();
prop.load(new ByteArrayInputStream(header));
rootBlockStart = Long.parseLong(prop.get("rootBlock").toString());
prop.load(new StringReader(new String(header, "UTF-8")));
rootChunkPos = Long.parseLong(prop.get("rootChunk").toString());
transaction = Long.parseLong(prop.get("transaction").toString());
lastMapId = Long.parseLong(prop.get("lastMapId").toString());
} catch (Exception e) {
throw convert(e);
}
......@@ -234,7 +289,6 @@ public class BtreeMapStore {
* Close the file. Uncommitted changes are ignored.
*/
public void close() {
store();
if (file != null) {
try {
log("file close");
......@@ -246,18 +300,18 @@ public class BtreeMapStore {
}
}
private long getPosition(long pageId) {
Block b = getBlock(pageId);
if (b == null) {
throw new RuntimeException("Block " + getBlockId(pageId) + " not found");
private long getPosition(long posId) {
Chunk c = getChunk(posId);
if (c == null) {
throw new RuntimeException("Chunk " + getChunkId(posId) + " not found");
}
long pos = b.start;
pos += (int) (pageId & Integer.MAX_VALUE);
long pos = c.start;
pos += (int) (posId & Integer.MAX_VALUE);
return pos;
}
private static long getId(int blockId, int offset) {
return ((long) blockId << 32) | offset;
private static long getId(int chunkId, int offset) {
return ((long) chunkId << 32) | offset;
}
/**
......@@ -272,95 +326,74 @@ public class BtreeMapStore {
}
long trans = commit();
// the length estimate is not correct,
// as we don't know the exact positions and entry counts
int lenEstimate = 1 + 8;
for (BtreeMap<?, ?> m : mapsChanged.values()) {
meta.put("root." + m.getName(), String.valueOf(Long.MAX_VALUE));
Page p = m.getRoot();
if (p != null) {
lenEstimate += p.lengthIncludingTempChildren();
}
}
int blockId = ++lastBlockId;
Block b = new Block(blockId);
b.start = Long.MAX_VALUE;
b.entryCount = Integer.MAX_VALUE;
b.liveCount = Integer.MAX_VALUE;
blocks.put(b.id, b);
for (Block x : blocks.values()) {
int chunkId = ++lastChunkId;
Chunk c = new Chunk(chunkId);
c.entryCount = Integer.MAX_VALUE;
c.liveCount = Integer.MAX_VALUE;
c.start = Long.MAX_VALUE;
c.length = Long.MAX_VALUE;
chunks.put(c.id, c);
meta.put("chunk." + c.id, c.toString());
ArrayList<Integer> removedChunks = New.arrayList();
for (Chunk x : chunks.values()) {
if (x.liveCount == 0) {
meta.remove("block." + x.id);
meta.remove("chunk." + x.id);
removedChunks.add(x.id);
} else {
meta.put("block." + x.id, "temp-" + x.toString());
meta.put("chunk." + x.id, x.toString());
}
}
// modifying the meta can itself affect the metadata
// TODO solve this in a better way
ArrayList<Integer> removedBlocks = New.arrayList();
for (Block x : new ArrayList<Block>(blocks.values())) {
if (x.liveCount == 0) {
meta.remove("block." + x.id);
removedBlocks.add(x.id);
} else {
meta.put("block." + x.id, x.toString());
}
}
lenEstimate += meta.getRoot().lengthIncludingTempChildren();
b.length = lenEstimate;
blocks.remove(b.id);
long storePos = allocateBlock(lenEstimate);
blocks.put(b.id, b);
for (int id : removedBlocks) {
blocks.remove(id);
}
long pageId = getId(blockId, 1 + 8);
for (BtreeMap<?, ?> m : mapsChanged.values()) {
Page r = m.getRoot();
long p = r == null ? 0 : pageId;
meta.put("root." + m.getName(), "" + p);
if (r != null) {
pageId = r.updatePageIds(pageId);
}
for (int x : removedChunks) {
chunks.remove(x);
}
int metaRootOffset = (int) (pageId - getId(blockId, 0));
// add a dummy entry so the count is correct
meta.put("block." + b.id, b.toString());
int count = 0;
int maxLength = 1 + 4 + 4 + 4;
for (BtreeMap<?, ?> m : mapsChanged.values()) {
Page p = m.getRoot();
if (p != null) {
count += p.countTemp();
maxLength += p.getMaxLengthTempRecursive();
count += p.countTempRecursive();
meta.put("root." + m.getId(), String.valueOf(Long.MAX_VALUE));
} else {
meta.put("root." + m.getId(), "0");
}
}
count += meta.getRoot().countTemp();
b.start = storePos;
b.entryCount = count;
b.liveCount = b.entryCount;
meta.put("block." + b.id, b.toString());
pageId = meta.getRoot().updatePageIds(pageId);
int len = (int) (pageId - getId(blockId, 0));
maxLength += meta.getRoot().getMaxLengthTempRecursive();
count += meta.getRoot().countTempRecursive();
ByteBuffer buff = ByteBuffer.allocate(len);
buff.put((byte) 'd');
buff.putInt(b.id);
buff.putInt(metaRootOffset);
ByteBuffer buff = ByteBuffer.allocate(maxLength);
// need to patch the header later
buff.put((byte) 'c');
buff.putInt(0);
buff.putInt(0);
buff.putInt(0);
long idOffset = getId(chunkId, 0);
for (BtreeMap<?, ?> m : mapsChanged.values()) {
Page p = m.getRoot();
if (p != null) {
p.storeTemp(buff);
long root = p.writeTempRecursive(buff, idOffset);
meta.put("root." + m.getId(), "" + root);
}
}
meta.getRoot().storeTemp(buff);
if (buff.hasRemaining()) {
throw new RuntimeException("remaining: " + buff.remaining());
}
// fix metadata
c.entryCount = count;
c.liveCount = count;
meta.put("chunk." + c.id, c.toString());
meta.getRoot().writeTempRecursive(buff, idOffset);
buff.flip();
int length = buff.limit();
long storePos = allocateChunk(length);
int rootOffset = (int) (meta.getRoot().getId() - idOffset);
buff.rewind();
buff.put((byte) 'c');
buff.putInt(length);
buff.putInt(chunkId);
buff.putInt(rootOffset);
buff.rewind();
try {
file.position(storePos);
......@@ -368,26 +401,33 @@ public class BtreeMapStore {
} catch (IOException e) {
throw new RuntimeException(e);
}
rootBlockStart = storePos;
rootChunkPos = storePos;
writeHeader();
mapsChanged.clear();
temp.clear();
tempPageId = 0;
// update the start position and length
c.start = storePos;
c.length = length;
meta.put("chunk." + c.id, c.toString());
return trans;
}
private long allocateBlock(long length) {
private long allocateChunk(long length) {
BitSet set = new BitSet();
set.set(0);
set.set(1);
for (Block b : blocks.values()) {
if (b.start == Long.MAX_VALUE) {
for (Chunk c : chunks.values()) {
if (c.start == Long.MAX_VALUE) {
continue;
}
int first = (int) (b.start / pageSize);
int last = (int) ((b.start + b.length) / pageSize);
int first = (int) (c.start / blockSize);
int last = (int) ((c.start + c.length) / blockSize);
set.set(first, last +1);
}
int required = (int) (length / pageSize) + 1;
int required = (int) (length / blockSize) + 1;
for (int i = 0; i < set.size(); i++) {
if (!set.get(i)) {
boolean ok = true;
......@@ -398,11 +438,11 @@ public class BtreeMapStore {
}
}
if (ok) {
return i * pageSize;
return i * blockSize;
}
}
}
return set.size() * pageSize;
return set.size() * blockSize;
}
/**
......@@ -421,8 +461,9 @@ public class BtreeMapStore {
* @return the page id
*/
long registerTempPage(Page p) {
temp.add(p);
return -temp.size();
long id = --tempPageId;
temp.put(id, p);
return id;
}
/**
......@@ -434,37 +475,42 @@ public class BtreeMapStore {
return ++transaction;
}
private long readMetaRootId(long blockStart) {
private Chunk readChunkHeader(long pos) {
try {
file.position(blockStart);
file.position(pos);
ByteBuffer buff = ByteBuffer.wrap(new byte[16]);
file.read(buff);
buff.rewind();
if (buff.get() != 'd') {
if (buff.get() != 'c') {
throw new RuntimeException("File corrupt");
}
int blockId = buff.getInt();
int length = buff.getInt();
int chunkId = buff.getInt();
int offset = buff.getInt();
return getId(blockId, offset);
Chunk c = new Chunk(chunkId);
c.start = pos;
c.length = length;
c.metaRootOffset = offset;
return c;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Try to reduce the file size. Blocks with a low number of live items will
* Try to reduce the file size. Chunks with a low number of live items will
* be re-written.
*/
public void compact() {
if (blocks.size() <= 1) {
if (chunks.size() <= 1) {
return;
}
long liveCountTotal = 0, entryCountTotal = 0;
for (Block b : blocks.values()) {
entryCountTotal += b.entryCount;
liveCountTotal += b.liveCount;
for (Chunk c : chunks.values()) {
entryCountTotal += c.entryCount;
liveCountTotal += c.liveCount;
}
int averageEntryCount = (int) (entryCountTotal / blocks.size());
int averageEntryCount = (int) (entryCountTotal / chunks.size());
if (entryCountTotal == 0) {
return;
}
......@@ -472,52 +518,51 @@ public class BtreeMapStore {
if (percentTotal > 80) {
return;
}
ArrayList<Block> old = New.arrayList();
for (Block b : blocks.values()) {
int age = lastBlockId - b.id + 1;
b.collectPriority = b.getFillRate() / age;
old.add(b);
ArrayList<Chunk> old = New.arrayList();
for (Chunk c : chunks.values()) {
int age = lastChunkId - c.id + 1;
c.collectPriority = c.getFillRate() / age;
old.add(c);
}
Collections.sort(old, new Comparator<Block>() {
public int compare(Block o1, Block o2) {
Collections.sort(old, new Comparator<Chunk>() {
public int compare(Chunk o1, Chunk o2) {
return new Integer(o1.collectPriority).compareTo(o2.collectPriority);
}
});
int moveCount = 0;
Block move = null;
for (Block b : old) {
if (moveCount + b.liveCount > averageEntryCount) {
Chunk move = null;
for (Chunk c : old) {
if (moveCount + c.liveCount > averageEntryCount) {
break;
}
log(" block " + b.id + " " + b.getFillRate() + "% full; prio=" + b.collectPriority);
moveCount += b.liveCount;
move = b;
log(" chunk " + c.id + " " + c.getFillRate() + "% full; prio=" + c.collectPriority);
moveCount += c.liveCount;
move = c;
}
boolean remove = false;
for (Iterator<Block> it = old.iterator(); it.hasNext();) {
Block b = it.next();
if (move == b) {
for (Iterator<Chunk> it = old.iterator(); it.hasNext();) {
Chunk c = it.next();
if (move == c) {
remove = true;
} else if (remove) {
it.remove();
}
}
long oldMetaRootId = readMetaRootId(move.start);
long offset = getPosition(oldMetaRootId);
log(" meta:" + move.id + "/" + offset + " start: " + move.start);
BtreeMap<String, String> oldMeta = BtreeMap.open(this, "old-meta", STRING_TYPE, STRING_TYPE);
oldMeta.setRoot(oldMetaRootId);
Chunk header = readChunkHeader(move.start);
log(" meta:" + move.id + "/" + header.metaRootOffset + " start: " + move.start);
BtreeMap<String, String> oldMeta = BtreeMap.open(this, 0, "old-meta", STRING_TYPE, STRING_TYPE);
oldMeta.setRoot(getId(header.id, header.metaRootOffset));
Iterator<String> it = oldMeta.keyIterator(null);
ArrayList<Integer> oldBlocks = New.arrayList();
ArrayList<Integer> oldChunks = New.arrayList();
while (it.hasNext()) {
String k = it.next();
String s = oldMeta.get(k);
log(" " + k + " " + s.replace('\n', ' '));
if (k.startsWith("block.")) {
Block b = Block.fromString(s);
if (!blocks.containsKey(b.id)) {
oldBlocks.add(b.id);
blocks.put(b.id, b);
if (k.startsWith("chunk.")) {
Chunk c = Chunk.fromString(s);
if (!chunks.containsKey(c.id)) {
oldChunks.add(c.id);
chunks.put(c.id, c);
}
continue;
}
......@@ -528,11 +573,12 @@ public class BtreeMapStore {
if (!maps.containsKey(k)) {
continue;
}
String[] types = StringUtils.arraySplit(s, '/', false);
DataType kt = getDataType(types[0]);
DataType vt = getDataType(types[1]);
long oldDataRoot = Long.parseLong(oldMeta.get("root." + k));
BtreeMap<?, ?> oldData = BtreeMap.open(this, "old-" + k, kt, vt);
String[] idTypesList = StringUtils.arraySplit(s, '/', false);
long id = Long.parseLong(idTypesList[0]);
DataType kt = getDataType(idTypesList[1]);
DataType vt = getDataType(idTypesList[2]);
long oldDataRoot = Long.parseLong(oldMeta.get("root." + id));
BtreeMap<?, ?> oldData = BtreeMap.open(this, id, "old-" + k, kt, vt);
if (oldDataRoot == 0) {
// no rows
} else {
......@@ -549,9 +595,9 @@ public class BtreeMapStore {
// temporarily changed - ok
// TODO move old data if changed temporarily?
} else {
Block b = getBlock(p.getId());
if (old.contains(b)) {
log(" move key:" + o + " block:" + b.id);
Chunk c = getChunk(p.getId());
if (old.contains(c)) {
log(" move key:" + o + " chunk:" + c.id);
Object value = data.get(o);
data.remove(o);
data.put(o, value);
......@@ -560,8 +606,8 @@ public class BtreeMapStore {
}
}
}
for (int o : oldBlocks) {
blocks.remove(o);
for (int o : oldChunks) {
chunks.remove(o);
}
}
......@@ -574,7 +620,7 @@ public class BtreeMapStore {
*/
Page readPage(BtreeMap<?, ?> map, long id) {
if (id < 0) {
return temp.get((int) (-id - 1));
return temp.get(id);
}
Page p = cache.get(id);
if (p == null) {
......@@ -606,19 +652,25 @@ public class BtreeMapStore {
*/
void removePage(long id) {
if (id > 0) {
if (getBlock(id).liveCount == 0) {
cache.remove(id);
if (getChunk(id).liveCount == 0) {
throw new RuntimeException("Negative live count: " + id);
}
getBlock(id).liveCount--;
getChunk(id).liveCount--;
} else {
temp.remove(id);
if (temp.size() == 0) {
tempPageId = 0;
}
}
}
private static int getBlockId(long pageId) {
return (int) (pageId >>> 32);
private static int getChunkId(long pos) {
return (int) (pos >>> 32);
}
private Block getBlock(long pageId) {
return blocks.get(getBlockId(pageId));
private Chunk getChunk(long pos) {
return chunks.get(getChunkId(pos));
}
/**
......@@ -631,4 +683,12 @@ public class BtreeMapStore {
// System.out.println(string);
}
public void setMaxPageSize(int maxPageSize) {
this.maxPageSize = maxPageSize;
}
int getMaxPageSize() {
return maxPageSize;
}
}
h2/src/tools/org/h2/dev/store/btree/Block.java → h2/src/tools/org/h2/dev/store/btree/Chunk.java
浏览文件 @ a7d2ed06
......@@ -11,12 +11,12 @@ import java.io.IOException;
import java.util.Properties;
/**
* A block of data.
* A chunk of data, containing one or multiple pages
*/
class Block {
class Chunk {
/**
* The block id.
* The chunk id.
*/
int id;
......@@ -26,7 +26,7 @@ class Block {
long start;
/**
* The length in bytes.
* The length in bytes (may be larger than the actual value).
*/
long length;
......@@ -45,7 +45,12 @@ class Block {
*/
int collectPriority;
Block(int id) {
/**
* The offset of the meta root.
*/
int metaRootOffset;
Chunk(int id) {
this.id = id;
}
......@@ -55,17 +60,18 @@ class Block {
* @param s the string
* @return the block
*/
static Block fromString(String s) {
Block b = new Block(0);
static Chunk fromString(String s) {
Chunk c = new Chunk(0);
Properties prop = new Properties();
try {
prop.load(new ByteArrayInputStream(s.getBytes("UTF-8")));
b.id = Integer.parseInt(prop.get("id").toString());
b.start = Long.parseLong(prop.get("start").toString());
b.length = Long.parseLong(prop.get("length").toString());
b.entryCount = Integer.parseInt(prop.get("entryCount").toString());
b.liveCount = Integer.parseInt(prop.get("liveCount").toString());
return b;
c.id = Integer.parseInt(prop.get("id").toString());
c.start = Long.parseLong(prop.get("start").toString());
c.length = Long.parseLong(prop.get("length").toString());
c.entryCount = Integer.parseInt(prop.get("entryCount").toString());
c.liveCount = Integer.parseInt(prop.get("liveCount").toString());
c.metaRootOffset = Integer.parseInt(prop.get("metaRoot").toString());
return c;
} catch (IOException e) {
throw new RuntimeException(e);
}
......@@ -80,7 +86,7 @@ class Block {
}
public boolean equals(Object o) {
return o instanceof Block && ((Block) o).id == id;
return o instanceof Chunk && ((Chunk) o).id == id;
}
public String toString() {
......@@ -89,7 +95,8 @@ class Block {
"start:" + start + "\n" +
"length:" + length + "\n" +
"entryCount:" + entryCount + "\n" +
"liveCount:" + liveCount + "\n";
"liveCount:" + liveCount + "\n" +
"metaRoot:" + metaRootOffset + "\n";
}
}
......
h2/src/tools/org/h2/dev/store/btree/Cursor.java
浏览文件 @ a7d2ed06
......@@ -11,8 +11,11 @@ import java.util.Iterator;
/**
* A cursor to iterate over elements in ascending order.
*
* @param <K> the key type
*/
class Cursor<K> implements Iterator<K> {
private ArrayList<CursorPos> parents = new ArrayList<CursorPos>();
private K current;
......
h2/src/tools/org/h2/dev/store/btree/DataType.java
浏览文件 @ a7d2ed06
......@@ -23,20 +23,37 @@ public interface DataType {
int compare(Object a, Object b);
/**
* Get the length in bytes.
* Get the length in bytes used to store an object.
*
* @param obj the object
* @return the length
*/
int length(Object obj);
/**
* Get the maximum length in bytes used to store an object. In many cases,
* this method can be faster than calculating the exact length.
*
* @param obj the object
* @return the maximum length
*/
int getMaxLength(Object obj);
/**
* Estimate the used memory in bytes.
*
* @param obj the object
* @return the used memory
*/
int getMemory(Object obj);
/**
* Write the object.
*
* @param buff the target buffer
* @param x the value
*/
void write(ByteBuffer buff, Object x);
void write(ByteBuffer buff, Object obj);
/**
* Read an object.
......
h2/src/tools/org/h2/dev/store/btree/DataTypeFactory.java
浏览文件 @ a7d2ed06
......@@ -6,34 +6,18 @@
*/
package org.h2.dev.store.btree;
import java.io.IOException;
import java.io.StringReader;
/**
* A factory for data types.
*/
public class DataTypeFactory {
public interface DataTypeFactory {
/**
* Read the data type.
*
* @param buff the buffer
* @param s the string
* @return the type
*/
static DataType fromString(String s) {
StringReader r = new StringReader(s);
char c;
try {
c = (char) r.read();
} catch (IOException e) {
throw new RuntimeException(e);
}
switch (c) {
case 'i':
return new IntegerType();
}
throw new RuntimeException("Unknown data type " + c);
}
DataType fromString(String s);
/**
* Get the data type object for the given class.
......@@ -41,13 +25,6 @@ public class DataTypeFactory {
* @param objectClass the class
* @return the data type object
*/
static DataType getDataType(Class<?> objectClass) {
if (objectClass == Integer.class) {
return new IntegerType();
} else if (objectClass == String.class) {
return new StringType();
}
throw new RuntimeException("Unsupported object class " + objectClass.toString());
}
DataType getDataType(Class<?> objectClass);
}
h2/src/tools/org/h2/dev/store/btree/DataUtils.java
浏览文件 @ a7d2ed06
......@@ -13,13 +13,18 @@ import java.nio.ByteBuffer;
*/
public class DataUtils {
/**
* The maximum length of a variable size int.
*/
public static final int MAX_VAR_INT_LEN = 5;
/**
* Get the length of the variable size int.
*
* @param x the value
* @return the length in bytes
*/
static int getVarIntLen(int x) {
public static int getVarIntLen(int x) {
if ((x & (-1 << 7)) == 0) {
return 1;
} else if ((x & (-1 << 14)) == 0) {
......@@ -55,7 +60,7 @@ public class DataUtils {
* @param buff the source buffer
* @return the value
*/
static int readVarInt(ByteBuffer buff) {
public static int readVarInt(ByteBuffer buff) {
int b = buff.get();
if (b >= 0) {
return b;
......@@ -111,7 +116,7 @@ public class DataUtils {
* @param buff the target buffer
* @param x the value
*/
static void writeVarInt(ByteBuffer buff, int x) {
public static void writeVarInt(ByteBuffer buff, int x) {
while ((x & ~0x7f) != 0) {
buff.put((byte) (0x80 | (x & 0x7f)));
x >>>= 7;
......@@ -133,4 +138,22 @@ public class DataUtils {
buff.put((byte) x);
}
static void copyWithGap(Object src, Object dst, int oldSize, int gapIndex) {
if (gapIndex > 0) {
System.arraycopy(src, 0, dst, 0, gapIndex);
}
if (gapIndex < oldSize) {
System.arraycopy(src, gapIndex, dst, gapIndex + 1, oldSize - gapIndex);
}
}
static void copyExcept(Object src, Object dst, int oldSize, int removeIndex) {
if (removeIndex > 0 && oldSize > 0) {
System.arraycopy(src, 0, dst, 0, removeIndex);
}
if (removeIndex < oldSize) {
System.arraycopy(src, removeIndex + 1, dst, removeIndex, oldSize - removeIndex - 1);
}
}
}
h2/src/tools/org/h2/dev/store/btree/Dump.java 0 → 100644
浏览文件 @ a7d2ed06
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store.btree;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringReader;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Arrays;
import java.util.Properties;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
/**
* Convert a database file to a human-readable text dump.
*/
public class Dump {
private static int blockSize = 4 * 1024;
public static void main(String... args) {
String fileName = "test.h3";
for (int i = 0; i < args.length; i++) {
if ("-file".equals(args[i])) {
fileName = args[++i];
}
}
dump(fileName);
}
public static void dump(String fileName) {
if (!FileUtils.exists(fileName)) {
System.out.println("File not found: " + fileName);
return;
}
FileChannel file = null;
try {
file = FilePath.get(fileName).open("r");
long fileLength = file.size();
file.position(0);
byte[] header = new byte[blockSize];
file.read(ByteBuffer.wrap(header));
Properties prop = new Properties();
prop.load(new ByteArrayInputStream(header));
prop.load(new StringReader(new String(header, "UTF-8")));
System.out.println("file " + fileName);
System.out.println(" length " + fileLength);
System.out.println(" " + prop);
ByteBuffer block = ByteBuffer.wrap(new byte[16]);
for (long pos = 0; pos < fileLength;) {
file.position(pos);
block.rewind();
FileUtils.readFully(file, block);
block.rewind();
if (block.get() != 'c') {
pos += blockSize;
continue;
}
int length = block.getInt();
int chunkId = block.getInt();
int metaRootOffset = block.getInt();
System.out.println(" chunk " + chunkId + " at " + pos +
" length " + length + " offset " + metaRootOffset);
ByteBuffer chunk = ByteBuffer.allocate(length);
file.position(pos);
FileUtils.readFully(file, chunk);
int p = block.position();
pos = (pos + length + blockSize) / blockSize * blockSize;
length -= p;
while (length > 0) {
chunk.position(p);
int len = chunk.getInt();
long mapId = chunk.getLong();
int type = chunk.get();
int count = DataUtils.readVarInt(chunk);
if (type == 1) {
long[] children = new long[count];
for (int i = 0; i < count; i++) {
children[i] = chunk.getLong();
}
System.out.println(" map " + mapId + " at " + p + " node, " + count + " children: " + Arrays.toString(children));
} else {
System.out.println(" map " + mapId + " at " + p + " leaf, " + count + " rows");
}
p += len;
length -= len;
}
}
} catch (IOException e) {
System.out.println("ERROR: " + e);
} finally {
try {
file.close();
} catch (IOException e) {
// ignore
}
}
System.out.println();
}
}
h2/src/tools/org/h2/dev/store/btree/Page.java
浏览文件 @ a7d2ed06
......@@ -10,19 +10,20 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
/**
* A btree page implementation.
* A btree page (a node or a leaf).
* <p>
* For nodes, the key at a given index is larger than the largest key of the
* child at the same index.
*/
class Page {
private static final int MAX_SIZE = 20;
public class Page {
private final BtreeMap<?, ?> map;
private long id;
private long storedId;
private long transaction;
private Object[] keys;
private Object[] values;
private long[] children;
private int cachedCompare;
private Page(BtreeMap<?, ?> map) {
this.map = map;
......@@ -57,7 +58,7 @@ class Page {
*/
static Page read(BtreeMap<?, ?> map, long id, ByteBuffer buff) {
Page p = new Page(map);
p.id = p.storedId = id;
p.id = id;
p.read(buff);
return p;
}
......@@ -72,6 +73,7 @@ class Page {
map.removePage(id);
Page p2 = create(map, keys, values, children);
p2.transaction = t;
p2.cachedCompare = cachedCompare;
return p2;
}
......@@ -153,18 +155,39 @@ class Page {
private int findKey(Object key) {
int low = 0, high = keys.length - 1;
int x = cachedCompare - 1;
if (x < 0 || x > high) {
x = (low + high) >>> 1;
}
while (low <= high) {
int x = (low + high) >>> 1;
int compare = map.compare(key, keys[x]);
if (compare > 0) {
low = x + 1;
} else if (compare < 0) {
high = x - 1;
} else {
cachedCompare = x + 1;
return x;
}
x = (low + high) >>> 1;
}
cachedCompare = low;
return -(low + 1);
// regular binary search (without caching)
// int low = 0, high = keys.length - 1;
// while (low <= high) {
// int x = (low + high) >>> 1;
// int compare = map.compare(key, keys[x]);
// if (compare > 0) {
// low = x + 1;
// } else if (compare < 0) {
// high = x - 1;
// } else {
// return x;
// }
// }
// return -(low + 1);
}
/**
......@@ -176,8 +199,8 @@ class Page {
*/
static void min(Page p, ArrayList<CursorPos> parents, Object key) {
while (p != null) {
int x = key == null ? 0 : p.findKey(key);
if (p.children != null) {
int x = key == null ? -1 : p.findKey(key);
if (x < 0) {
x = -x - 1;
} else {
......@@ -189,6 +212,7 @@ class Page {
parents.add(c);
p = p.map.readPage(p.children[x]);
} else {
int x = key == null ? 0 : p.findKey(key);
if (x < 0) {
x = -x - 1;
}
......@@ -207,7 +231,7 @@ class Page {
* @param parents the stack of parent page positions
* @return the next key
*/
public static Object nextKey(ArrayList<CursorPos> parents) {
static Object nextKey(ArrayList<CursorPos> parents) {
if (parents.size() == 0) {
return null;
}
......@@ -224,7 +248,7 @@ class Page {
return null;
}
p = parents.remove(parents.size() - 1);
index = p.index++;
index = ++p.index;
if (index < p.page.children.length) {
parents.add(p);
Page x = p.page;
......@@ -302,7 +326,7 @@ class Page {
parent.children[parentIndex] = p.id;
}
if (!p.isLeaf()) {
if (p.keyCount() >= MAX_SIZE) {
if (p.keyCount() >= map.getMaxPageSize()) {
// TODO almost duplicate code
int pos = p.keyCount() / 2;
Object k = p.keys[pos];
......@@ -331,7 +355,7 @@ class Page {
}
index = -index - 1;
p.insert(index, key, value, 0);
if (p.keyCount() >= MAX_SIZE) {
if (p.keyCount() >= map.getMaxPageSize()) {
int pos = p.keyCount() / 2;
Object k = p.keys[pos];
Page split = p.splitLeaf(pos);
......@@ -347,6 +371,8 @@ class Page {
}
if (index < 0) {
index = -index - 1;
} else {
index++;
}
parent = p;
parentIndex = index;
......@@ -356,6 +382,18 @@ class Page {
return top;
}
/**
* Remove this page and all child pages.
*/
void removeAllRecursive() {
if (children != null) {
for (long c : children) {
map.readPage(c).removeAllRecursive();
}
}
map.removePage(id);
}
/**
* Remove a key-value pair.
*
......@@ -368,6 +406,7 @@ class Page {
if (p.isLeaf()) {
if (index >= 0) {
if (p.keyCount() == 1) {
p.map.removePage(p.id);
return null;
}
p = p.copyOnWrite();
......@@ -380,6 +419,8 @@ class Page {
// node
if (index < 0) {
index = -index - 1;
} else {
index++;
}
Page c = p.map.readPage(p.children[index]);
Page c2 = remove(c, key);
......@@ -390,7 +431,8 @@ class Page {
p = p.copyOnWrite();
p.remove(index);
if (p.keyCount() == 0) {
p = c2;
p.map.removePage(p.id);
p = p.map.readPage(p.children[0]);
}
} else {
p = p.copyOnWrite();
......@@ -401,18 +443,18 @@ class Page {
private void insert(int index, Object key, Object value, long child) {
Object[] newKeys = new Object[keys.length + 1];
copyWithGap(keys, newKeys, keys.length, index);
DataUtils.copyWithGap(keys, newKeys, keys.length, index);
newKeys[index] = key;
keys = newKeys;
if (values != null) {
Object[] newValues = new Object[values.length + 1];
copyWithGap(values, newValues, values.length, index);
DataUtils.copyWithGap(values, newValues, values.length, index);
newValues[index] = value;
values = newValues;
}
if (children != null) {
long[] newChildren = new long[children.length + 1];
copyWithGap(children, newChildren, children.length, index + 1);
DataUtils.copyWithGap(children, newChildren, children.length, index + 1);
newChildren[index + 1] = child;
children = newChildren;
}
......@@ -420,21 +462,28 @@ class Page {
private void remove(int index) {
Object[] newKeys = new Object[keys.length - 1];
copyExcept(keys, newKeys, keys.length, index);
int keyIndex = index >= keys.length ? index - 1 : index;
DataUtils.copyExcept(keys, newKeys, keys.length, keyIndex);
keys = newKeys;
if (values != null) {
Object[] newValues = new Object[values.length - 1];
copyExcept(values, newValues, values.length, index);
DataUtils.copyExcept(values, newValues, values.length, index);
values = newValues;
}
if (children != null) {
long[] newChildren = new long[children.length - 1];
copyExcept(children, newChildren, children.length, index);
DataUtils.copyExcept(children, newChildren, children.length, index);
children = newChildren;
}
}
private void read(ByteBuffer buff) {
// len
buff.getInt();
long id = buff.getLong();
if (id != map.getId()) {
throw new RuntimeException("Page map id missmatch, expected " + map.getId() + " got " + id);
}
boolean node = buff.get() == 1;
if (node) {
int len = DataUtils.readVarInt(buff);
......@@ -442,9 +491,9 @@ class Page {
keys = new Object[len - 1];
for (int i = 0; i < len; i++) {
children[i] = buff.getLong();
if (i < keys.length) {
keys[i] = map.getKeyType().read(buff);
}
}
for (int i = 0; i < len - 1; i++) {
keys[i] = map.getKeyType().read(buff);
}
} else {
int len = DataUtils.readVarInt(buff);
......@@ -462,17 +511,19 @@ class Page {
*
* @param buff the target buffer
*/
void write(ByteBuffer buff) {
private void write(ByteBuffer buff) {
int pos = buff.position();
buff.putInt(0);
buff.putLong(map.getId());
if (children != null) {
buff.put((byte) 1);
int len = children.length;
DataUtils.writeVarInt(buff, len);
for (int i = 0; i < len; i++) {
long c = map.readPage(children[i]).storedId;
buff.putLong(c);
if (i < keys.length) {
map.getKeyType().write(buff, keys[i]);
}
buff.putLong(children[i]);
}
for (int i = 0; i < len - 1; i++) {
map.getKeyType().write(buff, keys[i]);
}
} else {
buff.put((byte) 0);
......@@ -483,66 +534,62 @@ class Page {
map.getValueType().write(buff, values[i]);
}
}
int len = buff.position() - pos;
buff.putInt(pos, len);
}
/**
* Get the length in bytes, including temporary children.
*
* @return the length
*/
int lengthIncludingTempChildren() {
int byteCount = length();
if (children != null) {
int len = children.length;
for (int i = 0; i < len; i++) {
long c = children[i];
if (c < 0) {
byteCount += map.readPage(c).lengthIncludingTempChildren();
}
}
}
return byteCount;
}
/**
* Update the page ids recursively.
* Get the maximum length in bytes to store temporary records, recursively.
*
* @param pageId the new page id
* @return the next page id
*/
long updatePageIds(long pageId) {
this.storedId = pageId;
pageId += length();
int getMaxLengthTempRecursive() {
int maxLength = 4 + 8 + 1;
if (children != null) {
int len = children.length;
maxLength += DataUtils.MAX_VAR_INT_LEN;
maxLength += 8 * len;
for (int i = 0; i < len - 1; i++) {
maxLength += map.getKeyType().getMaxLength(keys[i]);
}
for (int i = 0; i < len; i++) {
long c = children[i];
if (c < 0) {
pageId = map.readPage(c).updatePageIds(pageId);
maxLength += map.readPage(c).getMaxLengthTempRecursive();
}
}
} else {
int len = keys.length;
maxLength += DataUtils.MAX_VAR_INT_LEN;
for (int i = 0; i < len; i++) {
maxLength += map.getKeyType().getMaxLength(keys[i]);
maxLength += map.getValueType().getMaxLength(values[i]);
}
}
return pageId;
return maxLength;
}
/**
* Store this page.
* Store this page and all children that are changed,
* in reverse order, and update the id and child ids.
*
* @param buff the target buffer
* @param idOffset the offset of the id
* @return the page id
*/
long storeTemp(ByteBuffer buff) {
write(buff);
long writeTempRecursive(ByteBuffer buff, long idOffset) {
if (children != null) {
int len = children.length;
for (int i = 0; i < len; i++) {
long c = children[i];
if (c < 0) {
children[i] = map.readPage(c).storeTemp(buff);
children[i] = map.readPage(c).writeTempRecursive(buff, idOffset);
}
}
}
this.id = storedId;
this.id = idOffset + buff.position();
write(buff);
return id;
}
......@@ -551,63 +598,18 @@ class Page {
*
* @return the number of temporary pages
*/
int countTemp() {
int countTempRecursive() {
int count = 1;
if (children != null) {
int size = children.length;
for (int i = 0; i < size; i++) {
long c = children[i];
if (c < 0) {
count += map.readPage(c).countTemp();
count += map.readPage(c).countTempRecursive();
}
}
}
return count;
}
/**
* Get the length in bytes.
*
* @return the length
*/
int length() {
int byteCount = 1;
if (children != null) {
int len = children.length;
byteCount += DataUtils.getVarIntLen(len);
for (int i = 0; i < len; i++) {
byteCount += 8;
if (i < keys.length) {
byteCount += map.getKeyType().length(keys[i]);
}
}
} else {
int len = keys.length;
byteCount += DataUtils.getVarIntLen(len);
for (int i = 0; i < len; i++) {
byteCount += map.getKeyType().length(keys[i]);
byteCount += map.getValueType().length(values[i]);
}
}
return byteCount;
}
private static void copyWithGap(Object src, Object dst, int oldSize, int gapIndex) {
if (gapIndex > 0) {
System.arraycopy(src, 0, dst, 0, gapIndex);
}
if (gapIndex < oldSize) {
System.arraycopy(src, gapIndex, dst, gapIndex + 1, oldSize - gapIndex);
}
}
private static void copyExcept(Object src, Object dst, int oldSize, int removeIndex) {
if (removeIndex > 0 && oldSize > 0) {
System.arraycopy(src, 0, dst, 0, removeIndex);
}
if (removeIndex < oldSize) {
System.arraycopy(src, removeIndex + 1, dst, removeIndex, oldSize - removeIndex - 1);
}
}
}
h2/src/tools/org/h2/dev/store/btree/StringType.java
浏览文件 @ a7d2ed06
......@@ -11,39 +11,67 @@ import java.nio.ByteBuffer;
/**
* A string type.
*/
class StringType implements DataType {
public class StringType implements DataType {
public int compare(Object a, Object b) {
return a.toString().compareTo(b.toString());
}
public int length(Object obj) {
try {
byte[] bytes = obj.toString().getBytes("UTF-8");
return DataUtils.getVarIntLen(bytes.length) + bytes.length;
} catch (Exception e) {
throw new RuntimeException(e);
int plus = 0;
String s = obj.toString();
int len = s.length();
for (int i = 0; i < len; i++) {
char c = s.charAt(i);
if (c >= 0x800) {
plus += 2;
} else if (c >= 0x80) {
plus++;
}
}
return DataUtils.getVarIntLen(len) + len + plus;
}
public int getMaxLength(Object obj) {
return DataUtils.MAX_VAR_INT_LEN + obj.toString().length() * 3;
}
public int getMemory(Object obj) {
return obj.toString().length() * 2 + 48;
}
public String read(ByteBuffer buff) {
int len = DataUtils.readVarInt(buff);
byte[] bytes = new byte[len];
buff.get(bytes);
try {
return new String(bytes, "UTF-8");
} catch (Exception e) {
throw new RuntimeException(e);
char[] chars = new char[len];
for (int i = 0; i < len; i++) {
int x = buff.get() & 0xff;
if (x < 0x80) {
chars[i] = (char) x;
} else if (x >= 0xe0) {
chars[i] = (char) (((x & 0xf) << 12) + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f));
} else {
chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f));
}
}
return new String(chars);
}
public void write(ByteBuffer buff, Object x) {
try {
byte[] bytes = x.toString().getBytes("UTF-8");
DataUtils.writeVarInt(buff, bytes.length);
buff.put(bytes);
} catch (Exception e) {
throw new RuntimeException(e);
public void write(ByteBuffer buff, Object obj) {
String s = obj.toString();
int len = s.length();
DataUtils.writeVarInt(buff, len);
for (int i = 0; i < len; i++) {
int c = s.charAt(i);
if (c < 0x80) {
buff.put((byte) c);
} else if (c >= 0x800) {
buff.put((byte) (0xe0 | (c >> 12)));
buff.put((byte) (((c >> 6) & 0x3f)));
buff.put((byte) (c & 0x3f));
} else {
buff.put((byte) (0xc0 | (c >> 6)));
buff.put((byte) (c & 0x3f));
}
}
}
......
h2/src/tools/org/h2/dev/store/tree/StoredMap.java
浏览文件 @ a7d2ed06
......@@ -132,7 +132,7 @@ public class StoredMap<K, V> {
/**
* A value type.
*/
static interface ValueType {
interface ValueType {
/**
* Get the length in bytes.
......@@ -170,7 +170,7 @@ public class StoredMap<K, V> {
/**
* A key type.
*/
static interface KeyType extends ValueType {
interface KeyType extends ValueType {
/**
* Compare two keys.
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论