Skip to content

H
h2database
提交 5b811571 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

A persistent tree map (work in progress)

上级 b655f8e8
显示空白字符变更
内嵌 并排
正在显示 包含 33 行增加 和 1631 行删除
h2/src/test/org/h2/test/unit/TestTreeMapStore.java
浏览文件 @ 5b811571
...@@ -8,8 +8,10 @@ package org.h2.test.unit; ...@@ -8,8 +8,10 @@ package org.h2.test.unit;
import java.util.Iterator; import java.util.Iterator;
import java.util.Random; import java.util.Random;
import java.util.TreeMap; import java.util.TreeMap;
import org.h2.dev.store.StoredMap; import org.h2.dev.store.btree.BtreeMap;
import org.h2.dev.store.TreeMapStore; import org.h2.dev.store.btree.BtreeMapStore;
import org.h2.dev.store.tree.StoredMap;
import org.h2.dev.store.tree.TreeMapStore;
import org.h2.jaqu.bytecode.Null; import org.h2.jaqu.bytecode.Null;
import org.h2.store.fs.FileUtils; import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase; import org.h2.test.TestBase;
...@@ -29,6 +31,10 @@ public class TestTreeMapStore extends TestBase { ...@@ -29,6 +31,10 @@ public class TestTreeMapStore extends TestBase {
} }
public void test() throws Exception { public void test() throws Exception {
// btree
testBtreeStore();
// left leaning red-black tree
testDefragment(); testDefragment();
testReuseSpace(); testReuseSpace();
testRandom(); testRandom();
...@@ -37,6 +43,31 @@ public class TestTreeMapStore extends TestBase { ...@@ -37,6 +43,31 @@ public class TestTreeMapStore extends TestBase {
testSimple(); testSimple();
} }
private void testBtreeStore() {
String fileName = getBaseDir() + "/data.h3";
FileUtils.delete(fileName);
BtreeMapStore s = BtreeMapStore.open(fileName);
BtreeMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
int count = 5;
for (int i = 0; i < count; i++) {
m.put(i, "hello " + i);
}
s.store();
m.remove(0);
assertNull(m.get(0));
for (int i = 1; i < count; i++) {
assertEquals("hello " + i, m.get(i));
}
s.close();
// s = BtreeMapStore.open(fileName);
// m = s.openMap("data", Integer.class, String.class);
// assertNull(m.get(0));
// for (int i = 1; i < count; i++) {
// assertEquals("hello " + i, m.get(i));
// }
// s.close();
}
private void testDefragment() { private void testDefragment() {
String fileName = getBaseDir() + "/data.h3"; String fileName = getBaseDir() + "/data.h3";
FileUtils.delete(fileName); FileUtils.delete(fileName);
......
h2/src/tools/org/h2/dev/store/Node.java deleted 100644 → 0
浏览文件 @ b655f8e8
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store;
import java.nio.ByteBuffer;
/**
* A left-leaning red black tree implementation.
*/
class Node {
private static final int FLAG_BLACK = 1;
// private static final int FLAG_BACK_REFERENCES = 2;
private final StoredMap<?, ?> map;
private long id;
private long leftId, rightId;
private long transaction;
private Object key;
private Object data;
private Node left, right;
private int flags;
private Node(StoredMap<?, ?> map) {
this.map = map;
}
/**
* Create a new node.
*
* @param map the map
* @param key the key
* @param data the value
* @return the node
*/
static Node create(StoredMap<?, ?> map, Object key, Object data) {
Node n = new Node(map);
n.key = key;
n.data = data;
n.transaction = map.getTransaction();
n.id = map.nextTempNodeId();
return n;
}
/**
* Read a node.
*
* @param map the map
* @param id the node id
* @param buff the source buffer
* @return the node
*/
static Node read(StoredMap<?, ?> map, long id, ByteBuffer buff) {
Node n = new Node(map);
n.id = id;
n.read(buff);
return n;
}
/**
* Get the left child.
*
* @return the left child
*/
Node getLeft() {
if (left == null && leftId != 0) {
return map.readNode(leftId);
}
return left;
}
/**
* Get the right child.
*
* @return the right child
*/
Node getRight() {
if (right == null && rightId != 0) {
return map.readNode(rightId);
}
return right;
}
/**
* Get the node id of the left child.
*
* @return the node id
*/
long getLeftId() {
return leftId;
}
/**
* Set the node id of the left child.
*
* @param leftId the node id
*/
void setLeftId(long leftId) {
this.leftId = leftId;
left = null;
}
/**
* Get the node id of the right child.
*
* @return the node id
*/
long getRightId() {
return rightId;
}
/**
* Set the node id of the right child.
*
* @param rightId the node id
*/
void setRightId(long rightId) {
this.rightId = rightId;
left = null;
}
private void setLeft(Node l) {
this.left = l;
this.leftId = l == null ? 0 : l.getId();
}
private void setRight(Node r) {
this.right = r;
this.rightId = r == null ? 0 : r.getId();
}
private Node copyOnWrite() {
if (transaction == map.getTransaction()) {
return this;
}
map.removeNode(id);
Node n2 = create(map, key, data);
n2.leftId = leftId;
n2.left = left;
n2.rightId = rightId;
n2.right = right;
n2.flags = flags;
return n2;
}
public String toString() {
StringBuilder buff = new StringBuilder();
buff.append(key);
if (left != null || right != null || leftId != 0 || rightId != 0) {
buff.append("{");
if (left != null) {
buff.append(left.toString());
} else if (leftId != 0) {
buff.append(leftId);
}
buff.append(",");
if (right != null) {
buff.append(right.toString());
} else if (rightId != 0) {
buff.append(rightId);
}
buff.append("}");
}
return buff.toString();
}
private void flipColor() {
flags = flags ^ FLAG_BLACK;
setLeft(getLeft().copyOnWrite());
getLeft().flags = getLeft().flags ^ FLAG_BLACK;
setRight(getRight().copyOnWrite());
getRight().flags = getRight().flags ^ FLAG_BLACK;
}
/**
* Get the node id.
*
* @return the node id
*/
long getId() {
return id;
}
/**
* Set the node id.
*
* @param id the new id
*/
void setId(long id) {
this.id = id;
}
/**
* Get the key.
*
* @return the key
*/
Object getKey() {
return key;
}
/**
* Get the value.
*
* @return the value
*/
Object getData() {
return data;
}
private Node rotateLeft() {
Node x = getRight().copyOnWrite();
setRight(x.getLeft());
x.setLeft(this);
x.flags = flags;
// make red
flags = flags & ~FLAG_BLACK;
return x;
}
private Node rotateRight() {
Node x = getLeft().copyOnWrite();
setLeft(x.getRight());
x.setRight(this);
x.flags = flags;
// make red
flags = flags & ~FLAG_BLACK;
return x;
}
private Node moveRedLeft() {
flipColor();
if (isRed(getRight().getLeft())) {
setRight(getRight().rotateRight());
Node n = rotateLeft();
n.flipColor();
return n;
}
return this;
}
private Node moveRedRight() {
flipColor();
if (isRed(getLeft().getLeft())) {
Node n = rotateRight();
n.flipColor();
return n;
}
return this;
}
private Node getMin() {
Node n = this;
while (n.getLeft() != null) {
n = n.getLeft();
}
return n;
}
private Node removeMin() {
if (getLeft() == null) {
map.removeNode(id);
return null;
}
Node n = copyOnWrite();
if (!isRed(n.getLeft()) && !isRed(n.getLeft().getLeft())) {
n = n.moveRedLeft();
}
n.setLeft(n.getLeft().removeMin());
return n.fixUp();
}
/**
* Remove the node.
*
* @param n the root node
* @param key the key
* @return the new root node
*/
static Node remove(Node n, Object key) {
if (getNode(n, key) == null) {
return n;
}
return n.remove(key);
}
/**
* Compare the key with the key of this node.
*
* @param key the key
* @return -1 if the key is smaller than this nodes key, 1 if bigger, and 0
* if equal
*/
int compare(Object key) {
return map.compare(key, this.key);
}
private Node remove(Object key) {
Node n = copyOnWrite();
if (map.compare(key, n.key) < 0) {
if (!isRed(n.getLeft()) && !isRed(n.getLeft().getLeft())) {
n = n.moveRedLeft();
}
n.setLeft(n.getLeft().remove(key));
} else {
if (isRed(n.getLeft())) {
n = n.rotateRight();
}
if (n.compare(key) == 0 && n.getRight() == null) {
map.removeNode(id);
return null;
}
if (!isRed(n.getRight()) && !isRed(n.getRight().getLeft())) {
n = n.moveRedRight();
}
if (n.compare(key) == 0) {
Node min = n.getRight().getMin();
n.key = min.key;
n.data = min.data;
n.setRight(n.getRight().removeMin());
} else {
n.setRight(n.getRight().remove(key));
}
}
return n.fixUp();
}
/**
* Get the node.
*
* @param n the root
* @param key the key
* @return the node, or null
*/
static Node getNode(Node n, Object key) {
while (n != null) {
int compare = n.compare(key);
if (compare == 0) {
return n;
} else if (compare > 0) {
n = n.getRight();
} else {
n = n.getLeft();
}
}
return null;
}
/**
* Put the node in the map.
*
* @param map the map
* @param n the node
* @param key the key
* @param data the value
* @return the root node
*/
static Node put(StoredMap<?, ?> map, Node n, Object key, Object data) {
if (n == null) {
n = Node.create(map, key, data);
return n;
}
n = n.copyOnWrite();
int compare = n.compare(key);
if (compare == 0) {
n.data = data;
} else if (compare < 0) {
n.setLeft(put(map, n.getLeft(), key, data));
} else {
n.setRight(put(map, n.getRight(), key, data));
}
return n.fixUp();
}
private Node fixUp() {
Node n = this;
if (isRed(getRight())) {
n = rotateLeft();
}
if (isRed(n.getLeft()) && isRed(n.getLeft().getLeft())) {
n = n.rotateRight();
}
if (isRed(n.getLeft()) && isRed(n.getRight())) {
n.flipColor();
}
return n;
}
private boolean isRed(Node n) {
return n != null && (n.flags & FLAG_BLACK) == 0;
}
private void read(ByteBuffer buff) {
flags = buff.get();
leftId = buff.getLong();
rightId = buff.getLong();
key = map.getKeyType().read(buff);
data = map.getValueType().read(buff);
}
/**
* Store the node.
*
* @param buff the target buffer
*/
void write(ByteBuffer buff) {
buff.put((byte) flags);
buff.putLong(leftId);
buff.putLong(rightId);
map.getKeyType().write(buff, key);
map.getValueType().write(buff, data);
}
/**
* Get the length in bytes.
*
* @return the length
*/
int length() {
return map.getKeyType().length(key) +
map.getValueType().length(data) + 17;
}
}
h2/src/tools/org/h2/dev/store/StoredMap.java deleted 100644 → 0
浏览文件 @ b655f8e8
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Iterator;
/**
* A stored map.
*
* @param <K> the key class
* @param <V> the value class
*/
public class StoredMap<K, V> {
private final TreeMapStore store;
private final String name;
private final KeyType keyType;
private final ValueType valueType;
private Node root;
private StoredMap(TreeMapStore store, String name, Class<K> keyClass, Class<V> valueClass) {
this.store = store;
this.name = name;
if (keyClass == Integer.class) {
keyType = new IntegerType();
} else if (keyClass == String.class) {
keyType = new StringType();
} else {
throw new RuntimeException("Unsupported key class " + keyClass.toString());
}
if (valueClass == Integer.class) {
valueType = new IntegerType();
} else if (valueClass == String.class) {
valueType = new StringType();
} else {
throw new RuntimeException("Unsupported value class " + keyClass.toString());
}
}
/**
* Get the class with the given tag name.
*
* @param name the tag name
* @return the class
*/
static Class<?> getClass(String name) {
if (name.equals("i")) {
return Integer.class;
} else if (name.equals("s")) {
return String.class;
}
throw new RuntimeException("Unknown class name " + name);
}
/**
* Open a map.
*
* @param <K> the key type
* @param <V> the value type
* @param store the tree store
* @param name the name of the map
* @param keyClass the key class
* @param valueClass the value class
* @return the map
*/
static <K, V> StoredMap<K, V> open(TreeMapStore store, String name, Class<K> keyClass, Class<V> valueClass) {
return new StoredMap<K, V>(store, name, keyClass, valueClass);
}
/**
* Store a key-value pair.
*
* @param key the key
* @param data the value
*/
public void put(K key, V data) {
if (!isChanged()) {
store.markChanged(name, this);
}
root = Node.put(this, root, key, data);
}
/**
* Get a value.
*
* @param key the key
* @return the value
*/
@SuppressWarnings("unchecked")
public V get(K key) {
Node n = Node.getNode(root, key);
return (V) (n == null ? null : n.getData());
}
/**
* Get the node with the given key.
*
* @param key the key
* @return the node
*/
Node getNode(Object key) {
return Node.getNode(root, key);
}
/**
* Remove a key-value pair.
*
* @param key the key
*/
public void remove(K key) {
if (!isChanged()) {
store.markChanged(name, this);
}
root = Node.remove(root, key);
}
/**
* Was this map changed.
*
* @return true if yes
*/
boolean isChanged() {
return root != null && root.getId() < 0;
}
/**
* A value type.
*/
static interface ValueType {
/**
* Get the length in bytes.
*
* @param obj the object
* @return the length
*/
int length(Object obj);
/**
* Write the object.
*
* @param buff the target buffer
* @param x the value
*/
void write(ByteBuffer buff, Object x);
/**
* Read an object.
*
* @param buff the source buffer
* @return the object
*/
Object read(ByteBuffer buff);
/**
* Get the tag name of the class.
*
* @return the tag name
*/
String getName();
}
/**
* A key type.
*/
static interface KeyType extends ValueType {
/**
* Compare two keys.
*
* @param a the first key
* @param b the second key
* @return -1 if the first key is smaller, 1 if larger, and 0 if equal
*/
int compare(Object a, Object b);
}
/**
* Compare two keys.
*
* @param a the first key
* @param b the second key
* @return -1 if the first key is smaller, 1 if bigger, 0 if equal
*/
int compare(Object a, Object b) {
return keyType.compare(a, b);
}
/**
* An integer type.
*/
static class IntegerType implements KeyType {
public int compare(Object a, Object b) {
return ((Integer) a).compareTo((Integer) b);
}
public int length(Object obj) {
return getVarIntLen((Integer) obj);
}
public Integer read(ByteBuffer buff) {
return readVarInt(buff);
}
public void write(ByteBuffer buff, Object x) {
writeVarInt(buff, (Integer) x);
}
public String getName() {
return "i";
}
}
/**
* A string type.
*/
static class StringType implements KeyType {
public int compare(Object a, Object b) {
return a.toString().compareTo(b.toString());
}
public int length(Object obj) {
try {
byte[] bytes = obj.toString().getBytes("UTF-8");
return getVarIntLen(bytes.length) + bytes.length;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public String read(ByteBuffer buff) {
int len = readVarInt(buff);
byte[] bytes = new byte[len];
buff.get(bytes);
try {
return new String(bytes, "UTF-8");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void write(ByteBuffer buff, Object x) {
try {
byte[] bytes = x.toString().getBytes("UTF-8");
writeVarInt(buff, bytes.length);
buff.put(bytes);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public String getName() {
return "s";
}
}
/**
* Get the key type.
*
* @return the key type
*/
KeyType getKeyType() {
return keyType;
}
/**
* Get the value type.
*
* @return the value type
*/
ValueType getValueType() {
return valueType;
}
long getTransaction() {
return store.getTransaction();
}
/**
* Get the next temporary node id.
*
* @return the node id
*/
long nextTempNodeId() {
return store.nextTempNodeId();
}
/**
* Read a node.
*
* @param id the node id
* @return the node
*/
Node readNode(long id) {
return store.readNode(this, id);
}
/**
* Remove a node.
*
* @param id the node id
*/
void removeNode(long id) {
store.removeNode(id);
}
/**
* Set the position of the root node.
*
* @param rootPos the position
*/
void setRoot(long rootPos) {
root = readNode(rootPos);
}
/**
* Iterate over all keys.
*
* @param from the first key to return
* @return the iterator
*/
public Iterator<K> keyIterator(K from) {
return new Cursor(root, from);
}
/**
* A cursor to iterate over elements in ascending order.
*/
class Cursor implements Iterator<K> {
Node current;
ArrayList<Node> parents = new ArrayList<Node>();
Cursor(Node root, K from) {
min(root, from);
}
private void min(Node n, K key) {
while (n != null) {
int compare = key == null ? -1 : n.compare(key);
if (compare == 0) {
current = n;
return;
} else if (compare > 0) {
n = n.getRight();
} else {
parents.add(n);
n = n.getLeft();
}
}
if (parents.size() == 0) {
current = null;
return;
}
current = parents.remove(parents.size() - 1);
}
@SuppressWarnings("unchecked")
public K next() {
Node c = current;
if (c != null) {
fetchNext();
}
return c == null ? null : (K) c.getKey();
}
private void fetchNext() {
Node r = current.getRight();
if (r != null) {
min(r, null);
return;
}
if (parents.size() == 0) {
current = null;
return;
}
current = parents.remove(parents.size() - 1);
}
public boolean hasNext() {
return current != null;
}
public void remove() {
throw new UnsupportedOperationException();
}
}
/**
* Get the root node.
*
* @return the root node
*/
Node getRoot() {
return root;
}
/**
* Get the map name.
*
* @return the name
*/
String getName() {
return name;
}
/**
* Read a variable size int.
*
* @param buff the source buffer
* @return the value
*/
static int readVarInt(ByteBuffer buff) {
int b = buff.get();
if (b >= 0) {
return b;
}
// a separate function so that this one can be inlined
return readVarIntRest(buff, b);
}
private static int readVarIntRest(ByteBuffer buff, int b) {
int x = b & 0x7f;
b = buff.get();
if (b >= 0) {
return x | (b << 7);
}
x |= (b & 0x7f) << 7;
b = buff.get();
if (b >= 0) {
return x | (b << 14);
}
x |= (b & 0x7f) << 14;
b = buff.get();
if (b >= 0) {
return x | b << 21;
}
x |= ((b & 0x7f) << 21) | (buff.get() << 28);
return x;
}
/**
* Get the length of the variable size int.
*
* @param x the value
* @return the length in bytes
*/
static int getVarIntLen(int x) {
if ((x & (-1 << 7)) == 0) {
return 1;
} else if ((x & (-1 << 14)) == 0) {
return 2;
} else if ((x & (-1 << 21)) == 0) {
return 3;
} else if ((x & (-1 << 28)) == 0) {
return 4;
}
return 5;
}
/**
* Write a variable size int.
*
* @param buff the target buffer
* @param x the value
*/
static void writeVarInt(ByteBuffer buff, int x) {
while ((x & ~0x7f) != 0) {
buff.put((byte) (0x80 | (x & 0x7f)));
x >>>= 7;
}
buff.put((byte) x);
}
}
h2/src/tools/org/h2/dev/store/TreeMapStore.java deleted 100644 → 0
浏览文件 @ b655f8e8
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Properties;
import java.util.TreeMap;
import org.h2.store.fs.FilePath;
import org.h2.util.New;
import org.h2.util.SmallLRUCache;
import org.h2.util.StringUtils;
/*
file format:
header
header
[ chunk ] *
header:
# H3 store #
pageSize=4096
r=1
chunk:
'd' [id] [metaRootPos] data ...
todo:
- garbage collection
- use page checksums
- compress chunks
- encode length in pos (1=32, 2=128, 3=512,...)
- don't use any 't' blocks
- floating header (avoid duplicate header)
for each chunk, store chunk (a counter)
for each page, store chunk id and offset to root
for each chunk, store position of expected next chunks
*/
/**
* A persistent storage for tree maps.
*/
public class TreeMapStore {
private final String fileName;
private FileChannel file;
private int pageSize = 4 * 1024;
private long rootBlockStart;
private HashMap<Long, Node> cache = SmallLRUCache.newInstance(50000);
private TreeMap<Integer, Block> blocks = new TreeMap<Integer, Block>();
private StoredMap<String, String> meta;
private HashMap<String, StoredMap<?, ?>> maps = New.hashMap();
private HashMap<String, StoredMap<?, ?>> mapsChanged = New.hashMap();
// TODO use an int instead? (with rollover to 0)
private long transaction;
private int tempNodeId;
private int lastBlockId;
private int loadCount;
private TreeMapStore(String fileName) {
this.fileName = fileName;
}
/**
* Open a tree store.
*
* @param fileName the file name
* @return the store
*/
public static TreeMapStore open(String fileName) {
TreeMapStore s = new TreeMapStore(fileName);
s.open();
return s;
}
/**
* Open a map.
*
* @param <K> the key type
* @param <V> the value type
* @param name the name of the map
* @param keyClass the key class
* @param valueClass the value class
* @return the map
*/
public <K, V> StoredMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
@SuppressWarnings("unchecked")
StoredMap<K, V> m = (StoredMap<K, V>) maps.get(name);
if (m == null) {
String root = meta.get("map." + name);
m = StoredMap.open(this, name, keyClass, valueClass);
maps.put(name, m);
if (root != null) {
root = StringUtils.arraySplit(root, ',', false)[0];
if (!root.equals("0")) {
m.setRoot(Long.parseLong(root));
}
}
}
return m;
}
/**
* Mark a map as changed.
*
* @param name the map name
* @param the map
*/
void markChanged(String name, StoredMap<?, ?> map) {
if (map != meta) {
mapsChanged.put(name, map);
}
}
private void open() {
meta = StoredMap.open(this, "meta", String.class, String.class);
new File(fileName).getParentFile().mkdirs();
try {
log("file open");
file = FilePathCache.wrap(FilePath.get(fileName).open("rw"));
if (file.size() == 0) {
writeHeader();
} else {
readHeader();
readMeta();
}
} catch (Exception e) {
throw convert(e);
}
}
private void readMeta() {
long rootId = readMetaRootId(rootBlockStart);
lastBlockId = getBlockId(rootId);
Block b = new Block(lastBlockId);
b.start = rootBlockStart;
blocks.put(b.id, b);
meta.setRoot(rootId);
Iterator<String> it = meta.keyIterator("block.");
while (it.hasNext()) {
String s = it.next();
if (!s.startsWith("block.")) {
break;
}
b = Block.fromString(meta.get(s));
lastBlockId = Math.max(b.id, lastBlockId);
blocks.put(b.id, b);
}
}
private void writeHeader() {
try {
ByteBuffer header = ByteBuffer.wrap((
"# H2 1.5\n" +
"read-version: 1\n" +
"write-version: 1\n" +
"rootBlock: " + rootBlockStart + "\n" +
"transaction: " + transaction + "\n").getBytes());
file.position(0);
file.write(header);
file.position(pageSize);
file.write(header);
} catch (Exception e) {
throw convert(e);
}
}
private void readHeader() {
try {
file.position(0);
byte[] header = new byte[pageSize];
// TODO read fully; read both headers
file.read(ByteBuffer.wrap(header));
Properties prop = new Properties();
prop.load(new ByteArrayInputStream(header));
rootBlockStart = Long.parseLong(prop.get("rootBlock").toString());
transaction = Long.parseLong(prop.get("transaction").toString());
} catch (Exception e) {
throw convert(e);
}
}
public String toString() {
return "cache size: " + cache.size() + " loadCount: " + loadCount + "\n" + blocks.toString().replace('\n', ' ');
}
private static RuntimeException convert(Exception e) {
throw new RuntimeException("Exception: " + e, e);
}
/**
* Close the file.
*/
public void close() {
store();
if (file != null) {
try {
log("file close");
file.close();
} catch (Exception e) {
file = null;
throw convert(e);
}
}
}
private int length(Node n) {
int len = 0;
if (n != null) {
len += n.length();
if (n.getLeftId() < 0) {
len += length(n.getLeft());
}
if (n.getRightId() < 0) {
len += length(n.getRight());
}
}
return len;
}
private long updateId(Node n, long offset) {
if (n != null) {
n.setId(offset);
cache.put(offset, n);
offset += n.length();
if (n.getLeftId() < 0) {
offset = updateId(n.getLeft(), offset);
}
if (n.getRightId() < 0) {
offset = updateId(n.getRight(), offset);
}
}
return offset;
}
private int count(Node n) {
if (n == null) {
return 0;
}
int count = 1;
if (n.getLeftId() < 0) {
count += count(n.getLeft());
}
if (n.getRightId() < 0) {
count += count(n.getRight());
}
return count;
}
private void store(ByteBuffer buff, Node n) {
if (n == null) {
return;
}
Node left = n.getLeftId() < 0 ? n.getLeft() : null;
if (left != null) {
n.setLeftId(left.getId());
}
Node right = n.getRightId() < 0 ? n.getRight() : null;
if (right != null) {
n.setRightId(right.getId());
}
n.write(buff);
if (left != null) {
store(buff, left);
}
if (right != null) {
store(buff, right);
}
}
private long getPosition(long nodeId) {
Block b = getBlock(nodeId);
if (b == null) {
throw new RuntimeException("Block " + getBlockId(nodeId) + " not found");
}
long pos = b.start;
pos += (int) (nodeId & Integer.MAX_VALUE);
return pos;
}
private long getId(int blockId, int offset) {
return ((long) blockId << 32) | offset;
}
/**
* Persist all changes to disk.
*/
public void store() {
if (!meta.isChanged() && mapsChanged.size() == 0) {
// TODO truncate file if empty
return;
}
commit();
// the length estimate is not correct,
// as we don't know the exact positions and entry counts
int lenEstimate = 1 + 8;
for (StoredMap<?, ?> m : mapsChanged.values()) {
meta.put("map." + m.getName(), String.valueOf(Long.MAX_VALUE) +
"," + m.getKeyType().getName() + "," + m.getValueType().getName());
lenEstimate += length(m.getRoot());
}
int blockId = ++lastBlockId;
Block b = new Block(blockId);
b.start = Long.MAX_VALUE;
b.entryCount = Integer.MAX_VALUE;
b.liveCount = Integer.MAX_VALUE;
blocks.put(b.id, b);
for (Block x : blocks.values()) {
if (x.liveCount == 0) {
meta.remove("block." + x.id);
} else {
meta.put("block." + x.id, "temp " + x.toString());
}
}
// modifying the meta can itself affect the metadata
// TODO solve this in a better way
for (Block x : new ArrayList<Block>(blocks.values())) {
if (x.liveCount == 0) {
meta.remove("block." + x.id);
blocks.remove(x.id);
} else {
meta.put("block." + x.id, x.toString());
}
}
lenEstimate += length(meta.getRoot());
b.length = lenEstimate;
long storePos = allocateBlock(lenEstimate);
long nodeId = getId(blockId, 1 + 8);
for (StoredMap<?, ?> m : mapsChanged.values()) {
Node r = m.getRoot();
long p = r == null ? 0 : nodeId;
meta.put("map." + m.getName(), String.valueOf(p) + "," + m.getKeyType().getName() + "," + m.getValueType().getName());
nodeId = updateId(r, nodeId);
}
int metaNodeOffset = (int) (nodeId - getId(blockId, 0));
// add a dummy entry so the count is correct
meta.put("block." + b.id, b.toString());
int count = 0;
for (StoredMap<?, ?> m : mapsChanged.values()) {
count += count(m.getRoot());
}
count += count(meta.getRoot());
b.start = storePos;
b.entryCount = count;
b.liveCount = b.entryCount;
meta.put("block." + b.id, b.toString());
nodeId = updateId(meta.getRoot(), nodeId);
int len = (int) (nodeId - getId(blockId, 0));
ByteBuffer buff = ByteBuffer.allocate(len);
buff.put((byte) 'd');
buff.putInt(b.id);
buff.putInt(metaNodeOffset);
for (StoredMap<?, ?> m : mapsChanged.values()) {
store(buff, m.getRoot());
}
store(buff, meta.getRoot());
if (buff.hasRemaining()) {
throw new RuntimeException("remaining: " + buff.remaining());
}
buff.rewind();
try {
file.position(storePos);
file.write(buff);
} catch (IOException e) {
throw new RuntimeException(e);
}
rootBlockStart = storePos;
writeHeader();
tempNodeId = 0;
mapsChanged.clear();
}
private long allocateBlock(long length) {
BitSet set = new BitSet();
set.set(0);
set.set(1);
for (Block b : blocks.values()) {
if (b.start == Long.MAX_VALUE) {
continue;
}
int first = (int) (b.start / pageSize);
int last = (int) ((b.start + b.length) / pageSize);
set.set(first, last +1);
}
int required = (int) (length / pageSize) + 1;
for (int i = 0; i < set.size(); i++) {
if (!set.get(i)) {
boolean ok = true;
for (int j = 1; j <= required; j++) {
if (set.get(i + j)) {
ok = false;
break;
}
}
if (ok) {
return i * pageSize;
}
}
}
return set.size() * pageSize;
}
/**
* Get the current transaction number.
*
* @return the transaction number
*/
long getTransaction() {
return transaction;
}
/**
* Get the next temporary node id.
*
* @return the node id
*/
long nextTempNodeId() {
return -(++tempNodeId);
}
/**
* Commit the current transaction.
*
* @return the transaction id
*/
public long commit() {
return ++transaction;
}
private long readMetaRootId(long blockStart) {
try {
file.position(blockStart);
ByteBuffer buff = ByteBuffer.wrap(new byte[16]);
file.read(buff);
buff.rewind();
if (buff.get() != 'd') {
throw new RuntimeException("File corrupt");
}
int blockId = buff.getInt();
int offset = buff.getInt();
return getId(blockId, offset);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Try to reduce the file size. Blocks with a low number of live items will
* be re-written.
*/
public void compact() {
if (blocks.size() <= 1) {
return;
}
long liveCountTotal = 0, entryCountTotal = 0;
for (Block b : blocks.values()) {
entryCountTotal += b.entryCount;
liveCountTotal += b.liveCount;
}
int averageEntryCount = (int) (entryCountTotal / blocks.size());
if (entryCountTotal == 0) {
return;
}
int percentTotal = (int) (100 * liveCountTotal / entryCountTotal);
if (percentTotal > 80) {
return;
}
ArrayList<Block> old = New.arrayList();
for (Block b : blocks.values()) {
int age = lastBlockId - b.id + 1;
b.collectPriority = b.getFillRate() / age;
old.add(b);
}
Collections.sort(old, new Comparator<Block>() {
public int compare(Block o1, Block o2) {
return new Integer(o1.collectPriority).compareTo(o2.collectPriority);
}
});
int moveCount = 0;
Block move = null;
for (Block b : old) {
if (moveCount + b.liveCount > averageEntryCount) {
break;
}
log(" block " + b.id + " " + b.getFillRate() + "% full; prio=" + b.collectPriority);
moveCount += b.liveCount;
move = b;
}
boolean remove = false;
for (Iterator<Block> it = old.iterator(); it.hasNext();) {
Block b = it.next();
if (move == b) {
remove = true;
} else if (remove) {
it.remove();
}
}
long oldMetaRootId = readMetaRootId(move.start);
long offset = getPosition(oldMetaRootId);
log(" meta:" + move.id + "/" + offset);
StoredMap<String, String> oldMeta = StoredMap.open(this, "old-meta", String.class, String.class);
oldMeta.setRoot(oldMetaRootId);
Iterator<String> it = oldMeta.keyIterator(null);
ArrayList<Integer> oldBlocks = New.arrayList();
while (it.hasNext()) {
String k = it.next();
String v = oldMeta.get(k);
log(" " + k + " " + v.replace('\n', ' '));
if (k.startsWith("block.")) {
String s = oldMeta.get(k);
Block b = Block.fromString(s);
if (!blocks.containsKey(b.id)) {
oldBlocks.add(b.id);
blocks.put(b.id, b);
}
continue;
}
if (!k.startsWith("map.")) {
continue;
}
k = k.substring("map.".length());
if (!maps.containsKey(k)) {
continue;
}
String[] d = StringUtils.arraySplit(v, ',', false);
Class<?> kt = StoredMap.getClass(d[1]);
Class<?> vt = StoredMap.getClass(d[2]);
StoredMap<?, ?> oldData = StoredMap.open(this, "old-" + k, kt, vt);
long oldDataRoot = Long.parseLong(d[0]);
oldData.setRoot(oldDataRoot);
@SuppressWarnings("unchecked")
StoredMap<Object, Object> data = (StoredMap<Object, Object>) maps.get(k);
Iterator<?> dataIt = oldData.keyIterator(null);
while (dataIt.hasNext()) {
Object o = dataIt.next();
Node n = data.getNode(o);
if (n == null) {
// was removed later - ignore
} else if (n.getId() < 0) {
// temporarily changed - ok
} else {
Block b = getBlock(n.getId());
if (old.contains(b)) {
log(" move key:" + o + " block:" + b.id);
data.remove(o);
data.put(o, n.getData());
}
}
}
}
for (int o : oldBlocks) {
blocks.remove(o);
}
}
/**
* Read a node.
*
* @param map the map
* @param id the node id
* @return the node
*/
Node readNode(StoredMap<?, ?> map, long id) {
Node n = cache.get(id);
if (n == null) {
try {
long pos = getPosition(id);
file.position(pos);
ByteBuffer buff = ByteBuffer.wrap(new byte[1024]);
// TODO read fully; read only required bytes
do {
int len = file.read(buff);
if (len < 0) {
break;
}
} while (buff.remaining() > 0);
buff.rewind();
n = Node.read(map, id, buff);
} catch (Exception e) {
throw new RuntimeException(e);
}
cache.put(id, n);
}
return n;
}
/**
* Remove a node.
*
* @param id the node id
*/
void removeNode(long id) {
if (id > 0) {
if (getBlock(id).liveCount == 0) {
throw new RuntimeException("Negative live count: " + id);
}
getBlock(id).liveCount--;
}
}
private int getBlockId(long nodeId) {
return (int) (nodeId >>> 32);
}
private Block getBlock(long nodeId) {
return blocks.get(getBlockId(nodeId));
}
/**
* A block of data.
*/
static class Block implements Comparable<Block> {
public int collectPriority;
int id;
long start;
long length;
int entryCount;
int liveCount;
Block(int id) {
this.id = id;
}
/**
* Build a block from the given string.
*
* @param s the string
* @return the block
*/
static Block fromString(String s) {
Block b = new Block(0);
Properties prop = new Properties();
try {
prop.load(new ByteArrayInputStream(s.getBytes("UTF-8")));
b.id = Integer.parseInt(prop.get("id").toString());
b.start = Long.parseLong(prop.get("start").toString());
b.length = Long.parseLong(prop.get("length").toString());
b.entryCount = Integer.parseInt(prop.get("entryCount").toString());
b.liveCount = Integer.parseInt(prop.get("liveCount").toString());
return b;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public int getFillRate() {
return entryCount == 0 ? 0 : 100 * liveCount / entryCount;
}
public int compareTo(Block o) {
return start == o.start ? 0 : start < o.start ? -1 : 1;
}
public int hashCode() {
return id;
}
public boolean equals(Object o) {
return o instanceof Block && ((Block) o).id == id;
}
public String toString() {
return
"id:" + id + "\n" +
"start:" + start + "\n" +
"length:" + length + "\n" +
"entryCount:" + entryCount + "\n" +
"liveCount:" + liveCount + "\n";
}
}
/**
* Log the string, if logging is enabled.
*
* @param string the string to log
*/
public void log(String string) {
// System.out.println(string);
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论