Skip to content

H
h2database
提交 e949b40f authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

A persistent multi-version map - generic data type serialization

上级 5167b8ef
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 1798 行增加 和 63 行删除
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ e949b40f
......@@ -108,6 +108,7 @@ import org.h2.test.store.TestConcurrent;
import org.h2.test.store.TestDataUtils;
import org.h2.test.store.TestMVStore;
import org.h2.test.store.TestMVRTree;
import org.h2.test.store.TestObjectType;
import org.h2.test.synth.TestBtreeIndex;
import org.h2.test.synth.TestCrashAPI;
import org.h2.test.synth.TestDiskFull;
......@@ -671,6 +672,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
new TestDataUtils().runTest(this);
new TestMVRTree().runTest(this);
new TestMVStore().runTest(this);
new TestObjectType().runTest(this);
// unit
new TestAutoReconnect().runTest(this);
......
h2/src/test/org/h2/test/store/ObjectType.java 0 → 100644
浏览文件 @ e949b40f
差异被折叠。
h2/src/test/org/h2/test/store/TestDataUtils.java
浏览文件 @ e949b40f
......@@ -26,6 +26,7 @@ public class TestDataUtils extends TestBase {
public void test() throws Exception {
testMap();
testMaxShortVarIntVarLong();
testVarIntVarLong();
testCheckValue();
testPagePos();
......@@ -50,6 +51,22 @@ public class TestDataUtils extends TestBase {
assertEquals("\"test\"", m.get("d"));
}
private void testMaxShortVarIntVarLong() {
ByteBuffer buff = ByteBuffer.allocate(100);
DataUtils.writeVarInt(buff, DataUtils.COMPRESSED_VAR_INT_MAX);
assertEquals(3, buff.position());
buff.rewind();
DataUtils.writeVarInt(buff, DataUtils.COMPRESSED_VAR_INT_MAX + 1);
assertEquals(4, buff.position());
buff.rewind();
DataUtils.writeVarLong(buff, DataUtils.COMPRESSED_VAR_LONG_MAX);
assertEquals(7, buff.position());
buff.rewind();
DataUtils.writeVarLong(buff, DataUtils.COMPRESSED_VAR_LONG_MAX + 1);
assertEquals(8, buff.position());
buff.rewind();
}
private void testVarIntVarLong() {
ByteBuffer buff = ByteBuffer.allocate(100);
for (long x = 0; x < 1000; x++) {
......
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ e949b40f
......@@ -12,7 +12,6 @@ import java.util.Random;
import java.util.TreeMap;
import org.h2.dev.store.btree.MVMap;
import org.h2.dev.store.btree.MVStore;
import org.h2.jaqu.bytecode.Null;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.New;
......@@ -32,6 +31,7 @@ public class TestMVStore extends TestBase {
}
public void test() throws InterruptedException {
testObjects();
testExample();
testIterateOverChanges();
testOpenStoreCloseLoop();
......@@ -52,8 +52,31 @@ public class TestMVStore extends TestBase {
testSimple();
}
private void testObjects() {
String fileName = getBaseDir() + "/testObjects.h3";
FileUtils.delete(fileName);
MVStore s;
Map<Object, Object> map;
s = MVStore.open(fileName, new TestMapFactory());
map = s.openMap("test");
map.put(1, "Hello");
map.put("2", 200);
map.put(new Object[1], new Object[]{1, "2"});
s.store();
s.close();
s = MVStore.open(fileName, new TestMapFactory());
map = s.openMap("test");
assertEquals("Hello", map.get(1).toString());
assertEquals(200, ((Integer) map.get("2")).intValue());
Object[] x = (Object[]) map.get(new Object[1]);
assertEquals(2, x.length);
assertEquals(1, ((Integer) x[0]).intValue());
assertEquals("2", (String) x[1]);
s.close();
}
private void testExample() {
String fileName = getBaseDir() + "/testOpenClose.h3";
String fileName = getBaseDir() + "/testExample.h3";
FileUtils.delete(fileName);
// open the store (in-memory if fileName is null)
......@@ -649,29 +672,34 @@ public class TestMVStore extends TestBase {
}
private void testKeyValueClasses() {
String fileName = getBaseDir() + "/testKeyValueClasses.h3";
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
is.put(1, "Hello");
MVMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
ii.put(1, 10);
MVMap<String, Integer> si = s.openMap("stringInt", String.class, Integer.class);
si.put("Test", 10);
MVMap<String, String> ss = s.openMap("stringString", String.class, String.class);
ss.put("Hello", "World");
MVStore s;
s = MVStore.open(null);
s.openMap("test", String.class, String.class);
try {
s.openMap("invalid", Null.class, Integer.class);
s.openMap("unsupportedKey", ArrayList.class, String.class);
fail();
} catch (RuntimeException e) {
// expected
}
try {
s.openMap("invalid", Integer.class, Null.class);
s.openMap("unsupportedValue", String.class, ArrayList.class);
fail();
} catch (RuntimeException e) {
// expected
}
s.close();
String fileName = getBaseDir() + "/testKeyValueClasses.h3";
FileUtils.delete(fileName);
s = openStore(fileName);
MVMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
is.put(1, "Hello");
MVMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
ii.put(1, 10);
MVMap<String, Integer> si = s.openMap("stringInt", String.class, Integer.class);
si.put("Test", 10);
MVMap<String, String> ss = s.openMap("stringString", String.class, String.class);
ss.put("Hello", "World");
s.store();
s.close();
s = openStore(fileName);
......
h2/src/test/org/h2/test/store/TestMapFactory.java
浏览文件 @ e949b40f
......@@ -41,6 +41,8 @@ public class TestMapFactory implements MapFactory {
return RowType.fromString(s, this);
case 's':
return SpatialType.fromString(s);
case 'o':
return new ObjectType();
}
throw new RuntimeException("Unknown data type " + s);
}
......@@ -49,6 +51,8 @@ public class TestMapFactory implements MapFactory {
public String getDataType(Class<?> objectClass) {
if (objectClass == Integer.class) {
return "i";
} else if (Object.class == Object.class) {
return "o";
}
throw new RuntimeException("Unsupported object class " + objectClass.toString());
}
......
h2/src/test/org/h2/test/store/TestObjectType.java 0 → 100644
浏览文件 @ e949b40f
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Random;
import java.util.UUID;
import org.h2.test.TestBase;
/**
* Test the ObjectType class.
*/
public class TestObjectType extends TestBase {
/**
* Run just this test.
*
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
}
@Override
public void test() {
testCommonValues();
}
private void testCommonValues() {
BigInteger largeBigInt = BigInteger.probablePrime(200, new Random(1));
ObjectType ot = new ObjectType();
assertEquals("o", ot.asString());
Object[] array = {
false, true,
Byte.MIN_VALUE, (byte) -1, (byte) 0, (byte) 1, Byte.MAX_VALUE,
Short.MIN_VALUE, (short) -1, (short) 0, (short) 1, Short.MAX_VALUE,
Integer.MIN_VALUE, Integer.MIN_VALUE + 1,
-1000, -100, -1, 0, 1, 2, 14,
15, 16, 17, 100, Integer.MAX_VALUE - 1, Integer.MAX_VALUE,
Long.MIN_VALUE, Long.MIN_VALUE + 1, -1000L, -1L, 0L, 1L, 2L, 14L,
15L, 16L, 17L, 100L, Long.MAX_VALUE - 1, Long.MAX_VALUE,
largeBigInt.negate(), BigInteger.valueOf(-1), BigInteger.ZERO,
BigInteger.ONE, BigInteger.TEN, largeBigInt,
Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, -1f, -0f, 0f,
Float.MIN_VALUE, 1f, Float.MAX_VALUE,
Float.POSITIVE_INFINITY, Float.NaN,
Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, -1d, -0d, 0d,
Double.MIN_VALUE, 1d, Double.MAX_VALUE,
Double.POSITIVE_INFINITY, Double.NaN,
BigDecimal.valueOf(Double.MAX_VALUE).negate(),
new BigDecimal(largeBigInt).negate(),
BigDecimal.valueOf(-100.0), BigDecimal.ZERO, BigDecimal.ONE,
BigDecimal.TEN, BigDecimal.valueOf(Long.MAX_VALUE),
new BigDecimal(largeBigInt),
BigDecimal.valueOf(Double.MAX_VALUE),
Character.MIN_VALUE, '0', 'a', Character.MAX_VALUE,
"", " ", " ", "123456789012345", "1234567890123456",
new String(new char[100]).replace((char) 0, 'x'),
new String(new char[100000]).replace((char) 0, 'x'), "y",
"\u1234", "\u2345", "\u6789", "\uffff",
new UUID(Long.MIN_VALUE, Long.MIN_VALUE),
new UUID(Long.MIN_VALUE, 0), new UUID(0, 0),
new UUID(Long.MAX_VALUE, Long.MAX_VALUE),
new byte[0], new byte[1], new byte[15], new byte[16],
new byte[10000], new byte[] { (byte) 1 },
new int[0], new int[1], new int[15], new int[16],
new int[10000], new int[] { (byte) 1 },
new long[0], new long[1], new long[15], new long[16],
new long[10000], new long[] { (byte) 1 },
new char[0], new char[1], new char[10000], new char[] { (char) 1 },
new java.util.Date(0), new java.util.Date(1000),
new Timestamp(2000), new Timestamp(3000),
new java.util.Date(4000), new java.util.Date(5000),
new Object[0], new Object[] { 1 },
new Object[] { 0.0, "Hello", null, Double.NaN },
new Object[100]
};
Object otherType = false;
Object last = null;
for (Object x : array) {
test(otherType, x);
if (last != null) {
int comp = ot.compare(x, last);
if (comp <= 0) {
ot.compare(x, last);
fail(x.getClass().getName() + ": " + x.toString() + " " + comp);
}
assertTrue(x.toString(), ot.compare(last, x) < 0);
}
if (last != null && last.getClass() != x.getClass()) {
otherType = last;
}
last = x;
}
}
private void test(Object last, Object x) {
ObjectType ot = new ObjectType();
// switch to the last type before every operation,
// to test switching types
ot.getMemory(last);
assertTrue(ot.getMemory(x) >= 0);
ot.getMemory(last);
assertTrue(ot.getMaxLength(x) >= 1);
ot.getMemory(last);
assertEquals(0, ot.compare(x, x));
ByteBuffer buff = ByteBuffer.allocate(ot.getMaxLength(x) + 1);
ot.getMemory(last);
ot.write(buff, x);
buff.put((byte) 123);
buff.flip();
ot.getMemory(last);
Object y = ot.read(buff);
assertEquals(123, buff.get());
assertEquals(0, buff.remaining());
assertEquals(x.getClass().getName(), y.getClass().getName());
ot.getMemory(last);
assertEquals(0, ot.compare(x, y));
if (x.getClass().isArray()) {
if (x instanceof byte[]) {
assertTrue(Arrays.equals((byte[]) x, (byte[]) y));
} else if (x instanceof char[]) {
assertTrue(Arrays.equals((char[]) x, (char[]) y));
} else if (x instanceof int[]) {
assertTrue(Arrays.equals((int[]) x, (int[]) y));
} else if (x instanceof long[]) {
assertTrue(Arrays.equals((long[]) x, (long[]) y));
} else {
assertTrue(Arrays.equals((Object[]) x, (Object[]) y));
}
} else {
assertEquals(x.hashCode(), y.hashCode());
assertTrue(x.equals(y));
}
}
}
h2/src/tools/org/h2/dev/store/btree/DataUtils.java
浏览文件 @ e949b40f
......@@ -42,6 +42,18 @@ public class DataUtils {
*/
public static final int MAX_VAR_LONG_LEN = 10;
/**
* The maximum integer that needs less space when using variable size
* encoding (only 3 bytes instead of 4).
*/
public static final int COMPRESSED_VAR_INT_MAX = 0x1fffff;
/**
* The maximum long that needs less space when using variable size
* encoding (only 7 bytes instead of 8).
*/
public static final long COMPRESSED_VAR_LONG_MAX = 0x1ffffffffffffL;
/**
* Get the length of the variable size int.
*
......@@ -149,6 +161,51 @@ public class DataUtils {
buff.put((byte) x);
}
/**
* Write characters from a string (without the length).
*
* @param buff the target buffer
* @param s the string
* @param len the number of characters
*/
public static void writeStringData(ByteBuffer buff, String s, int len) {
for (int i = 0; i < len; i++) {
int c = s.charAt(i);
if (c < 0x80) {
buff.put((byte) c);
} else if (c >= 0x800) {
buff.put((byte) (0xe0 | (c >> 12)));
buff.put((byte) (((c >> 6) & 0x3f)));
buff.put((byte) (c & 0x3f));
} else {
buff.put((byte) (0xc0 | (c >> 6)));
buff.put((byte) (c & 0x3f));
}
}
}
/**
* Read a string.
*
* @param buff the source buffer
* @param len the number of characters
* @return the value
*/
public static String readString(ByteBuffer buff, int len) {
char[] chars = new char[len];
for (int i = 0; i < len; i++) {
int x = buff.get() & 0xff;
if (x < 0x80) {
chars[i] = (char) x;
} else if (x >= 0xe0) {
chars[i] = (char) (((x & 0xf) << 12) + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f));
} else {
chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f));
}
}
return new String(chars);
}
/**
* Write a variable size int.
*
......
h2/src/tools/org/h2/dev/store/btree/MVStore.java
浏览文件 @ e949b40f
......@@ -35,7 +35,6 @@ header:
H:3,blockSize=4096,...
TODO:
- support all objects (using serialization)
- concurrent iterator (when to increment version; read on first hasNext())
- how to iterate (just) over deleted pages / entries
- compact: use total max length instead of page count (liveCount)
......@@ -60,6 +59,9 @@ TODO:
- recovery: ensure data is not overwritten for 1 minute
- pluggable caching (specially for in-memory file systems)
- file locking
- allocate memory Utils.newBytes
- unified exception handling
- check if locale specific string comparison can make data disappear
*/
......@@ -73,7 +75,10 @@ public class MVStore {
*/
public static final boolean ASSERT = false;
private static final StringType STRING_TYPE = new StringType();
/**
* A string data type.
*/
public static final StringType STRING_TYPE = new StringType();
private final String fileName;
private final MapFactory mapFactory;
......@@ -169,6 +174,41 @@ public class MVStore {
return (T) m;
}
/**
* Open a map with the previous key and value type (if the map already
* exists), or Object if not.
*
* @param <K> the key type
* @param <V> the value type
* @param name the name of the map
* @return the map
*/
public <K, V> MVMap<K, V> openMap(String name) {
String keyType = getDataType(Object.class);
String valueType = getDataType(Object.class);
@SuppressWarnings("unchecked")
MVMap<K, V> m = (MVMap<K, V>) openMap(name, "", keyType, valueType);
return m;
}
/**
* Open a map.
*
* @param <K> the key type
* @param <V> the value type
* @param name the name of the map
* @param keyClass the key class
* @param valueClass the value class
* @return the map
*/
public <K, V> MVMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
String keyType = getDataType(keyClass);
String valueType = getDataType(valueClass);
@SuppressWarnings("unchecked")
MVMap<K, V> m = (MVMap<K, V>) openMap(name, "", keyType, valueType);
return m;
}
/**
* Open a map.
*
......@@ -187,6 +227,7 @@ public class MVStore {
int id;
long root;
long createVersion;
// TODO use the json formatting for map metadata
if (identifier == null) {
id = ++lastMapId;
createVersion = currentVersion;
......@@ -259,24 +300,6 @@ public class MVStore {
}
}
/**
* Open a map.
*
* @param <K> the key type
* @param <V> the value type
* @param name the name of the map
* @param keyClass the key class
* @param valueClass the value class
* @return the map
*/
public <K, V> MVMap<K, V> openMap(String name, Class<K> keyClass, Class<V> valueClass) {
String keyType = getDataType(keyClass);
String valueType = getDataType(valueClass);
@SuppressWarnings("unchecked")
MVMap<K, V> m = (MVMap<K, V>) openMap(name, "", keyType, valueType);
return m;
}
/**
* Remove a map.
*
......
h2/src/tools/org/h2/dev/store/btree/MapFactory.java
浏览文件 @ e949b40f
......@@ -7,7 +7,7 @@
package org.h2.dev.store.btree;
/**
* A factory for data types.
* A factory for maps and data types.
*/
public interface MapFactory {
......
h2/src/tools/org/h2/dev/store/btree/StringType.java
浏览文件 @ e949b40f
......@@ -18,46 +18,23 @@ public class StringType implements DataType {
}
public int getMaxLength(Object obj) {
return DataUtils.MAX_VAR_INT_LEN + obj.toString().length() * 3;
return DataUtils.MAX_VAR_INT_LEN + 3 * obj.toString().length();
}
public int getMemory(Object obj) {
return obj.toString().length() * 2 + 48;
return 24 + 2 * obj.toString().length();
}
public String read(ByteBuffer buff) {
int len = DataUtils.readVarInt(buff);
char[] chars = new char[len];
for (int i = 0; i < len; i++) {
int x = buff.get() & 0xff;
if (x < 0x80) {
chars[i] = (char) x;
} else if (x >= 0xe0) {
chars[i] = (char) (((x & 0xf) << 12) + ((buff.get() & 0x3f) << 6) + (buff.get() & 0x3f));
} else {
chars[i] = (char) (((x & 0x1f) << 6) + (buff.get() & 0x3f));
}
}
return new String(chars);
return DataUtils.readString(buff, len);
}
public void write(ByteBuffer buff, Object obj) {
String s = obj.toString();
int len = s.length();
DataUtils.writeVarInt(buff, len);
for (int i = 0; i < len; i++) {
int c = s.charAt(i);
if (c < 0x80) {
buff.put((byte) c);
} else if (c >= 0x800) {
buff.put((byte) (0xe0 | (c >> 12)));
buff.put((byte) (((c >> 6) & 0x3f)));
buff.put((byte) (c & 0x3f));
} else {
buff.put((byte) (0xc0 | (c >> 6)));
buff.put((byte) (c & 0x3f));
}
}
DataUtils.writeStringData(buff, s, len);
}
public String asString() {
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论