提交 871b0a8e authored 作者: Thomas Mueller's avatar Thomas Mueller

A persistent multi-version map: prepare initial release

上级 f2afe926
......@@ -94,6 +94,7 @@ translate -->
<a href="build.html">Build</a><br />
<a href="links.html">Links</a><br />
<a href="jaqu.html">JaQu</a><br />
<a href="mvstore.html">MVStore</a><br />
<br />
</div>
......
......@@ -7,7 +7,7 @@ Initial Developer: H2 Group
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /><title>
JaQu
MVStore
</title><link rel="stylesheet" type="text/css" href="stylesheet.css" />
<!-- [search] { -->
<script type="text/javascript" src="navigation.js"></script>
......@@ -34,33 +34,38 @@ JaQu
Future Plans</a><br />
<h2 id="overview">Overview</h2>
<p>
The MVStore is work in progress, and is planned to be the next storage subsystem of H2.
</p>
<ul><li>MVStore stands for multi-version store.
</li><li>Each store contains a number of maps (using the <code>java.util.Map</code> interface).
</li><li>The data can be persisted to disk (like a key-value store or a database).
</li><li>Fully in-memory operation is supported.
</li><li>It is intended to be fast, simple to use, and small.
</li><li>Old versions of the data can be read concurrently with all other operations.
</li><li>Transaction are supported (currently only one transaction can be open at any time).
</li><li>Transaction are supported (currently only one transaction at a time).
</li><li>Transactions (even if they are persisted) can be rolled back.
</li><li>The tool is very modular. It supports pluggable data types / serialization,
pluggable map implementations (B-tree and R-tree currently), BLOB storage,
and a file system abstraction that supports encryption and compressed
read-only files.
and a file system abstraction to support encryption and compressed read-only files.
</li></ul>
<h2 id="example_code">Example Code</h2>
<h3>Map Operations and Versioning</h3>
<p>
The following sample code show how to create a store,
open a map, add some data, and access the current and an old version.
</p>
<pre>
// open the store (in-memory if fileName is null)
MVStore s = MVStore.open(fileName);
// create/get the map "data"
// the String.class, String.class will be optional later
MVMap<String, String> map = s.openMap("data",
String.class, String.class);
// create/get the map named "data"
MVMap<Integer, String> map = s.openMap("data");
// add some data
map.put("1", "Hello");
map.put("2", "World");
map.put(1, "Hello");
map.put(2, "World");
// get the current version, for later use
long oldVersion = s.getCurrentVersion();
......@@ -71,11 +76,11 @@ s.incrementVersion();
// more changes, in the new version
// changes can be rolled back if required
// changes always go into 'head' (the newest version)
map.put("1", "Hi");
map.remove("2");
map.put(1, "Hi");
map.remove(2);
// access the old data (before incrementVersion)
MVMap<String, String> oldMap =
MVMap<Integer, String> oldMap =
map.openVersion(oldVersion);
// store the newest data to disk
......@@ -84,16 +89,62 @@ s.store();
// print the old version (can be done
// concurrently with further modifications)
// this will print Hello World
System.out.println(oldMap.get("1"));
System.out.println(oldMap.get("2"));
System.out.println(oldMap.get(1));
System.out.println(oldMap.get(2));
oldMap.close();
// print the newest version ("Hi")
System.out.println(map.get("1"));
System.out.println(map.get(1));
// close the store - this doesn't write to disk
s.close();
</pre>
<h3>Store Builder</h3>
<p>
The <code>MVStoreBuilder</code> provides a fluid interface
to build a store if more complex configuration options are used.
</p>
<pre>
MVStore s = MVStoreBuilder.
fileBased(fileName).
cacheSizeMB(10).
readOnly().
open();
</pre>
<h3>R-Tree</h3>
<p>
The <code>MVRTreeMap</code> is an R-tree implementation
that supports fast spatial queries.
</p>
<pre>
// create an in-memory store
MVStore s = MVStore.open(null);
// create an R-tree map
// the key has 2 dimensions, the value is a string
MVRTreeMap<String> r = MVRTreeMap.create(2, new ObjectType());
// open the map
r = s.openMap("data", r);
// add two key-value pairs
// the first value is the key id (to make the key unique)
// then the min x, max x, min y, max y
r.add(new SpatialKey(0, -3f, -2f, 2f, 3f), "left");
r.add(new SpatialKey(1, 3f, 4f, 4f, 5f), "right");
// iterate over the intersecting keys
Iterator<SpatialKey> it =
r.findIntersectingKeys(new SpatialKey(0, 0f, 9f, 3f, 6f));
for (SpatialKey k; it.hasNext();) {
k = it.next();
System.out.println(k + ": " + r.get(k));
}
s.close();
</pre>
<h2 id="features">Features</h2>
......@@ -103,13 +154,18 @@ Each store supports a set of named maps.
A map is sorted by key, and supports the common lookup operations,
including access to the first and last key, iterate over some or all keys, and so on.
</p><p>
Also supported, and unusual for maps, is fast index lookup, as if the map would be a list
(get the key at the given index, get the index of a certain key), and fast skipping within an iterator.
Internally, each map is organized in the form of a counted B+-tree.
Also supported, and very uncommon for maps, is fast index lookup.
The keys of the map can be accessed like a list
(get the key at the given index, get the index of a certain key).
That means getting the median of two keys is trivial,
and it allows to very quickly count ranges.
The iterator supports fast skipping.
This is possible because internally, each map is organized in the form of a counted B+-tree.
</p><p>
In database terms, a map can be a table, where the key of the map is the primary key of the table,
In database terms, a map can be used like a table, where the key of the map is the primary key of the table,
and the value is the row. A map can also represent an index, where the key of the map is the key
of the index, and the value of the map is the primary key of the table.
of the index, and the value of the map is the primary key of the table (for non-unique indexes
the key of the map must also contain the primary key).
</p>
<h3>Versions / Transactions</h3>
......@@ -189,9 +245,11 @@ use less memory on average.
<h3>Pluggable Data Types</h3>
<p>
Serialization is pluggable. The default serialization currently only supports string values and keys,
but there is a sample plugin that supports many common data types,
and uses Java serialization for other objects.
Serialization is pluggable. The default serialization currently supports many common data types,
and uses Java serialization for other objects. The following classes are currently directly supported:
<code>Boolean, Byte, Short, Character, Integer, Long, Float, Double, BigInteger, BigDecimal,
byte[], char[], int[], long[], String, UUID</code>.
The plan is to add more common classes (date, time, timestamp, object array).
</p><p>
Parameterized data types are supported
(for example one could build a string data type that limits the length for some reason).
......@@ -232,7 +290,7 @@ which should be resistant against scan operations.
Concurrent modification operations on the maps are currently not supported,
however it is planned to support an additional map implementation
that supports concurrent writes
(at the cost of speed if used in a single thread, same as ConcurrentHashMap).
(at the cost of speed if used in a single thread, same as <code>ConcurrentHashMap</code>).
</p>
<h3>File System Abstraction, File Locking and Online Backup</h3>
......@@ -251,9 +309,18 @@ Files can be opened in read-only mode, in which case a shared lock is used.
The persisted data can be backed up to a different file at any time,
even during write operations (online backup).
To do that, automatic disk space reuse needs to be first disabled, so that
new data is always appended at the end of the file. Then, the file
new data is always appended at the end of the file.
Then, the file can be copied (the file handle is available to the application).
</p>
<h3>Tools</h3>
<p>
There is a builder for store instances (<code>MVStoreBuilder</code>)
with a fluent API to simplify building a store instance.
</p>
<p>
There is a tool (<code>MVStoreTool</code>) to dump the contents of a file.
</p>
<h2 id="differences">Similar Projects and Differences to Other Storage Engines</h2>
<p>
......@@ -261,7 +328,7 @@ Unlike similar storage engines like LevelDB and Kyoto Cabinet, the MVStore is wr
and can easily be embedded in a Java application.
</p><p>
The MVStore is somewhat similar to the Berkeley DB Java Edition because it is also written in Java,
and also a log structured storage, but the MVStore licensing is more liberal.
and is also a log structured storage, but the H2 license is more liberal.
</p><p>
Like SQLite, the MVStore keeps all data in one file.
The plan is to make the MVStore easier to use and faster than SQLite on Android
......@@ -269,7 +336,7 @@ The plan is to make the MVStore easier to use and faster than SQLite on Android
</p><p>
The API of the MVStore is similar to MapDB (previously known as JDBM) from Jan Kotek,
and some code is shared between MapDB and JDBM.
However, the MVStore is a log structured storage.
However, unlike MapDB, the MVStore uses is a log structured storage.
</p>
<h2 id="current_state">Current State</h2>
......
......@@ -555,120 +555,120 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
beforeTest();
// db
new TestScriptSimple().runTest(this);
new TestScript().runTest(this);
new TestAlter().runTest(this);
new TestAlterSchemaRename().runTest(this);
new TestAutoRecompile().runTest(this);
new TestBitField().runTest(this);
new TestBackup().runTest(this);
new TestBigDb().runTest(this);
new TestBigResult().runTest(this);
new TestCases().runTest(this);
new TestCheckpoint().runTest(this);
new TestCluster().runTest(this);
new TestCompatibility().runTest(this);
new TestCsv().runTest(this);
new TestDateStorage().runTest(this);
new TestDeadlock().runTest(this);
new TestEncryptedDb().runTest(this);
new TestExclusive().runTest(this);
new TestFullText().runTest(this);
new TestFunctionOverload().runTest(this);
new TestFunctions().runTest(this);
new TestInit().runTest(this);
new TestIndex().runTest(this);
new TestLargeBlob().runTest(this);
new TestLinkedTable().runTest(this);
new TestListener().runTest(this);
new TestLob().runTest(this);
new TestMemoryUsage().runTest(this);
new TestMultiConn().runTest(this);
new TestMultiDimension().runTest(this);
new TestMultiThread().runTest(this);
new TestMultiThreadedKernel().runTest(this);
new TestOpenClose().runTest(this);
new TestOptimizations().runTest(this);
new TestOutOfMemory().runTest(this);
new TestPowerOff().runTest(this);
new TestQueryCache().runTest(this);
new TestReadOnly().runTest(this);
new TestRecursiveQueries().runTest(this);
new TestRights().runTest(this);
new TestRunscript().runTest(this);
new TestSQLInjection().runTest(this);
new TestSessionsLocks().runTest(this);
new TestSelectCountNonNullColumn().runTest(this);
new TestSequence().runTest(this);
new TestSpaceReuse().runTest(this);
new TestSpeed().runTest(this);
new TestTableEngines().runTest(this);
new TestTempTables().runTest(this);
new TestTransaction().runTest(this);
new TestTriggersConstraints().runTest(this);
new TestTwoPhaseCommit().runTest(this);
new TestView().runTest(this);
new TestViewAlterTable().runTest(this);
new TestViewDropView().runTest(this);
// jaqu
new AliasMapTest().runTest(this);
new AnnotationsTest().runTest(this);
new ClobTest().runTest(this);
new ModelsTest().runTest(this);
new SamplesTest().runTest(this);
new UpdateTest().runTest(this);
// jdbc
new TestBatchUpdates().runTest(this);
new TestCallableStatement().runTest(this);
new TestCancel().runTest(this);
new TestDatabaseEventListener().runTest(this);
new TestDriver().runTest(this);
new TestJavaObject().runTest(this);
new TestLimitUpdates().runTest(this);
new TestLobApi().runTest(this);
new TestManyJdbcObjects().runTest(this);
new TestMetaData().runTest(this);
new TestNativeSQL().runTest(this);
new TestPreparedStatement().runTest(this);
new TestResultSet().runTest(this);
new TestStatement().runTest(this);
new TestTransactionIsolation().runTest(this);
new TestUpdatableResultSet().runTest(this);
new TestZloty().runTest(this);
// jdbcx
new TestConnectionPool().runTest(this);
new TestDataSource().runTest(this);
new TestXA().runTest(this);
new TestXASimple().runTest(this);
// server
new TestAutoServer().runTest(this);
new TestNestedLoop().runTest(this);
new TestWeb().runTest(this);
// mvcc & row level locking
new TestMvcc1().runTest(this);
new TestMvcc2().runTest(this);
new TestMvcc3().runTest(this);
new TestMvccMultiThreaded().runTest(this);
new TestRowLocks().runTest(this);
// synth
new TestBtreeIndex().runTest(this);
new TestDiskFull().runTest(this);
new TestCrashAPI().runTest(this);
new TestFuzzOptimizations().runTest(this);
new TestLimit().runTest(this);
new TestRandomSQL().runTest(this);
new TestRandomCompare().runTest(this);
new TestKillRestart().runTest(this);
new TestKillRestartMulti().runTest(this);
new TestMultiThreaded().runTest(this);
new TestOuterJoins().runTest(this);
new TestNestedJoins().runTest(this);
// new TestScriptSimple().runTest(this);
// new TestScript().runTest(this);
// new TestAlter().runTest(this);
// new TestAlterSchemaRename().runTest(this);
// new TestAutoRecompile().runTest(this);
// new TestBitField().runTest(this);
// new TestBackup().runTest(this);
// new TestBigDb().runTest(this);
// new TestBigResult().runTest(this);
// new TestCases().runTest(this);
// new TestCheckpoint().runTest(this);
// new TestCluster().runTest(this);
// new TestCompatibility().runTest(this);
// new TestCsv().runTest(this);
// new TestDateStorage().runTest(this);
// new TestDeadlock().runTest(this);
// new TestEncryptedDb().runTest(this);
// new TestExclusive().runTest(this);
// new TestFullText().runTest(this);
// new TestFunctionOverload().runTest(this);
// new TestFunctions().runTest(this);
// new TestInit().runTest(this);
// new TestIndex().runTest(this);
// new TestLargeBlob().runTest(this);
// new TestLinkedTable().runTest(this);
// new TestListener().runTest(this);
// new TestLob().runTest(this);
// new TestMemoryUsage().runTest(this);
// new TestMultiConn().runTest(this);
// new TestMultiDimension().runTest(this);
// new TestMultiThread().runTest(this);
// new TestMultiThreadedKernel().runTest(this);
// new TestOpenClose().runTest(this);
// new TestOptimizations().runTest(this);
// new TestOutOfMemory().runTest(this);
// new TestPowerOff().runTest(this);
// new TestQueryCache().runTest(this);
// new TestReadOnly().runTest(this);
// new TestRecursiveQueries().runTest(this);
// new TestRights().runTest(this);
// new TestRunscript().runTest(this);
// new TestSQLInjection().runTest(this);
// new TestSessionsLocks().runTest(this);
// new TestSelectCountNonNullColumn().runTest(this);
// new TestSequence().runTest(this);
// new TestSpaceReuse().runTest(this);
// new TestSpeed().runTest(this);
// new TestTableEngines().runTest(this);
// new TestTempTables().runTest(this);
// new TestTransaction().runTest(this);
// new TestTriggersConstraints().runTest(this);
// new TestTwoPhaseCommit().runTest(this);
// new TestView().runTest(this);
// new TestViewAlterTable().runTest(this);
// new TestViewDropView().runTest(this);
//
// // jaqu
// new AliasMapTest().runTest(this);
// new AnnotationsTest().runTest(this);
// new ClobTest().runTest(this);
// new ModelsTest().runTest(this);
// new SamplesTest().runTest(this);
// new UpdateTest().runTest(this);
//
// // jdbc
// new TestBatchUpdates().runTest(this);
// new TestCallableStatement().runTest(this);
// new TestCancel().runTest(this);
// new TestDatabaseEventListener().runTest(this);
// new TestDriver().runTest(this);
// new TestJavaObject().runTest(this);
// new TestLimitUpdates().runTest(this);
// new TestLobApi().runTest(this);
// new TestManyJdbcObjects().runTest(this);
// new TestMetaData().runTest(this);
// new TestNativeSQL().runTest(this);
// new TestPreparedStatement().runTest(this);
// new TestResultSet().runTest(this);
// new TestStatement().runTest(this);
// new TestTransactionIsolation().runTest(this);
// new TestUpdatableResultSet().runTest(this);
// new TestZloty().runTest(this);
//
// // jdbcx
// new TestConnectionPool().runTest(this);
// new TestDataSource().runTest(this);
// new TestXA().runTest(this);
// new TestXASimple().runTest(this);
//
// // server
// new TestAutoServer().runTest(this);
// new TestNestedLoop().runTest(this);
// new TestWeb().runTest(this);
//
// // mvcc & row level locking
// new TestMvcc1().runTest(this);
// new TestMvcc2().runTest(this);
// new TestMvcc3().runTest(this);
// new TestMvccMultiThreaded().runTest(this);
// new TestRowLocks().runTest(this);
//
// // synth
// new TestBtreeIndex().runTest(this);
// new TestDiskFull().runTest(this);
// new TestCrashAPI().runTest(this);
// new TestFuzzOptimizations().runTest(this);
// new TestLimit().runTest(this);
// new TestRandomSQL().runTest(this);
// new TestRandomCompare().runTest(this);
// new TestKillRestart().runTest(this);
// new TestKillRestartMulti().runTest(this);
// new TestMultiThreaded().runTest(this);
// new TestOuterJoins().runTest(this);
// new TestNestedJoins().runTest(this);
afterTest();
}
......
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
import java.nio.ByteBuffer;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataUtils;
/**
* An integer type.
*/
class IntegerType implements DataType {
public static final IntegerType INSTANCE = new IntegerType();
public int compare(Object a, Object b) {
return ((Integer) a).compareTo((Integer) b);
}
public int getMaxLength(Object obj) {
return DataUtils.MAX_VAR_INT_LEN;
}
public int getMemory(Object obj) {
return 20;
}
public Integer read(ByteBuffer buff) {
return DataUtils.readVarInt(buff);
}
public void write(ByteBuffer buff, Object x) {
DataUtils.writeVarInt(buff, (Integer) x);
}
public String asString() {
return "i";
}
}
......@@ -7,9 +7,9 @@
package org.h2.test.store;
import java.nio.ByteBuffer;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataUtils;
import org.h2.dev.store.btree.DataTypeFactory;
import org.h2.dev.store.type.DataType;
import org.h2.dev.store.type.DataTypeFactory;
import org.h2.util.StringUtils;
/**
......@@ -17,6 +17,8 @@ import org.h2.util.StringUtils;
*/
public class RowType implements DataType {
static final String PREFIX = "org.h2.test.store.row";
private final DataType[] types;
RowType(DataType[] types) {
......@@ -86,7 +88,7 @@ public class RowType implements DataType {
public String asString() {
StringBuilder buff = new StringBuilder();
buff.append('r');
buff.append(PREFIX);
buff.append('(');
for (int i = 0; i < types.length; i++) {
if (i > 0) {
......@@ -106,10 +108,10 @@ public class RowType implements DataType {
* @return the row type
*/
static RowType fromString(String t, DataTypeFactory factory) {
if (!t.startsWith("r(") || !t.endsWith(")")) {
if (!t.startsWith(PREFIX) || !t.endsWith(")")) {
throw new RuntimeException("Unknown type: " + t);
}
t = t.substring(2, t.length() - 1);
t = t.substring(PREFIX.length(), t.length() - 1);
String[] array = StringUtils.arraySplit(t, ',', false);
DataType[] types = new DataType[array.length];
for (int i = 0; i < array.length; i++) {
......
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License, Version
* 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html). Initial Developer: H2 Group
*/
package org.h2.test.store;
import org.h2.dev.store.type.DataType;
import org.h2.dev.store.type.DataTypeFactory;
/**
* A data type factory.
*/
public class SampleTypeFactory implements DataTypeFactory {
private DataTypeFactory parent;
@Override
public void setParent(DataTypeFactory parent) {
this.parent = parent;
}
@Override
public DataType buildDataType(String s) {
// if ("org.h2.test.store.int".equals(s)) {
// return new IntegerType();
// } else
if (s.startsWith(RowType.PREFIX)) {
return RowType.fromString(s, this);
}
return parent.buildDataType(s);
}
@Override
public String getDataType(Class<?> objectClass) {
// if (objectClass == Integer.class) {
// return "org.h2.test.store.int";
// }
return parent.getDataType(objectClass);
}
}
......@@ -16,7 +16,7 @@ import org.h2.dev.store.btree.MVStore;
/**
* A custom map returning the keys and values values 1 .. 10.
*/
public class SequenceMap extends MVMap<Integer, String> {
public class SequenceMap extends MVMap<Long, Long> {
/**
* The minimum value.
......@@ -29,7 +29,7 @@ public class SequenceMap extends MVMap<Integer, String> {
int max = 10;
public SequenceMap() {
super(IntegerType.INSTANCE, IntegerType.INSTANCE);
super(null, null);
}
public void open(MVStore store, HashMap<String, String> config) {
......@@ -37,14 +37,14 @@ public class SequenceMap extends MVMap<Integer, String> {
setReadOnly(true);
}
public Set<Integer> keySet() {
return new AbstractSet<Integer>() {
public Set<Long> keySet() {
return new AbstractSet<Long>() {
@Override
public Iterator<Integer> iterator() {
return new Iterator<Integer>() {
public Iterator<Long> iterator() {
return new Iterator<Long>() {
int x = min;
long x = min;
@Override
public boolean hasNext() {
......@@ -52,8 +52,8 @@ public class SequenceMap extends MVMap<Integer, String> {
}
@Override
public Integer next() {
return Integer.valueOf(x++);
public Long next() {
return Long.valueOf(x++);
}
@Override
......
......@@ -17,6 +17,7 @@ import java.util.Random;
import org.h2.dev.store.btree.MVMap;
import org.h2.dev.store.btree.MVStore;
import org.h2.dev.store.btree.MVStoreBuilder;
import org.h2.dev.store.type.ObjectTypeFactory;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.Task;
......@@ -132,8 +133,8 @@ public class TestConcurrent extends TestMVStore {
private void testConcurrentIterate() {
MVStore s = MVStoreBuilder.inMemory().
with(new SampleDataTypeFactory()).open();
s.setMaxPageSize(3);
with(new ObjectTypeFactory()).open();
s.setPageSize(3);
final MVMap<Integer, Integer> map = s.openMap("test");
final int len = 10;
final Random r = new Random();
......
......@@ -21,7 +21,10 @@ import javax.imageio.ImageIO;
import javax.imageio.ImageWriter;
import javax.imageio.stream.FileImageOutputStream;
import org.h2.dev.store.btree.MVStore;
import org.h2.dev.store.btree.StringType;
import org.h2.dev.store.rtree.MVRTreeMap;
import org.h2.dev.store.rtree.SpatialKey;
import org.h2.dev.store.type.ObjectType;
import org.h2.dev.store.type.StringType;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
import org.h2.util.New;
......@@ -41,10 +44,38 @@ public class TestMVRTree extends TestMVStore {
}
public void test() {
testExample();
testMany();
testSimple();
testRandom();
testCustomMapType();
}
private void testExample() {
// create an in-memory store
MVStore s = MVStore.open(null);
// create an R-tree map
// the key has 2 dimensions, the value is a string
MVRTreeMap<String> r = MVRTreeMap.create(2, new ObjectType());
// open the map
r = s.openMap("data", r);
// add two key-value pairs
// the first value is the key id (to make the key unique)
// then the min x, max x, min y, max y
r.add(new SpatialKey(0, -3f, -2f, 2f, 3f), "left");
r.add(new SpatialKey(1, 3f, 4f, 4f, 5f), "right");
// iterate over the intersecting keys
Iterator<SpatialKey> it = r.findIntersectingKeys(
new SpatialKey(0, 0f, 9f, 3f, 6f));
for (SpatialKey k; it.hasNext();) {
k = it.next();
// System.out.println(k + ": " + r.get(k));
assertTrue(k != null);
}
s.close();
}
private void testMany() {
......@@ -276,19 +307,4 @@ public class TestMVRTree extends TestMVStore {
s.close();
}
private void testCustomMapType() {
String fileName = getBaseDir() + "/testMapType.h3";
FileUtils.delete(fileName);
MVStore s;
s = openStore(fileName);
SequenceMap seq = new SequenceMap();
seq = s.openMap("data", seq);
StringBuilder buff = new StringBuilder();
for (int x : seq.keySet()) {
buff.append(x).append(';');
}
assertEquals("1;2;3;4;5;6;7;8;9;10;", buff.toString());
s.close();
}
}
......@@ -14,11 +14,13 @@ import java.util.Map;
import java.util.Random;
import java.util.TreeMap;
import org.h2.dev.store.btree.Cursor;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.MVMap;
import org.h2.dev.store.btree.MVStore;
import org.h2.dev.store.btree.MVStoreBuilder;
import org.h2.dev.store.btree.StringType;
import org.h2.dev.store.type.DataType;
import org.h2.dev.store.type.ObjectType;
import org.h2.dev.store.type.ObjectTypeFactory;
import org.h2.dev.store.type.StringType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
import org.h2.test.TestBase;
......@@ -40,6 +42,7 @@ public class TestMVStore extends TestBase {
public void test() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
testCustomMapType();
testCacheSize();
testConcurrentOpen();
testFileHeader();
......@@ -70,12 +73,27 @@ public class TestMVStore extends TestBase {
testSimple();
}
private void testCustomMapType() {
String fileName = getBaseDir() + "/testMapType.h3";
FileUtils.delete(fileName);
MVStore s;
s = openStore(fileName);
SequenceMap seq = new SequenceMap();
seq = s.openMap("data", seq);
StringBuilder buff = new StringBuilder();
for (long x : seq.keySet()) {
buff.append(x).append(';');
}
assertEquals("1;2;3;4;5;6;7;8;9;10;", buff.toString());
s.close();
}
private void testCacheSize() {
String fileName = getBaseDir() + "/testCacheSize.h3";
MVStore s;
MVMap<Integer, String> map;
s = MVStoreBuilder.fileBased(fileName).
with(new SampleDataTypeFactory()).open();
with(new ObjectTypeFactory()).open();
map = s.openMap("test");
// add 10 MB of data
for (int i = 0; i < 1024; i++) {
......@@ -89,7 +107,7 @@ public class TestMVStore extends TestBase {
for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) {
s = MVStoreBuilder.fileBased(fileName).
cacheSizeMB(1 + 3 * cacheSize).
with(new SampleDataTypeFactory()).open();
with(new ObjectTypeFactory()).open();
map = s.openMap("test");
for (int i = 0; i < 1024; i += 128) {
for (int j = 0; j < i; j++) {
......@@ -179,7 +197,7 @@ public class TestMVStore extends TestBase {
private void testIndexSkip() {
MVStore s = openStore(null);
s.setMaxPageSize(4);
s.setPageSize(4);
MVMap<Integer, Integer> map = s.openMap("test");
for (int i = 0; i < 100; i += 2) {
map.put(i, 10 * i);
......@@ -240,7 +258,7 @@ public class TestMVStore extends TestBase {
for (int i = 3; i < 20; i++) {
s = openStore(null);
s.setMaxPageSize(4);
s.setPageSize(4);
map = s.openMap("test");
for (int j = 3; j < i; j++) {
map.put(j * 2, j * 20);
......@@ -310,7 +328,7 @@ public class TestMVStore extends TestBase {
MVStore s;
Map<Integer, Integer> map;
s = MVStoreBuilder.inMemory().
with(new SampleDataTypeFactory()).open();
with(new ObjectTypeFactory()).open();
map = s.openMap("test");
int len = 100;
for (int i = 0; i < len; i++) {
......@@ -336,7 +354,7 @@ public class TestMVStore extends TestBase {
MVStore s;
Map<Object, Object> map;
s = MVStoreBuilder.fileBased(fileName).
with(new SampleDataTypeFactory()).open();
with(new ObjectTypeFactory()).open();
map = s.openMap("test");
map.put(1, "Hello");
map.put("2", 200);
......@@ -344,7 +362,7 @@ public class TestMVStore extends TestBase {
s.store();
s.close();
s = MVStoreBuilder.fileBased(fileName).
with(new SampleDataTypeFactory()).open();
with(new ObjectTypeFactory()).open();
map = s.openMap("test");
assertEquals("Hello", map.get(1).toString());
assertEquals(200, ((Integer) map.get("2")).intValue());
......@@ -362,14 +380,12 @@ public class TestMVStore extends TestBase {
// open the store (in-memory if fileName is null)
MVStore s = MVStore.open(fileName);
// create/get the map "data"
// the String.class, String.class will be optional later
MVMap<String, String> map = s.openMap("data",
String.class, String.class);
// create/get the map named "data"
MVMap<Integer, String> map = s.openMap("data");
// add some data
map.put("1", "Hello");
map.put("2", "World");
map.put(1, "Hello");
map.put(2, "World");
// get the current version, for later use
long oldVersion = s.getCurrentVersion();
......@@ -380,11 +396,11 @@ public class TestMVStore extends TestBase {
// more changes, in the new version
// changes can be rolled back if required
// changes always go into 'head' (the newest version)
map.put("1", "Hi");
map.remove("2");
map.put(1, "Hi");
map.remove(2);
// access the old data (before incrementVersion)
MVMap<String, String> oldMap =
MVMap<Integer, String> oldMap =
map.openVersion(oldVersion);
// store the newest data to disk
......@@ -393,12 +409,12 @@ public class TestMVStore extends TestBase {
// print the old version (can be done
// concurrently with further modifications)
// this will print Hello World
// System.out.println(oldMap.get("1"));
// System.out.println(oldMap.get("2"));
// System.out.println(oldMap.get(1));
// System.out.println(oldMap.get(2));
oldMap.close();
// print the newest version ("Hi")
// System.out.println(map.get("1"));
// System.out.println(map.get(1));
// close the store - this doesn't write to disk
s.close();
......@@ -429,7 +445,7 @@ public class TestMVStore extends TestBase {
String fileName = getBaseDir() + "/testIterate.h3";
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
s.setMaxPageSize(6);
s.setPageSize(6);
MVMap<Integer, String> m = s.openMap("data", Integer.class, String.class);
for (int i = 0; i < 100; i++) {
m.put(i, "Hi");
......@@ -538,14 +554,14 @@ public class TestMVStore extends TestBase {
MVStore s;
MVMap<Integer, String> m;
s = openStore(fileName);
s.setMaxPageSize(700);
s.setPageSize(700);
m = s.openMap("data", Integer.class, String.class);
for (int i = 0; i < 1000; i++) {
m.put(i, "Hello World");
assertEquals(i + 1, m.size());
}
assertEquals(1000, m.size());
assertEquals(279, s.getUnsavedPageCount());
assertEquals(284, s.getUnsavedPageCount());
s.store();
assertEquals(3, s.getFileWriteCount());
s.close();
......@@ -556,7 +572,7 @@ public class TestMVStore extends TestBase {
assertEquals(0, m.size());
s.store();
// ensure only nodes are read, but not leaves
assertEquals(36, s.getFileReadCount());
assertEquals(41, s.getFileReadCount());
assertEquals(2, s.getFileWriteCount());
s.close();
}
......@@ -652,7 +668,7 @@ public class TestMVStore extends TestBase {
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
assertEquals(0, s.getCurrentVersion());
s.setMaxPageSize(5);
s.setPageSize(5);
MVMap<String, String> m = s.openMap("data", String.class, String.class);
s.rollbackTo(0);
assertTrue(m.isClosed());
......@@ -753,11 +769,11 @@ public class TestMVStore extends TestBase {
FileUtils.delete(fileName);
MVStore s = openStore(fileName);
// s.setCompressor(null);
s.setMaxPageSize(40);
s.setPageSize(40);
MVMap<Integer, Object[]> m = new MVMap<Integer, Object[]>(
IntegerType.INSTANCE,
new ObjectType(),
new RowType(new DataType[] {
IntegerType.INSTANCE,
new ObjectType(),
StringType.INSTANCE,
StringType.INSTANCE
})
......@@ -965,26 +981,9 @@ public class TestMVStore extends TestBase {
}
private void testKeyValueClasses() {
MVStore s;
s = MVStore.open(null);
s.openMap("test", String.class, String.class);
try {
s.openMap("unsupportedKey", ArrayList.class, String.class);
fail();
} catch (RuntimeException e) {
// expected
}
try {
s.openMap("unsupportedValue", String.class, ArrayList.class);
fail();
} catch (RuntimeException e) {
// expected
}
s.close();
String fileName = getBaseDir() + "/testKeyValueClasses.h3";
FileUtils.delete(fileName);
s = openStore(fileName);
MVStore s = openStore(fileName);
MVMap<Integer, String> is = s.openMap("intString", Integer.class, String.class);
is.put(1, "Hello");
MVMap<Integer, Integer> ii = s.openMap("intInt", Integer.class, Integer.class);
......@@ -1089,8 +1088,8 @@ public class TestMVStore extends TestBase {
*/
protected static MVStore openStore(String fileName) {
MVStore store = MVStoreBuilder.fileBased(fileName).
with(new SampleDataTypeFactory()).open();
store.setMaxPageSize(1000);
with(new SampleTypeFactory()).open();
store.setPageSize(1000);
return store;
}
......
......@@ -13,6 +13,7 @@ import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Random;
import java.util.UUID;
import org.h2.dev.store.type.ObjectType;
import org.h2.test.TestBase;
/**
......
......@@ -16,6 +16,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import org.h2.dev.store.type.DataType;
import org.h2.util.New;
/**
......@@ -72,7 +73,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
checkWrite();
long writeVersion = store.getCurrentVersion();
Page p = root.copyOnWrite(writeVersion);
if (p.getMemory() > store.getMaxPageSize() && p.getKeyCount() > 1) {
if (p.getMemory() > store.getPageSize() && p.getKeyCount() > 1) {
int at = p.getKeyCount() / 2;
long totalCount = p.getTotalCount();
Object k = p.getKey(at);
......@@ -118,7 +119,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
index++;
}
Page c = p.getChildPage(index).copyOnWrite(writeVersion);
if (c.getMemory() > store.getMaxPageSize() && c.getKeyCount() > 1) {
if (c.getMemory() > store.getPageSize() && c.getKeyCount() > 1) {
// split on the way down
int at = c.getKeyCount() / 2;
Object k = c.getKey(at);
......@@ -957,8 +958,12 @@ public class MVMap<K, V> extends AbstractMap<K, V>
DataUtils.appendMap(buff, "name", name);
DataUtils.appendMap(buff, "type", getType());
DataUtils.appendMap(buff, "createVersion", createVersion);
if (keyType != null) {
DataUtils.appendMap(buff, "key", keyType.asString());
}
if (valueType != null) {
DataUtils.appendMap(buff, "value", valueType.asString());
}
return buff.toString();
}
......
......@@ -19,8 +19,12 @@ import java.util.Iterator;
import java.util.Map;
import org.h2.compress.CompressLZF;
import org.h2.compress.Compressor;
import org.h2.dev.store.FilePathCache;
import org.h2.dev.store.cache.CacheLongKeyLIRS;
import org.h2.dev.store.cache.FilePathCache;
import org.h2.dev.store.type.DataType;
import org.h2.dev.store.type.DataTypeFactory;
import org.h2.dev.store.type.ObjectTypeFactory;
import org.h2.dev.store.type.StringType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FileUtils;
import org.h2.util.New;
......@@ -36,6 +40,9 @@ header:
H:3,...
TODO:
- support serialization by default
- build script
- test concurrent storing in a background thread
- store store creation in file header, and seconds since creation in chunk header (plus a counter)
- recovery: keep some old chunks; don't overwritten for 5 minutes (configurable)
- allocate memory with Utils.newBytes and so on
......@@ -69,9 +76,13 @@ TODO:
- triggers (can be implemented with a custom map)
- store write operations per page (maybe defragment if much different than count)
- r-tree: nearest neighbor search
- use FileChannel by default (nio file system)
- use FileChannel by default (nio file system), but: an interrupt close the FileChannel
- auto-save temporary data if it uses too much memory, but revert it on startup if needed.
- map and chunk metadata: do not store default values
- support maps without values (non-unique indexes), and maps without keys (counted b-tree)
- use a small object cache (StringCache)
- dump values
- tool to import / manipulate CSV files
*/
......@@ -92,7 +103,7 @@ public class MVStore {
private final String fileName;
private final DataTypeFactory dataTypeFactory;
private int maxPageSize = 4 * 1024;
private int pageSize = 6 * 1024;
private FileChannel file;
private FileLock fileLock;
......@@ -104,8 +115,7 @@ public class MVStore {
* split in 16 segments. The stack move distance is 2% of the expected
* number of entries.
*/
private final CacheLongKeyLIRS<Page> cache = CacheLongKeyLIRS.newInstance(
16 * 1024 * 1024, 2048, 16, 16 * 1024 * 1024 / 2048 * 2 / 100);
private CacheLongKeyLIRS<Page> cache;
private int lastChunkId;
private final HashMap<Integer, Chunk> chunks = New.hashMap();
......@@ -145,12 +155,21 @@ public class MVStore {
MVStore(HashMap<String, Object> config) {
this.config = config;
this.fileName = (String) config.get("fileName");
this.dataTypeFactory = (DataTypeFactory) config.get("dataTypeFactory");
DataTypeFactory parent = new ObjectTypeFactory();
DataTypeFactory f = (DataTypeFactory) config.get("dataTypeFactory");
if (f == null) {
f = parent;
} else {
f.setParent(parent);
}
this.dataTypeFactory = f;
this.readOnly = "r".equals(config.get("openMode"));
this.compressor = "0".equals(config.get("compression")) ? null : new CompressLZF();
if (fileName != null) {
Object s = config.get("cacheSize");
if (s != null) {
cache.setMaxMemory(Integer.parseInt(s.toString()) * 1024 * 1024);
int mb = s == null ? 16 : Integer.parseInt(s.toString());
cache = CacheLongKeyLIRS.newInstance(
mb * 1024 * 1024, 2048, 16, mb * 1024 * 1024 / 2048 * 2 / 100);
}
}
......@@ -519,7 +538,7 @@ public class MVStore {
return currentVersion;
}
// the last chunk was not completely correct in the last save()
// the last chunk was not completely correct in the last store()
// this needs to be updated now (it's better not to update right after
// storing, because that would modify the meta map again)
Chunk c = chunks.get(lastChunkId);
......@@ -944,24 +963,24 @@ public class MVStore {
}
/**
* Set the maximum amount of memory a page should contain, in bytes. Larger
* pages are split. The default is 4 KB. This is not a limit in the page
* size, as pages with one entry can be larger. As a rule of thumb, pages
* should not be larger than 1 MB, for caching to work efficiently.
* Set the amount of memory a page should contain at most, in bytes. Larger
* pages are split. The default is 6 KB. This is not a limit in the page
* size (pages with one entry can get larger), it is just the point where
* pages are split.
*
* @param maxPageSize the page size
* @param pageSize the page size
*/
public void setMaxPageSize(int maxPageSize) {
this.maxPageSize = maxPageSize;
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
/**
* Get the maximum page size, in bytes.
* Get the page size, in bytes.
*
* @return the maximum page size
* @return the page size
*/
public int getMaxPageSize() {
return maxPageSize;
public int getPageSize() {
return pageSize;
}
Compressor getCompressor() {
......
......@@ -7,6 +7,7 @@
package org.h2.dev.store.btree;
import java.util.HashMap;
import org.h2.dev.store.type.DataTypeFactory;
import org.h2.util.New;
/**
......
......@@ -19,7 +19,7 @@ import org.h2.store.fs.FileUtils;
/**
* Utility methods used in combination with the MVStore.
*/
public class MVStoreUtils {
public class MVStoreTool {
/**
* Runs this tool.
......
......@@ -11,6 +11,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Arrays;
import org.h2.compress.Compressor;
import org.h2.dev.store.type.DataType;
/**
* A page (a node or a leaf).
......
......@@ -148,7 +148,7 @@ public class CacheLongKeyLIRS<V> {
* @param key the key (may not be null)
* @return the old value, or null if there was no resident entry
*/
public synchronized V remove(long key) {
public V remove(long key) {
int hash = getHash(key);
return getSegment(hash).remove(key, hash);
}
......
......@@ -4,13 +4,12 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store;
package org.h2.dev.store.cache;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import org.h2.dev.store.cache.CacheLongKeyLIRS;
import org.h2.store.fs.FileBase;
import org.h2.store.fs.FilePathWrapper;
......
......@@ -4,15 +4,15 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
package org.h2.dev.store.rtree;
import java.util.ArrayList;
import java.util.Iterator;
import org.h2.dev.store.btree.Cursor;
import org.h2.dev.store.btree.CursorPos;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.MVMap;
import org.h2.dev.store.btree.Page;
import org.h2.dev.store.type.DataType;
import org.h2.util.New;
/**
......@@ -50,7 +50,7 @@ public class MVRTreeMap<V> extends MVMap<SpatialKey, V> {
checkOpen();
return new RTreeCursor(this, root, x) {
protected boolean check(boolean leaf, SpatialKey key, SpatialKey test) {
return keyType.contains(key, test);
return keyType.isOverlap(key, test);
}
};
}
......@@ -201,7 +201,7 @@ public class MVRTreeMap<V> extends MVMap<SpatialKey, V> {
Page p = root.copyOnWrite(writeVersion);
Object result;
if (alwaysAdd || get(key) == null) {
if (p.getMemory() > store.getMaxPageSize() && p.getKeyCount() > 1) {
if (p.getMemory() > store.getPageSize() && p.getKeyCount() > 1) {
// only possible if this is the root, else we would have split earlier
// (this requires maxPageSize is fixed)
long totalCount = p.getTotalCount();
......@@ -283,7 +283,7 @@ public class MVRTreeMap<V> extends MVMap<SpatialKey, V> {
}
}
Page c = p.getChildPage(index).copyOnWrite(writeVersion);
if (c.getMemory() > store.getMaxPageSize() && c.getKeyCount() > 1) {
if (c.getMemory() > store.getPageSize() && c.getKeyCount() > 1) {
// split on the way down
Page split = split(c, writeVersion);
p = p.copyOnWrite(writeVersion);
......
......@@ -4,7 +4,7 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
package org.h2.dev.store.rtree;
import java.util.Arrays;
......
......@@ -4,12 +4,12 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
package org.h2.dev.store.rtree;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataUtils;
import org.h2.dev.store.type.DataType;
/**
* A spatial data type. This class supports up to 255 dimensions. Each dimension
......
......@@ -4,7 +4,7 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store.btree;
package org.h2.dev.store.type;
import java.nio.ByteBuffer;
......@@ -58,6 +58,9 @@ public interface DataType {
/**
* Get the stable string representation that is used to build this data
* type.
* <p>
* To avoid conflict with the default factory, the returned string should
* start with the package name of the type factory.
*
* @return the string representation
*/
......
......@@ -4,27 +4,36 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store.btree;
package org.h2.dev.store.type;
/**
* A factory for maps and data types.
*/
public interface DataTypeFactory {
/**
* Set the parent factory.
*
* @param parent the parent factory
*/
void setParent(DataTypeFactory parent);
/**
* Parse the data type.
*
* @param dataType the string and type specific meta data
* @return the type
* @return the type, or null if unknown
*/
DataType buildDataType(String dataType);
/**
* Get the data type object for the given class.
* Get the data type identifier for the given class.
* <p>
* To avoid conflict with the default factory, the returned string should
* start with the package name of the type factory.
*
* @param objectClass the class
* @return the data type object
* @return the data type identifier, or null if not supported
*/
String getDataType(Class<?> objectClass);
......
......@@ -4,13 +4,12 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.test.store;
package org.h2.dev.store.type;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.UUID;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataUtils;
import org.h2.util.Utils;
......@@ -652,7 +651,7 @@ public class ObjectType implements DataType {
/**
* The type for long objects.
*/
class LongType extends AutoDetectDataType {
public class LongType extends AutoDetectDataType {
LongType(ObjectType base) {
super(base, TYPE_LONG);
......
......@@ -3,43 +3,36 @@
* 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html). Initial Developer: H2 Group
*/
package org.h2.test.store;
package org.h2.dev.store.type;
import org.h2.dev.store.btree.DataType;
import org.h2.dev.store.btree.DataTypeFactory;
import org.h2.dev.store.btree.StringType;
import org.h2.dev.store.rtree.SpatialType;
/**
* A data type factory.
*/
public class SampleDataTypeFactory implements DataTypeFactory {
public class ObjectTypeFactory implements DataTypeFactory {
@Override
public DataType buildDataType(String s) {
if (s.length() == 0) {
return new StringType();
public void setParent(DataTypeFactory parent) {
// never called for this factory
}
switch (s.charAt(0)) {
case 'i':
return new IntegerType();
case 'r':
return RowType.fromString(s, this);
case 's':
@Override
public DataType buildDataType(String s) {
if ("s".equals(s)) {
return SpatialType.fromString(s);
case 'o':
} else if ("o".equals(s)) {
return new ObjectType();
}
throw new RuntimeException("Unknown data type " + s);
return null;
}
@Override
public String getDataType(Class<?> objectClass) {
if (objectClass == Integer.class) {
return "i";
} else if (Object.class == Object.class) {
return "o";
if (objectClass == SpatialType.class) {
return "s";
}
throw new RuntimeException("Unsupported object class " + objectClass.toString());
return "o";
}
}
......@@ -4,9 +4,10 @@
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.dev.store.btree;
package org.h2.dev.store.type;
import java.nio.ByteBuffer;
import org.h2.dev.store.btree.DataUtils;
/**
* A string type.
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论