Skip to content

H
h2database
提交 001f9d04 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

The MultiDimension tools was extended with a few helper methods. The API was unified a bit.

上级 725532f8
master noel-pr1 plus33-master pr/267 stumc-Issue#576 version-1.4.198 version-1.4.197 version-1.4.196 version-1.4.195 version-1.4.194 version-1.4.193 version-1.4.192 version-1.4.191 version-1.4.190 version-1.4.188 version-1.4.187 version-1.4.186 version-1.4.185 version-1.4.184 version-1.4.183 version-1.4.182 version-1.4.181 version-1.4.178 version-1.4.177 version-1.3 version-1.2 version-1.1 version-1.0
隐藏空白字符变更
内嵌 并排
正在显示 包含 265 行增加 和 70 行删除
h2/src/docsrc/html/changelog.html
浏览文件 @ 001f9d04
......@@ -18,7 +18,9 @@ Change Log
<h1>Change Log</h1>
<h2>Next Version (unreleased)</h2>
<ul><li>ODBC: additional connection settings can now be added to the database name.
<ul><li>The MultiDimension tools was extended with a few helper methods.
The API was unified a bit.
</li><li>ODBC: additional connection settings can now be added to the database name.
Example: ~/test;cipher=xtea. Therefore, encrypted databases are supported.
</li><li>Local temporary tables can now be created without having to commit a transaction
using CREATE LOCAL TEMPORARY TABLE TEMP(ID INT PRIMARY KEY) TRANSACTIONAL.
......
h2/src/main/org/h2/tools/MultiDimension.java
浏览文件 @ 001f9d04
......@@ -37,65 +37,107 @@ public class MultiDimension implements Comparator<long[]> {
return INSTANCE;
}
/**
* Normalize a value so that it is between the minimum and maximum value for
* the given number of dimensions.
*
* @param dimensions the number of dimensions
* @param value the value (must be in the range min..max)
* @param min the minimum value
* @param max the maximum value (must be larger than min)
* @return the normalized value in the range 0..getMaxValue(dimensions)
*/
public int normalize(int dimensions, double value, double min, double max) {
if (value < min || value > max) {
throw new IllegalArgumentException(min + "<" + value + "<" + max);
}
double x = (value - min) / (max - min);
return (int) (x * getMaxValue(dimensions));
}
/**
* Get the maximum value for the given dimension count. For two dimensions,
* each value must contain at most 32 bit, for 3: 21 bit, 4: 16 bit, 5: 12
* bit, 6: 10 bit, 7: 9 bit, 8: 8 bit.
*
* @param dimensions the number of dimensions
* @return the maximum value
*/
public int getMaxValue(int dimensions) {
if (dimensions < 2 || dimensions > 32) {
throw new IllegalArgumentException("" + dimensions);
}
int bitsPerValue = getBitsPerValue(dimensions);
return (int) ((1L << bitsPerValue) - 1);
}
private int getBitsPerValue(int dimensions) {
return Math.min(31, 64 / dimensions);
}
/**
* Convert the multi-dimensional value into a one-dimensional (scalar) value.
* This is done by interleaving the bits of the values.
* Each values must be bigger or equal to 0. The maximum value
* depends on the number of dimensions. For two keys, it is 32 bit,
* for 3: 21 bit, 4: 16 bit, 5: 12 bit, 6: 10 bit, 7: 9 bit, 8: 8 bit.
* Each values must be between 0 (including) and the maximum value
* for the given number of dimensions (getMaxValue, excluding).
* To normalize values to this range, use the normalize function.
*
* @param values the multi-dimensional value
* @return the scalar value
*/
public long interleave(int[] values) {
public long interleave(int... values) {
int dimensions = values.length;
int bitsPerValue = 64 / dimensions;
// for 2 keys: 0x800000; 3: 0x
long max = getMaxValue(dimensions);
int bitsPerValue = getBitsPerValue(dimensions);
long x = 0;
for (int i = 0; i < dimensions; i++) {
long k = values[i];
if (k < 0 || k >= max) {
throw new IllegalArgumentException("value out of range; value=" + values[i] + " min=0 max=" + max);
if (k < 0 || k > max) {
throw new IllegalArgumentException(0 + "<" + k + "<" + max);
}
for (int b = 0; b < bitsPerValue; b++) {
x |= (k & (1L << b)) << (i + (dimensions - 1) * b);
}
}
if (dimensions == 2) {
long xx = getMorton2(values[0], values[1]);
if (xx != x) {
throw new IllegalArgumentException("test");
}
}
return x;
}
/**
* Get the maximum value for the given dimension count
* @param dimensions the number of dimensions
* Convert the two-dimensional value into a one-dimensional (scalar) value.
* This is done by interleaving the bits of the values.
* Each values must be between 0 (including) and the maximum value
* for the given number of dimensions (getMaxValue, excluding).
* To normalize values to this range, use the normalize function.
*
* @return the maximum value
* @param x the value of the first dimension, normalized
* @param y the value of the second dimension, normalized
* @return the scalar value
*/
public static long getMaxValue(int dimensions) {
if (dimensions < 2 || dimensions > 64) {
throw new IllegalArgumentException("dimensions: " + dimensions);
public long interleave(int x, int y) {
if (x < 0) {
throw new IllegalArgumentException(0 + "<" + x);
}
if (y < 0) {
throw new IllegalArgumentException(0 + "<" + y);
}
long z = 0;
for (int i = 0; i < 32; i++) {
z |= (x & (1L << i)) << i;
z |= (y & (1L << i)) << (i + 1);
}
int bitsPerValue = 64 / dimensions;
// for 2 keys: 0x800000; 3: 0x
return 1L << bitsPerValue;
return z;
}
/**
* Gets one of the original multi-dimensional values from a scalar value.
*
* @param scalar the scalar value
* @param dimensions the number of dimensions
* @param scalar the scalar value
* @param dim the dimension of the returned value (starting from 0)
* @return the value
*/
public int deinterleave(long scalar, int dimensions, int dim) {
int bitsPerValue = 64 / dimensions;
public int deinterleave(int dimensions, long scalar, int dim) {
int bitsPerValue = getBitsPerValue(dimensions);
int value = 0;
for (int i = 0; i < bitsPerValue; i++) {
value |= (scalar >> (dim + (dimensions - 1) * i)) & (1L << i);
......@@ -103,14 +145,6 @@ public class MultiDimension implements Comparator<long[]> {
return value;
}
// public static int get(long z, int d) {
// int n = 0;
// for (int i = 0; i < 31; i++) {
// n |= (z & (1 << (i + i + d))) >> (i + d);
// }
// return n;
// }
/**
* Generates an optimized multi-dimensional range query.
* The query contains parameters. It can only be used with the H2 database.
......@@ -172,7 +206,7 @@ public class MultiDimension implements Comparator<long[]> {
private long[][] getMortonRanges(int[] min, int[] max) {
int len = min.length;
if (max.length != len) {
throw new IllegalArgumentException("dimensions mismatch");
throw new IllegalArgumentException(len + "=" + max.length);
}
for (int i = 0; i < len; i++) {
if (min[i] > max[i]) {
......@@ -184,21 +218,12 @@ public class MultiDimension implements Comparator<long[]> {
int total = getSize(min, max, len);
ArrayList<long[]> list = New.arrayList();
addMortonRanges(list, min, max, len, 0);
optimize(list, total);
combineEntries(list, total);
long[][] ranges = new long[list.size()][2];
list.toArray(ranges);
return ranges;
}
private long getMorton2(int x, int y) {
long z = 0;
for (int i = 0; i < 32; i++) {
z |= (x & (1L << i)) << i;
z |= (y & (1L << i)) << (i + 1);
}
return z;
}
private int getSize(int[] min, int[] max, int len) {
int size = 1;
for (int i = 0; i < len; i++) {
......@@ -208,9 +233,15 @@ public class MultiDimension implements Comparator<long[]> {
return size;
}
private void optimize(ArrayList<long[]> list, int total) {
/**
* Combine entries if the size of the list is too large.
*
* @param list the size of the list
* @param total
*/
private void combineEntries(ArrayList<long[]> list, int total) {
Collections.sort(list, this);
for (int minGap = 10;; minGap += minGap / 2) {
for (int minGap = 10; minGap < total; minGap += minGap / 2) {
for (int i = 0; i < list.size() - 1; i++) {
long[] current = list.get(i);
long[] next = list.get(i + 1);
......@@ -224,9 +255,9 @@ public class MultiDimension implements Comparator<long[]> {
for (long[] range : list) {
searched += range[1] - range[0] + 1;
}
if (searched > 2 * total || list.size() < 3 /* || minGap > total */) {
if (searched > 2 * total || list.size() < 100) {
break;
}
}
}
}
......@@ -236,18 +267,18 @@ public class MultiDimension implements Comparator<long[]> {
private void addMortonRanges(ArrayList<long[]> list, int[] min, int[] max, int len, int level) {
if (level > 100) {
throw new IllegalArgumentException("Stop");
throw new IllegalArgumentException("" + level);
}
int largest = 0, largestDiff = 0;
long size = 1;
for (int i = 0; i < len; i++) {
int diff = max[i] - min[i];
if (diff < 0) {
throw new IllegalArgumentException("Stop");
throw new IllegalArgumentException(""+ diff);
}
size *= diff + 1;
if (size < 0) {
throw new IllegalArgumentException("Stop");
throw new IllegalArgumentException("" + size);
}
if (diff > largestDiff) {
largestDiff = diff;
......@@ -256,7 +287,7 @@ public class MultiDimension implements Comparator<long[]> {
}
long low = interleave(min), high = interleave(max);
if (high < low) {
throw new IllegalArgumentException("Stop");
throw new IllegalArgumentException(high + "<" + low);
}
long range = high - low + 1;
if (range == size) {
......@@ -294,7 +325,7 @@ public class MultiDimension implements Comparator<long[]> {
scale--;
int m = roundUp(a + 2, 1 << scale) - 1;
if (m <= a || m >= b) {
throw new IllegalArgumentException("stop");
throw new IllegalArgumentException(a + "<" + m + "<" + b);
}
return m;
}
......
h2/src/test/org/h2/test/db/TestMultiDimension.java
浏览文件 @ 001f9d04
......@@ -27,20 +27,173 @@ public class TestMultiDimension extends TestBase {
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
TestBase test = TestBase.createCaller().init();
test.config.traceTest = true;
test.test();
}
public void test() throws SQLException {
Random rand = new Random(10);
testHelperMethods();
testPerformance2d();
testPerformance3d();
}
private void testHelperMethods() {
MultiDimension m = MultiDimension.getInstance();
assertEquals(Integer.MAX_VALUE, m.getMaxValue(2));
assertEquals(0, m.normalize(2, 0, 0, 100));
assertEquals(Integer.MAX_VALUE / 2, m.normalize(2, 50, 0, 100));
assertEquals(Integer.MAX_VALUE, m.normalize(2, 100, 0, 100));
assertEquals(Integer.MAX_VALUE / 10, m.normalize(2, 0.1, 0, 1));
assertEquals(0, m.normalize(2, 1, 1, 1));
assertEquals(0, m.normalize(2, 0, 0, 0));
assertEquals(3, m.interleave(1, 1));
assertEquals(3, m.interleave(new int[]{1, 1}));
assertEquals(5, m.interleave(3, 0));
assertEquals(5, m.interleave(new int[]{3, 0}));
assertEquals(10, m.interleave(0, 3));
assertEquals(10, m.interleave(new int[]{0, 3}));
long v = ((long) Integer.MAX_VALUE | ((long) Integer.MAX_VALUE << 31L));
assertEquals(v, m.interleave(Integer.MAX_VALUE, Integer.MAX_VALUE));
assertEquals(v, m.interleave(new int[] {Integer.MAX_VALUE, Integer.MAX_VALUE }));
Random random = new Random(1);
for (int i=0; i<1000; i++) {
int x = random.nextInt(Integer.MAX_VALUE), y = random.nextInt(Integer.MAX_VALUE);
v = m.interleave(new int[] {x, y});
long v2 = m.interleave(x, y);
assertEquals(v, v2);
int x1 = m.deinterleave(2, v, 0);
int y1 = m.deinterleave(2, v, 1);
assertEquals(x, x1);
assertEquals(y, y1);
}
for (int i = 0; i < 1000; i++) {
int x = rand.nextInt(1000), y = rand.nextInt(1000), z = rand.nextInt(1000);
int x = random.nextInt(1000), y = random.nextInt(1000), z = random.nextInt(1000);
MultiDimension tool = MultiDimension.getInstance();
long xyz = tool.interleave(new int[] { x, y, z });
assertEquals(x, tool.deinterleave(xyz, 3, 0));
assertEquals(y, tool.deinterleave(xyz, 3, 1));
assertEquals(z, tool.deinterleave(xyz, 3, 2));
assertEquals(x, tool.deinterleave(3, xyz, 0));
assertEquals(y, tool.deinterleave(3, xyz, 1));
assertEquals(z, tool.deinterleave(3, xyz, 2));
}
try {
m.getMaxValue(1);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.getMaxValue(33);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.normalize(2, 10, 11, 12);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.normalize(2, 5, 10, 0);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.normalize(2, 10, 0, 9);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.interleave(-1, 5);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.interleave(5, -1);
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
m.interleave(Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE);
fail();
} catch (IllegalArgumentException e) {
// expected
}
}
private void testPerformance2d() throws SQLException {
deleteDb("multiDimension");
Connection conn;
conn = getConnection("multiDimension");
Statement stat = conn.createStatement();
stat.execute("CREATE ALIAS MAP FOR \"" + getClass().getName() + ".interleave\"");
stat.execute("CREATE TABLE TEST(X INT NOT NULL, Y INT NOT NULL, " +
"XY BIGINT AS MAP(X, Y), DATA VARCHAR)");
stat.execute("CREATE INDEX IDX_X ON TEST(X, Y)");
stat.execute("CREATE INDEX IDX_XY ON TEST(XY)");
PreparedStatement prep = conn.prepareStatement(
"INSERT INTO TEST(X, Y, DATA) VALUES(?, ?, ?)");
// the MultiDimension tool is faster for 4225 (65^2) points
// the more the bigger the difference
int max = getSize(30, 65);
long time = System.currentTimeMillis();
for (int x = 0; x < max; x++) {
for (int y = 0; y < max; y++) {
long t2 = System.currentTimeMillis();
if (t2 - time > 1000) {
int percent = (int) (100.0 * ((double) x * max + y) / ((double) max * max));
trace(percent + "%");
time = t2;
}
prep.setInt(1, x);
prep.setInt(2, y);
prep.setString(3, "Test data");
prep.execute();
}
}
stat.execute("ANALYZE SAMPLE_SIZE 10000");
PreparedStatement prepRegular = conn.prepareStatement(
"SELECT * FROM TEST WHERE X BETWEEN ? AND ? " +
"AND Y BETWEEN ? AND ? ORDER BY X, Y");
MultiDimension multi = MultiDimension.getInstance();
String sql = multi.generatePreparedQuery("TEST", "XY", new String[] { "X", "Y" });
sql += " ORDER BY X, Y";
PreparedStatement prepMulti = conn.prepareStatement(sql);
long timeMulti = 0, timeRegular = 0;
int timeMax = getSize(500, 2000);
Random rand = new Random(1);
for (int i = 0; timeMulti < timeMax; i++) {
int size = rand.nextInt(max / 10);
int minX = rand.nextInt(max - size);
int minY = rand.nextInt(max - size);
int maxX = minX + size, maxY = minY + size;
time = System.currentTimeMillis();
ResultSet rs1 = multi.getResult(prepMulti, new int[] { minX, minY }, new int[] { maxX, maxY });
timeMulti += System.currentTimeMillis() - time;
time = System.currentTimeMillis();
prepRegular.setInt(1, minX);
prepRegular.setInt(2, maxX);
prepRegular.setInt(3, minY);
prepRegular.setInt(4, maxY);
ResultSet rs2 = prepRegular.executeQuery();
timeRegular += System.currentTimeMillis() - time;
while (rs1.next()) {
assertTrue(rs2.next());
assertEquals(rs1.getInt(1), rs2.getInt(1));
assertEquals(rs1.getInt(2), rs2.getInt(2));
}
assertFalse(rs2.next());
}
conn.close();
deleteDb("multiDimension");
trace("2d: regular: " + timeRegular + " MultiDimension: " + timeMulti);
}
private void testPerformance3d() throws SQLException {
deleteDb("multiDimension");
Connection conn;
conn = getConnection("multiDimension");
......@@ -52,8 +205,8 @@ public class TestMultiDimension extends TestBase {
stat.execute("CREATE INDEX IDX_XYZ ON TEST(XYZ)");
PreparedStatement prep = conn.prepareStatement(
"INSERT INTO TEST(X, Y, Z, DATA) VALUES(?, ?, ?, ?)");
// a reasonable max value to see the performance difference is 60; the
// higher the bigger the difference
// the MultiDimension tool is faster for 8000 (20^3) points
// the more the bigger the difference
int max = getSize(10, 20);
long time = System.currentTimeMillis();
for (int x = 0; x < max; x++) {
......@@ -61,14 +214,9 @@ public class TestMultiDimension extends TestBase {
for (int z = 0; z < max; z++) {
long t2 = System.currentTimeMillis();
if (t2 - time > 1000) {
int percent = (int) (100.0 * ((double) x * x * x) / ((double) max * max * max));
int percent = (int) (100.0 * ((double) x * max + y) / ((double) max * max));
trace(percent + "%");
time = t2;
try {
Thread.sleep(10);
} catch (Exception e) {
// ignore
}
}
prep.setInt(1, x);
prep.setInt(2, y);
......@@ -87,7 +235,8 @@ public class TestMultiDimension extends TestBase {
sql += " ORDER BY X, Y, Z";
PreparedStatement prepMulti = conn.prepareStatement(sql);
long timeMulti = 0, timeRegular = 0;
int timeMax = getSize(100, 2000);
int timeMax = getSize(500, 2000);
Random rand = new Random(1);
for (int i = 0; timeMulti < timeMax; i++) {
int size = rand.nextInt(max / 10);
int minX = rand.nextInt(max - size);
......@@ -115,6 +264,18 @@ public class TestMultiDimension extends TestBase {
}
conn.close();
deleteDb("multiDimension");
trace("3d: regular: " + timeRegular + " MultiDimension: " + timeMulti);
}
/**
* This method is called via reflection from the database.
*
* @param x the x value
* @param y the y value
* @return the bit-interleaved value
*/
public static long interleave(int x, int y) {
return MultiDimension.getInstance().interleave(x, y);
}
/**
......@@ -128,4 +289,5 @@ public class TestMultiDimension extends TestBase {
public static long interleave(int x, int y, int z) {
return MultiDimension.getInstance().interleave(new int[] { x, y, z });
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论