提交 50e52985 authored 作者: Thomas Mueller's avatar Thomas Mueller

Page store: reduce database size.

上级 d0f7074a
......@@ -7,7 +7,11 @@
package org.h2.index;
/**
* A page.
* A page. Format:
* <ul><li>0-3: parent page id (0 for root)
* </li><li>4-4: page type
* </li><li>page-type specific data
* </li></ul>
*/
public class Page {
......
......@@ -60,17 +60,21 @@ class PageBtreeLeaf extends PageBtree {
int last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
if (last - rowLength < start + OFFSET_LENGTH) {
if (entryCount > 1) {
return entryCount / 2;
// split at the insertion point to better fill pages
// split in half would be:
// return entryCount / 2;
int x = find(row, false, true, true);
return x < 2 ? 2 : x >= entryCount - 3 ? entryCount - 3 : x;
}
onlyPosition = true;
// change the offsets (now storing only positions)
int o = pageSize;
for (int i = 0; i < entryCount; i++) {
o -= index.getRowSize(data, getRow(i), onlyPosition);
o -= index.getRowSize(data, getRow(i), true);
offsets[i] = o;
}
last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
rowLength = index.getRowSize(data, row, onlyPosition);
rowLength = index.getRowSize(data, row, true);
if (SysProperties.CHECK && last - rowLength < start + OFFSET_LENGTH) {
throw Message.throwInternalError();
}
......
......@@ -105,11 +105,11 @@ class PageBtreeNode extends PageBtree {
// change the offsets (now storing only positions)
int o = pageSize;
for (int i = 0; i < entryCount; i++) {
o -= index.getRowSize(data, getRow(i), onlyPosition);
o -= index.getRowSize(data, getRow(i), true);
offsets[i] = o;
}
last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
rowLength = index.getRowSize(data, row, onlyPosition);
rowLength = index.getRowSize(data, row, true);
if (SysProperties.CHECK && last - rowLength < start + CHILD_OFFSET_PAIR_LENGTH) {
throw Message.throwInternalError();
}
......
......@@ -97,9 +97,11 @@ class PageDataLeaf extends PageData {
int pageSize = index.getPageStore().getPageSize();
int last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
if (entryCount > 0 && last - rowLength < start + KEY_OFFSET_PAIR_LENGTH) {
if (entryCount > 1) {
return entryCount / 2;
}
// split at the insertion point to better fill pages
// split in half would be:
// if (entryCount > 1) {
// return entryCount / 2;
// }
return find(row.getPos());
}
int offset = last - rowLength;
......
......@@ -19,6 +19,6 @@ org.h2.tools.RunScript.main=Options are case sensitive. Supported options are\:\
org.h2.tools.Script=Creates a SQL script file by extracting the schema and data of a database.
org.h2.tools.Script.main=Options are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-url "<url>"] The database URL (jdbc\:...)\n[-user <user>] The user name (default\: sa)\n[-password <pwd>] The password\n[-script <file>] The target script file name (default\: backup.sql)\n[-options ...] A list of options (only for embedded H2, see RUNSCRIPT)\n[-quiet] Do not print progress information
org.h2.tools.Server=Starts the H2 Console (web-) server, TCP, and PG server.
org.h2.tools.Server.main=When running without options, -tcp, -web, -browser and -pg are started.\nOptions are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-web] Start the web server with the H2 Console\n[-webAllowOthers] Allow other computers to connect\n[-webPort <port>] The port (default\: 8082)\n[-webSSL] Use encrypted (HTTPS) connections\n[-browser] Start a browser and open a page to connect to the web server\n[-tcp] Start the TCP server\n[-tcpAllowOthers] Allow other computers to connect\n[-tcpPort <port>] The port (default\: 9092)\n[-tcpSSL] Use encrypted (SSL) connections\n[-tcpPassword <pwd>] The password for shutting down a TCP server\n[-tcpShutdown "<url>"] Stop the TCP server; example\: tcp\://localhost\:9094\n[-tcpShutdownForce] Do not wait until all connections are closed\n[-pg] Start the PG server\n[-pgAllowOthers] Allow other computers to connect\n[-pgPort <port>] The port (default\: 5435)\n[-baseDir <dir>] The base directory for H2 databases; for all servers\n[-ifExists] Only existing databases may be opened; for all servers\n[-trace] Print additional trace information; for all servers
org.h2.tools.Server.main=When running without options, -tcp, -web, -browser and -pg are started.\nOptions are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-web] Start the web server with the H2 Console\n[-webAllowOthers] Allow other computers to connect - see below\n[-webPort <port>] The port (default\: 8082)\n[-webSSL] Use encrypted (HTTPS) connections\n[-browser] Start a browser and open a page to connect to the web server\n[-tcp] Start the TCP server\n[-tcpAllowOthers] Allow other computers to connect - see below\n[-tcpPort <port>] The port (default\: 9092)\n[-tcpSSL] Use encrypted (SSL) connections\n[-tcpPassword <pwd>] The password for shutting down a TCP server\n[-tcpShutdown "<url>"] Stop the TCP server; example\: tcp\://localhost\:9094\n[-tcpShutdownForce] Do not wait until all connections are closed\n[-pg] Start the PG server\n[-pgAllowOthers] Allow other computers to connect - see below\n[-pgPort <port>] The port (default\: 5435)\n[-baseDir <dir>] The base directory for H2 databases; for all servers\n[-ifExists] Only existing databases may be opened; for all servers\n[-trace] Print additional trace information; for all servers\nThe options -xAllowOthers are potentially risky.\nFor details, see Advanced Topics / Protection against Remote Access.
org.h2.tools.Shell=Interactive command line tool to access a database using JDBC.
org.h2.tools.Shell.main=Options are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-url "<url>"] The database URL (jdbc\:h2\:...)\n[-user <user>] The user name\n[-password <pwd>] The password\n[-driver <class>] The JDBC driver class to use (not required in most cases)\nIf special characters don't work as expected, you may need to use\n -Dfile.encoding\=UTF-8 (Mac OS X) or CP850 (Windows).
......@@ -3,16 +3,20 @@
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*
* The variable size number format code is a port from SQLite,
* but stored in reverse order (least significant bits in the first byte).
*/
package org.h2.store;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import org.h2.constant.SysProperties;
import org.h2.message.Message;
import org.h2.util.DateTimeUtils;
import org.h2.util.MemoryUtils;
import org.h2.value.Value;
import org.h2.value.ValueArray;
......@@ -41,6 +45,22 @@ import org.h2.value.ValueUuid;
*/
public class Data extends DataPage {
private static final int INT_0_15 = 32;
private static final int LONG_0_7 = 48;
private static final int DECIMAL_0_1 = 56;
private static final int DECIMAL_SMALL_0 = 58;
private static final int DECIMAL_SMALL = 59;
private static final int DOUBLE_0_1 = 60;
private static final int FLOAT_0_1 = 62;
private static final int BOOLEAN_FALSE = 64;
private static final int BOOLEAN_TRUE = 65;
private static final int INT_NEG = 66;
private static final int LONG_NEG = 67;
private static final int STRING_0_31 = 68;
private static final int BYTES_0_31 = 100;
private static final long MILLIS_PER_MINUTE = 1000 * 60;
private Data(DataHandler handler, byte[] data) {
super(handler, data);
}
......@@ -92,7 +112,21 @@ public class Data extends DataPage {
* @return the length
*/
public int getStringLen(String s) {
return getStringLenUTF8(s);
int len = s.length();
return getStringWithoutLengthLen(s, len) + getVarIntLen(len);
}
private int getStringWithoutLengthLen(String s, int len) {
int plus = 0;
for (int i = 0; i < len; i++) {
char c = s.charAt(i);
if (c >= 0x800) {
plus += 2;
} else if (c >= 0x80) {
plus++;
}
}
return len + plus;
}
/**
......@@ -102,10 +136,13 @@ public class Data extends DataPage {
* @return the value
*/
public String readString() {
int len = readVarInt();
return readString(len);
}
private String readString(int len) {
byte[] buff = data;
int p = pos;
int len = ((buff[p++] & 0xff) << 24) + ((buff[p++] & 0xff) << 16) + ((buff[p++] & 0xff) << 8)
+ (buff[p++] & 0xff);
char[] chars = new char[len];
for (int i = 0; i < len; i++) {
int x = buff[p++] & 0xff;
......@@ -121,7 +158,6 @@ public class Data extends DataPage {
return new String(chars);
}
/**
* Write a String value.
* The current position is incremented.
......@@ -130,23 +166,24 @@ public class Data extends DataPage {
*/
public void writeString(String s) {
int len = s.length();
writeVarInt(len);
writeStringWithoutLength(s, len);
}
private void writeStringWithoutLength(String s, int len) {
int p = pos;
byte[] buff = data;
buff[p++] = (byte) (len >> 24);
buff[p++] = (byte) (len >> 16);
buff[p++] = (byte) (len >> 8);
buff[p++] = (byte) len;
for (int i = 0; i < len; i++) {
int c = s.charAt(i);
if (c > 0 && c < 0x80) {
if (c < 0x80) {
buff[p++] = (byte) c;
} else if (c >= 0x800) {
buff[p++] = (byte) (0xe0 | (c >> 12));
buff[p++] = (byte) (0x80 | ((c >> 6) & 0x3f));
buff[p++] = (byte) (0x80 | (c & 0x3f));
buff[p++] = (byte) (((c >> 6) & 0x3f));
buff[p++] = (byte) (c & 0x3f);
} else {
buff[p++] = (byte) (0xc0 | (c >> 6));
buff[p++] = (byte) (0x80 | (c & 0x3f));
buff[p++] = (byte) (c & 0x3f);
}
}
pos = p;
......@@ -296,91 +333,189 @@ public class Data extends DataPage {
* @param v the value
*/
public void writeValue(Value v) throws SQLException {
// TODO text output: could be in the Value... classes
if (v == ValueNull.INSTANCE) {
data[pos++] = '-';
data[pos++] = 0;
return;
}
int start = pos;
data[pos++] = (byte) (v.getType() + 'a');
switch (v.getType()) {
int type = v.getType();
switch (type) {
case Value.BOOLEAN:
writeByte((byte) (v.getBoolean().booleanValue() ? BOOLEAN_TRUE : BOOLEAN_FALSE));
break;
case Value.BYTE:
writeByte((byte) type);
writeByte(v.getByte());
break;
case Value.SHORT:
case Value.INT:
writeInt(v.getInt());
writeByte((byte) type);
writeShortInt(v.getShort());
break;
case Value.LONG:
writeLong(v.getLong());
case Value.INT: {
int x = v.getInt();
if (x < 0) {
writeByte((byte) INT_NEG);
writeVarInt(-x);
} else if (x < 16) {
writeByte((byte) (INT_0_15 + x));
} else {
writeByte((byte) type);
writeVarInt(x);
}
break;
case Value.DECIMAL:
String s = v.getString();
writeString(s);
}
case Value.LONG: {
long x = v.getLong();
if (x < 0) {
writeByte((byte) LONG_NEG);
writeVarLong(-x);
} else if (x < 8) {
writeByte((byte) (LONG_0_7 + x));
} else {
writeByte((byte) type);
writeVarLong(x);
}
break;
}
case Value.DECIMAL: {
BigDecimal x = v.getBigDecimal();
if (BigDecimal.ZERO.equals(x)) {
writeByte((byte) DECIMAL_0_1);
} else if (BigDecimal.ONE.equals(x)) {
writeByte((byte) (DECIMAL_0_1 + 1));
} else {
int scale = x.scale();
BigInteger b = x.unscaledValue();
int bits = b.bitLength();
if (bits <= 63) {
if (scale == 0) {
writeByte((byte) DECIMAL_SMALL_0);
writeVarLong(b.longValue());
} else {
writeByte((byte) DECIMAL_SMALL);
writeVarInt(scale);
writeVarLong(b.longValue());
}
} else {
writeByte((byte) type);
writeVarInt(scale);
byte[] bytes = b.toByteArray();
writeVarInt(bytes.length);
write(bytes, 0, bytes.length);
}
}
break;
}
case Value.TIME:
writeLong(v.getTimeNoCopy().getTime());
writeByte((byte) type);
writeVarLong(DateTimeUtils.getTimeLocal(v.getTimeNoCopy()));
break;
case Value.DATE:
writeLong(v.getDateNoCopy().getTime());
case Value.DATE: {
writeByte((byte) type);
long x = DateTimeUtils.getTimeLocal(v.getDateNoCopy());
writeVarLong(x / MILLIS_PER_MINUTE);
break;
}
case Value.TIMESTAMP: {
writeByte((byte) type);
Timestamp ts = v.getTimestampNoCopy();
writeLong(ts.getTime());
writeInt(ts.getNanos());
writeVarLong(DateTimeUtils.getTimeLocal(ts));
writeVarInt(ts.getNanos());
break;
}
case Value.JAVA_OBJECT: {
writeByte((byte) type);
byte[] b = v.getBytesNoCopy();
writeVarInt(b.length);
write(b, 0, b.length);
break;
}
case Value.JAVA_OBJECT:
case Value.BYTES: {
byte[] b = v.getBytesNoCopy();
writeInt(b.length);
int len = b.length;
if (len < 32) {
writeByte((byte) (BYTES_0_31 + len));
write(b, 0, b.length);
} else {
writeByte((byte) type);
writeVarInt(b.length);
write(b, 0, b.length);
}
break;
}
case Value.UUID: {
writeByte((byte) type);
ValueUuid uuid = (ValueUuid) v;
writeLong(uuid.getHigh());
writeLong(uuid.getLow());
break;
}
case Value.STRING:
case Value.STRING: {
String s = v.getString();
int len = s.length();
if (len < 32) {
writeByte((byte) (STRING_0_31 + len));
writeStringWithoutLength(s, len);
} else {
writeByte((byte) type);
writeString(s);
}
break;
}
case Value.STRING_IGNORECASE:
case Value.STRING_FIXED:
writeByte((byte) type);
writeString(v.getString());
break;
case Value.DOUBLE:
writeLong(Double.doubleToLongBits(v.getDouble()));
case Value.DOUBLE: {
double x = v.getDouble();
if (x == 0.0 || x == 1.0) {
writeByte((byte) (DOUBLE_0_1 + x));
} else {
writeByte((byte) type);
writeVarLong(Long.reverse(Double.doubleToLongBits(x)));
}
break;
case Value.FLOAT:
writeInt(Float.floatToIntBits(v.getFloat()));
}
case Value.FLOAT: {
float x = v.getFloat();
if (x == 0.0f || x == 1.0f) {
writeByte((byte) (FLOAT_0_1 + x));
} else {
writeByte((byte) type);
writeVarInt(Integer.reverse(Float.floatToIntBits(v.getFloat())));
}
break;
}
case Value.BLOB:
case Value.CLOB: {
writeByte((byte) type);
ValueLob lob = (ValueLob) v;
lob.convertToFileIfRequired(handler);
byte[] small = lob.getSmall();
if (small == null) {
// -2 for historical reasons (-1 didn't store precision)
int type = -2;
int t = -1;
if (!lob.isLinked()) {
type = -3;
t = -2;
}
writeInt(type);
writeInt(lob.getTableId());
writeInt(lob.getObjectId());
writeLong(lob.getPrecision());
writeVarInt(t);
writeVarInt(lob.getTableId());
writeVarInt(lob.getObjectId());
writeVarLong(lob.getPrecision());
writeByte((byte) (lob.useCompression() ? 1 : 0));
if (type == -3) {
if (t == -2) {
writeString(lob.getFileName());
}
} else {
writeInt(small.length);
writeVarInt(small.length);
write(small, 0, small.length);
}
break;
}
case Value.ARRAY: {
writeByte((byte) type);
Value[] list = ((ValueArray) v).getList();
writeInt(list.length);
writeVarInt(list.length);
for (Value x : list) {
writeValue(x);
}
......@@ -397,121 +532,73 @@ public class Data extends DataPage {
}
}
/**
* Calculate the number of bytes required to encode the given value.
*
* @param v the value
* @return the number of bytes required to store this value
*/
public int getValueLen(Value v) throws SQLException {
if (v == ValueNull.INSTANCE) {
return 1;
}
switch (v.getType()) {
case Value.BOOLEAN:
case Value.BYTE:
case Value.SHORT:
case Value.INT:
return 1 + LENGTH_INT;
case Value.LONG:
return 1 + LENGTH_LONG;
case Value.DOUBLE:
return 1 + LENGTH_LONG;
case Value.FLOAT:
return 1 + LENGTH_INT;
case Value.STRING:
case Value.STRING_IGNORECASE:
case Value.STRING_FIXED:
return 1 + getStringLen(v.getString());
case Value.DECIMAL:
return 1 + getStringLen(v.getString());
case Value.JAVA_OBJECT:
case Value.BYTES: {
int len = v.getBytesNoCopy().length;
return 1 + LENGTH_INT + len;
}
case Value.UUID:
return 1 + LENGTH_LONG + LENGTH_LONG;
case Value.TIME:
return 1 + LENGTH_LONG;
case Value.DATE:
return 1 + LENGTH_LONG;
case Value.TIMESTAMP:
return 1 + LENGTH_LONG + LENGTH_INT;
case Value.BLOB:
case Value.CLOB: {
int len = 1;
ValueLob lob = (ValueLob) v;
lob.convertToFileIfRequired(handler);
byte[] small = lob.getSmall();
if (small != null) {
len += LENGTH_INT + small.length;
} else {
len += LENGTH_INT + LENGTH_INT + LENGTH_INT + LENGTH_LONG + 1;
if (!lob.isLinked()) {
len += getStringLen(lob.getFileName());
}
}
return len;
}
case Value.ARRAY: {
Value[] list = ((ValueArray) v).getList();
int len = 1 + LENGTH_INT;
for (Value x : list) {
len += getValueLen(x);
}
return len;
}
default:
throw Message.throwInternalError("type=" + v.getType());
}
}
/**
* Read a value.
*
* @return the value
*/
public Value readValue() throws SQLException {
int dataType = data[pos++];
if (dataType == '-') {
int type = data[pos++] & 255;
switch (type) {
case Value.NULL:
return ValueNull.INSTANCE;
}
dataType = dataType - 'a';
switch (dataType) {
case Value.BOOLEAN:
return ValueBoolean.get(readInt() == 1);
case Value.BYTE:
return ValueByte.get((byte) readInt());
case Value.SHORT:
return ValueShort.get((short) readInt());
case BOOLEAN_TRUE:
return ValueBoolean.get(true);
case BOOLEAN_FALSE:
return ValueBoolean.get(false);
case INT_NEG:
return ValueInt.get(-readVarInt());
case Value.INT:
return ValueInt.get(readInt());
return ValueInt.get(readVarInt());
case LONG_NEG:
return ValueLong.get(-readVarLong());
case Value.LONG:
return ValueLong.get(readLong());
case Value.DECIMAL:
return ValueDecimal.get(new BigDecimal(readString()));
case Value.DATE:
return ValueDate.getNoCopy(new Date(readLong()));
return ValueLong.get(readVarLong());
case Value.BYTE:
return ValueByte.get((byte) readByte());
case Value.SHORT:
return ValueShort.get((short) readShortInt());
case DECIMAL_0_1:
return (ValueDecimal) ValueDecimal.ZERO;
case DECIMAL_0_1 + 1:
return (ValueDecimal) ValueDecimal.ONE;
case DECIMAL_SMALL_0:
return ValueDecimal.get(BigDecimal.valueOf(readVarLong()));
case DECIMAL_SMALL: {
int scale = readVarInt();
return ValueDecimal.get(BigDecimal.valueOf(readVarLong(), scale));
}
case Value.DECIMAL: {
int scale = readVarInt();
int len = readVarInt();
byte[] data = MemoryUtils.newBytes(len);
read(data, 0, len);
BigInteger b = new BigInteger(data);
return ValueDecimal.get(new BigDecimal(b, scale));
}
case Value.DATE: {
long x = readVarLong() * MILLIS_PER_MINUTE;
return ValueDate.getNoCopy(new Date(DateTimeUtils.getTimeGMT(x)));
}
case Value.TIME:
// need to normalize the year, month and day
return ValueTime.get(new Time(readLong()));
return ValueTime.get(new Time(DateTimeUtils.getTimeGMT(readVarLong())));
case Value.TIMESTAMP: {
Timestamp ts = new Timestamp(readLong());
ts.setNanos(readInt());
Timestamp ts = new Timestamp(DateTimeUtils.getTimeGMT(readVarLong()));
ts.setNanos(readVarInt());
return ValueTimestamp.getNoCopy(ts);
}
case Value.JAVA_OBJECT: {
int len = readInt();
case Value.BYTES: {
int len = readVarInt();
byte[] b = MemoryUtils.newBytes(len);
read(b, 0, len);
return ValueJavaObject.getNoCopy(b);
return ValueBytes.getNoCopy(b);
}
case Value.BYTES: {
int len = readInt();
case Value.JAVA_OBJECT: {
int len = readVarInt();
byte[] b = MemoryUtils.newBytes(len);
read(b, 0, len);
return ValueBytes.getNoCopy(b);
return ValueJavaObject.getNoCopy(b);
}
case Value.UUID:
return ValueUuid.get(readLong(), readLong());
......@@ -521,37 +608,45 @@ public class Data extends DataPage {
return ValueStringIgnoreCase.get(readString());
case Value.STRING_FIXED:
return ValueStringFixed.get(readString());
case FLOAT_0_1:
return ValueFloat.get(0);
case FLOAT_0_1 + 1:
return ValueFloat.get(1);
case DOUBLE_0_1:
return ValueDouble.get(0);
case DOUBLE_0_1 + 1:
return ValueDouble.get(1);
case Value.DOUBLE:
return ValueDouble.get(Double.longBitsToDouble(readLong()));
return ValueDouble.get(Double.longBitsToDouble(Long.reverse(readVarLong())));
case Value.FLOAT:
return ValueFloat.get(Float.intBitsToFloat(readInt()));
return ValueFloat.get(Float.intBitsToFloat(Integer.reverse(readVarInt())));
case Value.BLOB:
case Value.CLOB: {
int smallLen = readInt();
int smallLen = readVarInt();
if (smallLen >= 0) {
byte[] small = MemoryUtils.newBytes(smallLen);
read(small, 0, smallLen);
return ValueLob.createSmallLob(dataType, small);
return ValueLob.createSmallLob(type, small);
}
int tableId = readInt();
int objectId = readInt();
int tableId = readVarInt();
int objectId = readVarInt();
long precision = 0;
boolean compression = false;
// -1: historical (didn't store precision)
// -2: regular
// -3: regular, but not linked (in this case: including file name)
if (smallLen == -2 || smallLen == -3) {
precision = readLong();
// TODO simplify
// -1: regular
// -2: regular, but not linked (in this case: including file name)
if (smallLen == -1 || smallLen == -2) {
precision = readVarLong();
compression = readByte() == 1;
}
ValueLob lob = ValueLob.open(dataType, handler, tableId, objectId, precision, compression);
if (smallLen == -3) {
ValueLob lob = ValueLob.open(type, handler, tableId, objectId, precision, compression);
if (smallLen == -2) {
lob.setFileName(readString(), false);
}
return lob;
}
case Value.ARRAY: {
int len = readInt();
int len = readVarInt();
Value[] list = new Value[len];
for (int i = 0; i < len; i++) {
list[i] = readValue();
......@@ -559,7 +654,162 @@ public class Data extends DataPage {
return ValueArray.get(list);
}
default:
throw Message.throwInternalError("type=" + dataType);
if (type >= INT_0_15 && type < INT_0_15 + 16) {
return ValueInt.get(type - INT_0_15);
} else if (type >= LONG_0_7 && type < LONG_0_7 + 8) {
return ValueLong.get(type - LONG_0_7);
} else if (type >= BYTES_0_31 && type < BYTES_0_31 + 32) {
int len = type - BYTES_0_31;
byte[] b = MemoryUtils.newBytes(len);
read(b, 0, len);
return ValueBytes.getNoCopy(b);
} else if (type >= STRING_0_31 && type < STRING_0_31 + 32) {
return ValueString.get(readString(type - STRING_0_31));
}
throw Message.throwInternalError("type=" + type);
}
}
/**
* Calculate the number of bytes required to encode the given value.
*
* @param v the value
* @return the number of bytes required to store this value
*/
public int getValueLen(Value v) throws SQLException {
if (v == ValueNull.INSTANCE) {
return 1;
}
switch (v.getType()) {
case Value.BOOLEAN:
return 1;
case Value.BYTE:
return 2;
case Value.SHORT:
return 3;
case Value.INT: {
int x = v.getInt();
if (x < 0) {
return 1 + getVarIntLen(-x);
} else if (x < 16) {
return 1;
} else {
return 1 + getVarIntLen(x);
}
}
case Value.LONG: {
long x = v.getLong();
if (x < 0) {
return 1 + getVarLongLen(-x);
} else if (x < 8) {
return 1;
} else {
return 1 + getVarLongLen(x);
}
}
case Value.DOUBLE: {
double x = v.getDouble();
if (x == 0.0 || x == 1.0) {
return 1;
}
return 1 + getVarLongLen(Long.reverse(Double.doubleToLongBits(x)));
}
case Value.FLOAT: {
float x = v.getFloat();
if (x == 0.0f || x == 1.0f) {
return 1;
}
return 1 + getVarIntLen(Integer.reverse(Float.floatToIntBits(v.getFloat())));
}
case Value.STRING: {
String s = v.getString();
int len = s.length();
if (len < 32) {
return 1 + getStringWithoutLengthLen(s, len);
}
return 1 + getStringLen(s);
}
case Value.STRING_IGNORECASE:
case Value.STRING_FIXED:
return 1 + getStringLen(v.getString());
case Value.DECIMAL: {
BigDecimal x = v.getBigDecimal();
if (BigDecimal.ZERO.equals(x)) {
return 1;
} else if (BigDecimal.ONE.equals(x)) {
return 1;
}
int scale = x.scale();
BigInteger b = x.unscaledValue();
int bits = b.bitLength();
if (bits <= 63) {
if (scale == 0) {
return 1 + getVarLongLen(b.longValue());
}
return 1 + getVarIntLen(scale) + getVarLongLen(b.longValue());
}
byte[] bytes = b.toByteArray();
return 1 + getVarIntLen(scale) + getVarIntLen(bytes.length) + bytes.length;
}
case Value.TIME:
return 1 + getVarLongLen(DateTimeUtils.getTimeLocal(v.getTimeNoCopy()));
case Value.DATE: {
long x = DateTimeUtils.getTimeLocal(v.getDateNoCopy());
return 1 + getVarLongLen(x / MILLIS_PER_MINUTE);
}
case Value.TIMESTAMP: {
Timestamp ts = v.getTimestampNoCopy();
return 1 + getVarLongLen(DateTimeUtils.getTimeLocal(ts)) + getVarIntLen(ts.getNanos());
}
case Value.JAVA_OBJECT: {
byte[] b = v.getBytesNoCopy();
return 1 + getVarIntLen(b.length) + b.length;
}
case Value.BYTES: {
byte[] b = v.getBytesNoCopy();
int len = b.length;
if (len < 32) {
return 1 + b.length;
}
return 1 + getVarIntLen(b.length) + b.length;
}
case Value.UUID:
return 1 + LENGTH_LONG + LENGTH_LONG;
case Value.BLOB:
case Value.CLOB: {
int len = 1;
ValueLob lob = (ValueLob) v;
lob.convertToFileIfRequired(handler);
byte[] small = lob.getSmall();
if (small == null) {
int t = -1;
if (!lob.isLinked()) {
t = -2;
}
len += getVarIntLen(t);
len += getVarIntLen(lob.getTableId());
len += getVarIntLen(lob.getObjectId());
len += getVarLongLen(lob.getPrecision());
len += 1;
if (t == -2) {
len += getStringLen(lob.getFileName());
}
} else {
len += getVarIntLen(small.length);
len += small.length;
}
return len;
}
case Value.ARRAY: {
Value[] list = ((ValueArray) v).getList();
int len = 1 + getVarIntLen(list.length);
for (Value x : list) {
len += getValueLen(x);
}
return len;
}
default:
throw Message.throwInternalError("type=" + v.getType());
}
}
......@@ -595,19 +845,6 @@ public class Data extends DataPage {
return ((buff[pos++] & 0xff) << 8) + (buff[pos++] & 0xff);
}
private static int getStringLenUTF8(String s) {
int plus = 4, len = s.length();
for (int i = 0; i < len; i++) {
char c = s.charAt(i);
if (c >= 0x800) {
plus += 2;
} else if (c == 0 || c >= 0x80) {
plus++;
}
}
return len + plus;
}
/**
* Shrink the array to this size.
*
......@@ -622,4 +859,82 @@ public class Data extends DataPage {
}
}
private int getVarIntLen(int x) {
if ((x & (-1 << 7)) == 0) {
return 1;
} else if ((x & (-1 << 14)) == 0) {
return 2;
} else if ((x & (-1 << 21)) == 0) {
return 3;
} else if ((x & (-1 << 28)) == 0) {
return 4;
}
return 5;
}
private void writeVarInt(int x) {
while ((x & ~0x7f) != 0) {
data[pos++] = (byte) (0x80 | (x & 0x7f));
x >>>= 7;
}
data[pos++] = (byte) x;
}
private int readVarInt() {
int b = data[pos++];
if (b >= 0) {
return b;
}
int x = b & 0x7f;
b = data[pos++];
if (b >= 0) {
return x | (b << 7);
}
x |= (b & 0x7f) << 7;
b = data[pos++];
if (b >= 0) {
return x | (b << 14);
}
x |= (b & 0x7f) << 14;
b = data[pos++];
if (b >= 0) {
return x | b << 21;
}
return x | ((b & 0x7f) << 21) | (data[pos++] << 28);
}
private int getVarLongLen(long x) {
int i = 1;
while (true) {
x >>>= 7;
if (x == 0) {
return i;
}
i++;
}
}
private void writeVarLong(long x) {
while ((x & ~0x7f) != 0) {
data[pos++] = (byte) ((x & 0x7f) | 0x80);
x >>>= 7;
}
data[pos++] = (byte) x;
}
private long readVarLong() {
long x = data[pos++];
if (x >= 0) {
return x;
}
x &= 0x7f;
for (int s = 7;; s += 7) {
long b = data[pos++];
x |= (b & 0x7f) << s;
if (b >= 0) {
return x;
}
}
}
}
......@@ -69,6 +69,22 @@ public class PageFreeList extends Record {
}
}
int getFirstFree() {
if (full) {
return -1;
}
int free = used.nextClearBit(0);
if (free >= pageCount) {
return -1;
}
return free;
}
int getLastUsed() {
int last = used.getLastSetBit();
return last == -1 ? -1 : last + getPos();
}
/**
* Mark a page as used.
*
......
......@@ -18,6 +18,7 @@ import org.h2.engine.Session;
import org.h2.index.Cursor;
import org.h2.index.Index;
import org.h2.index.IndexType;
import org.h2.index.Page;
import org.h2.index.PageBtreeIndex;
import org.h2.index.PageScanIndex;
import org.h2.log.InDoubtTransaction;
......@@ -71,17 +72,22 @@ import org.h2.value.ValueString;
*/
public class PageStore implements CacheWriter {
// TODO var int: see google protocol buffers
// TODO don't save parent (only root); remove setPageId
// TODO implement checksum - 0 for empty
// TODO b-tree index with fixed size values doesn't need offset and so on
// TODO shrinking: a way to load pages centrally
// TODO shrinking: Page.moveTo(int pageId).
// TODO utf-x: test if it's faster
// TODO value serialization: test (100% coverage)
// TODO scan index: support long keys, and use var long
// TODO don't save the direct parent (only root); remove setPageId
// TODO implement checksum; 0 for empty pages
// TODO remove parent, use tableId if required
// TODO replace CRC32
// TODO optimization: try to avoid allocating a byte array per page
// TODO optimization: check if calling Data.getValueLen slows things down
// TODO PageBtreeNode: 4 bytes offset - others use only 2
// TODO block compression: don't store the middle zeroes
// TODO block compression: maybe http://en.wikipedia.org/wiki/LZJB
// with RLE, specially for 0s.
// TODO undo pages: don't store the middle zeroes
// TODO undo pages compression: try http://en.wikipedia.org/wiki/LZJB
// TODO order pages so that searching for a key only seeks forward
// TODO completely re-use keys of deleted rows; maybe
// remember last page with deleted keys (in the root page?),
......@@ -341,13 +347,62 @@ public class PageStore implements CacheWriter {
writeCount++;
}
}
// TODO shrink file if required here
// int pageCount = getFreeList().getLastUsed() + 1;
// trace.debug("pageCount:" + pageCount);
// file.setLength((long) pageCount << pageSizeShift);
}
}
private void compact() throws SQLException {
trim();
int full = pageCount - 1;
int free = -1;
for (int i = 0; i < pageCount && free != -1; i++) {
free = getFreeList(i).getFirstFree();
}
if (free == -1 || free < 10) {
return;
}
Record rec = getRecord(full);
if (rec == null) {
} else {
Data page = Data.create(database, pageSize);
readPage(full, page);
int parent = page.readInt();
int type = page.readByte();
boolean last = (type & Page.FLAG_LAST) != 0;
type = type & ~Page.FLAG_LAST;
System.out.println("last page is " + type + " parent: " + parent);
switch (type) {
case Page.TYPE_EMPTY:
break;
case Page.TYPE_FREE_LIST:
break;
case Page.TYPE_DATA_LEAF:
break;
case Page.TYPE_DATA_NODE:
case Page.TYPE_DATA_OVERFLOW:
case Page.TYPE_BTREE_LEAF:
case Page.TYPE_BTREE_NODE:
case Page.TYPE_STREAM_TRUNK:
case Page.TYPE_STREAM_DATA:
}
}
}
/**
* Shrink the file so there are no empty pages at the end.
*/
public void trim() throws SQLException {
for (int i = getFreeListId(pageCount); i >= 0; i--) {
int last = getFreeList(i).getLastUsed();
if (last != -1) {
pageCount = last + 1;
break;
}
}
trace.debug("pageCount:" + pageCount);
file.setLength((long) pageCount << pageSizeShift);
}
private void switchLog() throws SQLException {
trace.debug("switchLog");
Session[] sessions = database.getSessions(true);
......@@ -540,8 +595,12 @@ public class PageStore implements CacheWriter {
}
}
private int getFreeListId(int pageId) {
return (pageId - PAGE_ID_FREE_LIST_ROOT) / freeListPagesPerList;
}
private PageFreeList getFreeListForPage(int pageId) throws SQLException {
return getFreeList((pageId - PAGE_ID_FREE_LIST_ROOT) / freeListPagesPerList);
return getFreeList(getFreeListId(pageId));
}
private PageFreeList getFreeList(int i) throws SQLException {
......
......@@ -720,6 +720,7 @@ public class Recover extends Tool implements DataHandler {
setDatabaseName(fileName.substring(0, fileName.length() - Constants.SUFFIX_PAGE_FILE.length()));
FileStore store = null;
PrintWriter writer = null;
int[] pageTypeCount = new int[Page.TYPE_STREAM_DATA + 2];
int emptyPages = 0;
try {
writer = getWriter(fileName, ".sql");
......@@ -789,6 +790,7 @@ public class Recover extends Tool implements DataHandler {
int type = s.readByte();
switch (type) {
case Page.TYPE_EMPTY:
pageTypeCount[type]++;
if (parentPageId != 0) {
writer.println("-- ERROR empty page with parent: " + parentPageId);
}
......@@ -798,29 +800,31 @@ public class Recover extends Tool implements DataHandler {
boolean last = (type & Page.FLAG_LAST) != 0;
type &= ~Page.FLAG_LAST;
switch (type) {
case Page.TYPE_DATA_OVERFLOW:
writer.println("-- page " + page + ": data overflow " + (last ? "(last)" : ""));
break;
case Page.TYPE_DATA_NODE: {
int entries = s.readShortInt();
int rowCount = s.readInt();
writer.println("-- page " + page + ": data node " + (last ? "(last)" : "") + " entries: " + entries + " rowCount: " + rowCount);
break;
}
// type 1
case Page.TYPE_DATA_LEAF: {
pageTypeCount[type]++;
setStorage(s.readInt());
int entries = s.readShortInt();
writer.println("-- page " + page + ": data leaf " + (last ? "(last)" : "") + " table: " + storageId + " entries: " + entries);
dumpPageDataLeaf(store, pageSize, writer, s, last, page, entries);
break;
}
case Page.TYPE_BTREE_NODE:
writer.println("-- page " + page + ": b-tree node" + (last ? "(last)" : ""));
if (trace) {
dumpPageBtreeNode(writer, s, !last);
// type 2
case Page.TYPE_DATA_NODE: {
pageTypeCount[type]++;
int entries = s.readShortInt();
int rowCount = s.readInt();
writer.println("-- page " + page + ": data node " + (last ? "(last)" : "") + " entries: " + entries + " rowCount: " + rowCount);
break;
}
// type 3
case Page.TYPE_DATA_OVERFLOW:
pageTypeCount[type]++;
writer.println("-- page " + page + ": data overflow " + (last ? "(last)" : ""));
break;
// type 4
case Page.TYPE_BTREE_LEAF: {
pageTypeCount[type]++;
setStorage(s.readInt());
int entries = s.readShortInt();
writer.println("-- page " + page + ": b-tree leaf " + (last ? "(last)" : "") + " table: " + storageId + " entries: " + entries);
......@@ -829,14 +833,28 @@ public class Recover extends Tool implements DataHandler {
}
break;
}
// type 5
case Page.TYPE_BTREE_NODE:
pageTypeCount[type]++;
writer.println("-- page " + page + ": b-tree node" + (last ? "(last)" : ""));
if (trace) {
dumpPageBtreeNode(writer, s, !last);
}
break;
// type 6
case Page.TYPE_FREE_LIST:
pageTypeCount[type]++;
writer.println("-- page " + page + ": free list " + (last ? "(last)" : ""));
free += dumpPageFreeList(writer, s, pageSize, page, pageCount);
break;
// type 7
case Page.TYPE_STREAM_TRUNK:
pageTypeCount[type]++;
writer.println("-- page " + page + ": log trunk");
break;
// type 8
case Page.TYPE_STREAM_DATA:
pageTypeCount[type]++;
writer.println("-- page " + page + ": log data");
break;
default:
......@@ -844,13 +862,19 @@ public class Recover extends Tool implements DataHandler {
break;
}
}
writer.println("-- page count: " + pageCount + " empty: " + emptyPages + " free: " + free);
writeSchema(writer);
try {
dumpPageLogStream(writer, store, logFirstTrunkPage, logFirstDataPage, pageSize);
} catch (EOFException e) {
// ignore
}
writer.println("-- page count: " + pageCount + " empty: " + emptyPages + " free: " + free);
for (int i = 0; i < pageTypeCount.length; i++) {
int count = pageTypeCount[i];
if (count > 0) {
writer.println("-- page count type: " + i + " " + (100 * count / pageCount) + "% count: " + count);
}
}
writer.close();
} catch (Throwable e) {
writeError(writer, e);
......
......@@ -398,4 +398,34 @@ public class DateTimeUtils {
return value;
}
/**
* Get the number of milliseconds since 1970-01-01 in the local timezone.
*
* @param d the date
* @return the milliseconds
*/
public static long getTimeLocal(java.util.Date d) {
Calendar c = getCalendar();
synchronized (c) {
c.setTime(d);
return c.getTime().getTime() + c.get(Calendar.ZONE_OFFSET);
}
}
/**
* Convert the number of milliseconds since 1970-01-01 in the local timezone
* to GMT.
*
* @param millis the number of milliseconds in the local timezone
* @return the number of milliseconds in GMT
*/
public static long getTimeGMT(long millis) {
Date d = new Date(millis);
Calendar c = getCalendar();
synchronized (c) {
c.setTime(d);
return c.getTime().getTime() - c.get(Calendar.ZONE_OFFSET);
}
}
}
......@@ -20,6 +20,16 @@ import org.h2.util.MathUtils;
*/
public class ValueDecimal extends Value {
/**
* The value 'zero'.
*/
public static final Object ZERO = new ValueDecimal(BigDecimal.ZERO);
/**
* The value 'one'.
*/
public static final Object ONE = new ValueDecimal(BigDecimal.ONE);
/**
* The default precision for a decimal value.
*/
......@@ -36,10 +46,6 @@ public class ValueDecimal extends Value {
static final int DEFAULT_DISPLAY_SIZE = 65535;
private static final int DIVIDE_SCALE_ADD = 25;
private static final BigDecimal DEC_ZERO = new BigDecimal("0");
private static final BigDecimal DEC_ONE = new BigDecimal("1");
private static final Object ZERO = new ValueDecimal(DEC_ZERO);
private static final Object ONE = new ValueDecimal(DEC_ONE);
private final BigDecimal value;
private String valueString;
......@@ -82,7 +88,7 @@ public class ValueDecimal extends Value {
}
BigDecimal bd = value.divide(dec.value, value.scale() + DIVIDE_SCALE_ADD, BigDecimal.ROUND_HALF_DOWN);
if (bd.signum() == 0) {
bd = DEC_ZERO;
bd = BigDecimal.ZERO;
} else if (bd.scale() > 0) {
if (!bd.unscaledValue().testBit(0)) {
String s = bd.toString();
......@@ -185,9 +191,9 @@ public class ValueDecimal extends Value {
* @return the value
*/
public static ValueDecimal get(BigDecimal dec) {
if (DEC_ZERO.equals(dec)) {
if (BigDecimal.ZERO.equals(dec)) {
return (ValueDecimal) ZERO;
} else if (DEC_ONE.equals(dec)) {
} else if (BigDecimal.ONE.equals(dec)) {
return (ValueDecimal) ONE;
}
return (ValueDecimal) Value.cache(new ValueDecimal(dec));
......
......@@ -349,6 +349,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
test.pageStore = true;
test.runTests();
TestPerformance.main("-init", "-db", "1");
// Recover.execute("data", null);
System.setProperty(SysProperties.H2_PAGE_STORE, "false");
test.pageStore = false;
......
......@@ -6,10 +6,15 @@
*/
package org.h2.test.unit;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import org.h2.constant.SysProperties;
import org.h2.message.Trace;
import org.h2.store.Data;
import org.h2.store.DataHandler;
import org.h2.store.DataPage;
import org.h2.store.FileStore;
......@@ -17,11 +22,25 @@ import org.h2.test.TestBase;
import org.h2.util.SmallLRUCache;
import org.h2.util.TempFileDeleter;
import org.h2.value.Value;
import org.h2.value.ValueArray;
import org.h2.value.ValueBoolean;
import org.h2.value.ValueByte;
import org.h2.value.ValueBytes;
import org.h2.value.ValueDate;
import org.h2.value.ValueDecimal;
import org.h2.value.ValueDouble;
import org.h2.value.ValueFloat;
import org.h2.value.ValueInt;
import org.h2.value.ValueJavaObject;
import org.h2.value.ValueLong;
import org.h2.value.ValueNull;
import org.h2.value.ValueShort;
import org.h2.value.ValueString;
import org.h2.value.ValueStringFixed;
import org.h2.value.ValueStringIgnoreCase;
import org.h2.value.ValueTime;
import org.h2.value.ValueTimestamp;
import org.h2.value.ValueUuid;
/**
* Data page tests.
......@@ -38,9 +57,104 @@ public class TestDataPage extends TestBase implements DataHandler {
}
public void test() throws SQLException {
testValues();
testAll();
}
private void testValues() throws SQLException {
testValue(ValueNull.INSTANCE);
testValue(ValueBoolean.get(false));
testValue(ValueBoolean.get(true));
for (int i = 0; i < 256; i++) {
testValue(ValueByte.get((byte) i));
}
for (int i = 0; i < 256 * 256; i += 10) {
testValue(ValueShort.get((short) i));
}
for (int i = 0; i < 256 * 256; i += 10) {
testValue(ValueInt.get(i));
testValue(ValueInt.get(-i));
testValue(ValueLong.get(i));
testValue(ValueLong.get(-i));
}
testValue(ValueInt.get(Integer.MAX_VALUE));
testValue(ValueInt.get(Integer.MIN_VALUE));
for (long i = 0; i < Integer.MAX_VALUE; i += 10 + i / 4) {
testValue(ValueInt.get((int) i));
testValue(ValueInt.get((int) -i));
}
testValue(ValueLong.get(Long.MAX_VALUE));
testValue(ValueLong.get(Long.MIN_VALUE));
for (long i = 0; i >= 0; i += 10 + i / 4) {
testValue(ValueLong.get(i));
testValue(ValueLong.get(-i));
}
testValue(ValueDecimal.get(BigDecimal.ZERO));
testValue(ValueDecimal.get(BigDecimal.ONE));
testValue(ValueDecimal.get(BigDecimal.TEN));
testValue(ValueDecimal.get(BigDecimal.ONE.negate()));
testValue(ValueDecimal.get(BigDecimal.TEN.negate()));
for (long i = 0; i >= 0; i += 10 + i / 4) {
testValue(ValueDecimal.get(new BigDecimal(i)));
testValue(ValueDecimal.get(new BigDecimal(-i)));
for (int j = 0; j < 200; j += 50) {
testValue(ValueDecimal.get(new BigDecimal(i).setScale(j)));
testValue(ValueDecimal.get(new BigDecimal(i * i).setScale(j)));
}
testValue(ValueDecimal.get(new BigDecimal(i * i)));
}
testValue(ValueDate.get(new Date(System.currentTimeMillis())));
testValue(ValueDate.get(new Date(0)));
testValue(ValueTime.get(new Time(System.currentTimeMillis())));
testValue(ValueTime.get(new Time(0)));
testValue(ValueTimestamp.get(new Timestamp(System.currentTimeMillis())));
testValue(ValueTimestamp.get(new Timestamp(0)));
testValue(ValueJavaObject.getNoCopy(new byte[0]));
testValue(ValueJavaObject.getNoCopy(new byte[100]));
for (int i = 0; i < 300; i++) {
testValue(ValueBytes.getNoCopy(new byte[i]));
}
for (int i = 0; i < 65000; i += 10 + i) {
testValue(ValueBytes.getNoCopy(new byte[i]));
}
testValue(ValueUuid.getNewRandom());
for (int i = 0; i < 100; i++) {
testValue(ValueString.get(new String(new char[i])));
}
for (int i = 0; i < 65000; i += 10 + i) {
testValue(ValueString.get(new String(new char[i])));
testValue(ValueStringFixed.get(new String(new char[i])));
testValue(ValueStringIgnoreCase.get(new String(new char[i])));
}
testValue(ValueFloat.get(0f));
testValue(ValueFloat.get(1f));
testValue(ValueFloat.get(-1f));
testValue(ValueDouble.get(0));
testValue(ValueDouble.get(1));
testValue(ValueDouble.get(-1));
for (int i = 0; i < 65000; i += 10 + i) {
for (double j = 0.1; j < 65000; j += 10 + j) {
testValue(ValueFloat.get((float) (i / j)));
testValue(ValueDouble.get(i / j));
testValue(ValueFloat.get((float) -(i / j)));
testValue(ValueDouble.get(-(i / j)));
}
}
testValue(ValueArray.get(new Value[0]));
testValue(ValueArray.get(new Value[] {ValueBoolean.get(true), ValueInt.get(10)}));
}
private void testValue(Value v) throws SQLException {
Data data = Data.create(null, 1024);
data.checkCapacity((int) v.getPrecision());
data.writeValue(v);
data.reset();
Value v2 = data.readValue();
assertEquals(v.getType(), v2.getType());
assertTrue(v.compareEqual(v2));
}
private void testAll() throws SQLException {
DataPage page = DataPage.create(this, 128);
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论