Skip to content

H
h2database
提交 796a5529 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

New lob storage.

上级 6b1bac51
隐藏空白字符变更
内嵌 并排
正在显示 包含 102 行增加 和 62 行删除
h2/src/docsrc/html/changelog.html
浏览文件 @ 796a5529
......@@ -18,7 +18,8 @@ Change Log
<h1>Change Log</h1>
<h2>Next Version (unreleased)</h2>
<ul><li>-
<ul><li>The experimental LOB storage mechanism now supports all features of the
old one. To use it, set the system property "h2.lobInDatabase" to "true".
</li></ul>
<h2>Version 1.2.133 (2010-04-10)</h2>
......
h2/src/main/org/h2/constant/SysProperties.java
浏览文件 @ 796a5529
......@@ -175,6 +175,14 @@ public class SysProperties {
*/
public static final int DEFAULT_MAX_LENGTH_INPLACE_LOB = getIntSetting("h2.defaultMaxLengthInplaceLob", 4096);
/**
* System property <code>h2.defaultMaxLengthInplaceLob2</code>
* (default: 128).<br />
* The default maximum length of an LOB that is stored with the record itself.
* Only used if h2.lobInDatabase is enabled.
*/
public static final int DEFAULT_MAX_LENGTH_INPLACE_LOB2 = getIntSetting("h2.defaultMaxLengthInplaceLob2", 128);
/**
* System property <code>h2.defaultResultSetConcurrency</code> (default:
* ResultSet.CONCUR_READ_ONLY).<br />
......
h2/src/main/org/h2/engine/Database.java
浏览文件 @ 796a5529
......@@ -129,7 +129,7 @@ public class Database implements DataHandler {
private int maxMemoryRows = Constants.DEFAULT_MAX_MEMORY_ROWS;
private int maxMemoryUndo = SysProperties.DEFAULT_MAX_MEMORY_UNDO;
private int lockMode = SysProperties.DEFAULT_LOCK_MODE;
private int maxLengthInplaceLob = SysProperties.DEFAULT_MAX_LENGTH_INPLACE_LOB;
private int maxLengthInplaceLob;
private int allowLiterals = Constants.ALLOW_LITERALS_ALL;
private int powerOffCount = initialPowerOffCount;
......@@ -173,6 +173,8 @@ public class Database implements DataHandler {
this.filePasswordHash = ci.getFilePasswordHash();
this.databaseName = name;
this.databaseShortName = parseDatabaseShortName();
this.maxLengthInplaceLob = SysProperties.LOB_IN_DATABASE ?
SysProperties.DEFAULT_MAX_LENGTH_INPLACE_LOB2 : SysProperties.DEFAULT_MAX_LENGTH_INPLACE_LOB;
this.cipher = cipher;
String lockMethodName = ci.getProperty("FILE_LOCK", null);
this.accessModeData = ci.getProperty("ACCESS_MODE_DATA", "rw").toLowerCase();
......
h2/src/main/org/h2/store/LobStorage.java
浏览文件 @ 796a5529
......@@ -16,10 +16,12 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Constants;
import org.h2.message.DbException;
import org.h2.tools.CompressTool;
import org.h2.util.IOUtils;
import org.h2.util.New;
import org.h2.util.StringUtils;
......@@ -47,8 +49,9 @@ public class LobStorage {
private static final long UNIQUE = 0xffff;
private Connection conn;
private HashMap<String, PreparedStatement> prepared = New.hashMap();
private long nextLob;
private AtomicLong nextLob = new AtomicLong();
private long nextBlock;
private CompressTool compress = CompressTool.getInstance();
private final DataHandler handler;
private boolean init;
......@@ -76,17 +79,17 @@ public class LobStorage {
stat.execute("CREATE TABLE IF NOT EXISTS " + LOBS + "(ID BIGINT PRIMARY KEY, LENGTH BIGINT, TABLE INT) HIDDEN");
stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_MAP + "(LOB BIGINT, SEQ INT, BLOCK BIGINT, PRIMARY KEY(LOB, SEQ)) HIDDEN");
stat.execute("CREATE INDEX IF NOT EXISTS INFORMATION_SCHEMA.INDEX_LOB_MAP_DATA_LOB ON " + LOB_MAP + "(BLOCK, LOB)");
stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, DATA BINARY) HIDDEN");
stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, COMPRESSED INT, DATA BINARY) HIDDEN");
ResultSet rs;
rs = stat.executeQuery("SELECT MAX(BLOCK) FROM " + LOB_DATA);
rs.next();
nextBlock = rs.getLong(1) + 1;
if (HASH) {
nextBlock = Math.max(UNIQUE + 1, nextLob);
nextBlock = Math.max(UNIQUE + 1, nextLob.get());
}
rs = stat.executeQuery("SELECT MAX(ID) FROM " + LOBS);
rs.next();
nextLob = rs.getLong(1) + 1;
nextLob.set(rs.getLong(1) + 1);
} catch (SQLException e) {
throw DbException.convert(e);
}
......@@ -125,7 +128,13 @@ public class LobStorage {
*/
public static Value createSmallLob(int type, byte[] small) {
if (SysProperties.LOB_IN_DATABASE) {
return ValueLob2.createSmallLob(type, small);
int precision;
if (type == Value.CLOB) {
precision = StringUtils.utf8Decode(small).length();
} else {
precision = small.length;
}
return ValueLob2.createSmallLob(type, small, precision);
}
return ValueLob.createSmallLob(type, small);
}
......@@ -142,6 +151,7 @@ public class LobStorage {
private long remaining;
private long lob;
private int seq;
private CompressTool compress;
public LobInputStream(Connection conn, long lob) throws IOException {
this.conn = conn;
......@@ -152,7 +162,7 @@ public class LobStorage {
prep.setLong(1, lob);
ResultSet rs = prep.executeQuery();
if (!rs.next()) {
throw DbException.get(ErrorCode.IO_EXCEPTION_1, "lob: "+ lob + " seq: " + seq).getSQLException();
throw DbException.get(ErrorCode.IO_EXCEPTION_1, "Missing lob: "+ lob).getSQLException();
}
remaining = rs.getLong(1);
rs.close();
......@@ -210,7 +220,7 @@ public class LobStorage {
try {
if (prepSelect == null) {
prepSelect = conn.prepareStatement(
"SELECT DATA FROM " + LOB_MAP + " M " +
"SELECT COMPRESSED, DATA FROM " + LOB_MAP + " M " +
"INNER JOIN " + LOB_DATA + " D ON M.BLOCK = D.BLOCK " +
"WHERE M.LOB = ? AND M.SEQ = ?");
}
......@@ -218,10 +228,17 @@ public class LobStorage {
prepSelect.setInt(2, seq);
ResultSet rs = prepSelect.executeQuery();
if (!rs.next()) {
throw DbException.get(ErrorCode.IO_EXCEPTION_1, "lob: "+ lob + " seq: " + seq).getSQLException();
throw DbException.get(ErrorCode.IO_EXCEPTION_1, "Missing lob entry: "+ lob + "/" + seq).getSQLException();
}
seq++;
buffer = rs.getBytes(1);
int compressed = rs.getInt(1);
buffer = rs.getBytes(2);
if (compressed != 0) {
if (compress == null) {
compress = CompressTool.getInstance();
}
buffer = compress.expand(buffer);
}
pos = 0;
} catch (SQLException e) {
throw DbException.convertToIOException(e);
......@@ -273,14 +290,15 @@ public class LobStorage {
}
private ValueLob2 addLob(InputStream in, long maxLength, int type) {
int todo;
// TODO support in-place lobs, and group lobs much smaller than the page size
byte[] buff = new byte[BLOCK_LENGTH];
if (maxLength < 0) {
maxLength = Long.MAX_VALUE;
}
long length = 0;
long lobId = nextLob++;
long lobId;
lobId = nextLob.getAndIncrement();
int maxLengthInPlaceLob = handler.getMaxLengthInplaceLob();
String compressAlgorithm = handler.getLobCompressionAlgorithm(type);
try {
try {
for (int seq = 0; maxLength > 0; seq++) {
......@@ -298,40 +316,12 @@ public class LobStorage {
} else {
b = buff;
}
long block;
boolean blockExists = false;
if (HASH) {
block = Arrays.hashCode(b) & UNIQUE;
int todoSynchronize;
PreparedStatement prep = prepare(
"SELECT DATA FROM " + LOB_DATA +
" WHERE BLOCK = ?");
prep.setLong(1, block);
ResultSet rs = prep.executeQuery();
if (rs.next()) {
byte[] compare = rs.getBytes(1);
if (Arrays.equals(b, compare)) {
blockExists = true;
} else {
block = nextBlock++;
}
}
} else {
block = nextBlock++;
}
if (!blockExists) {
PreparedStatement prep = prepare(
"INSERT INTO " + LOB_DATA + "(BLOCK, DATA) VALUES(?, ?)");
prep.setLong(1, block);
prep.setBytes(2, b);
prep.execute();
if (seq == 0 && b.length < BLOCK_LENGTH && b.length <= maxLengthInPlaceLob) {
// CLOB: the precision will be fixed later
ValueLob2 v = ValueLob2.createSmallLob(type, b, b.length);
return v;
}
PreparedStatement prep = prepare(
"INSERT INTO " + LOB_MAP + "(LOB, SEQ, BLOCK) VALUES(?, ?, ?)");
prep.setLong(1, lobId);
prep.setInt(2, seq);
prep.setLong(3, block);
prep.execute();
storeBlock(lobId, seq, b, compressAlgorithm);
}
PreparedStatement prep = prepare(
"INSERT INTO " + LOBS + "(ID, LENGTH, TABLE) VALUES(?, ?, ?)");
......@@ -350,6 +340,47 @@ public class LobStorage {
}
}
synchronized void storeBlock(long lobId, int seq, byte[] b, String compressAlgorithm) throws SQLException {
long block;
boolean blockExists = false;
if (compressAlgorithm != null) {
b = compress.compress(b, compressAlgorithm);
}
if (HASH) {
block = Arrays.hashCode(b) & UNIQUE;
PreparedStatement prep = prepare(
"SELECT COMPRESSED, DATA FROM " + LOB_DATA +
" WHERE BLOCK = ?");
prep.setLong(1, block);
ResultSet rs = prep.executeQuery();
if (rs.next()) {
boolean compressed = rs.getInt(1) != 0;
byte[] compare = rs.getBytes(2);
if (Arrays.equals(b, compare) && compressed == (compressAlgorithm != null)) {
blockExists = true;
} else {
block = nextBlock++;
}
}
} else {
block = nextBlock++;
}
if (!blockExists) {
PreparedStatement prep = prepare(
"INSERT INTO " + LOB_DATA + "(BLOCK, COMPRESSED, DATA) VALUES(?, ?, ?)");
prep.setLong(1, block);
prep.setInt(2, compressAlgorithm == null ? 0 : 1);
prep.setBytes(3, b);
prep.execute();
}
PreparedStatement prep = prepare(
"INSERT INTO " + LOB_MAP + "(LOB, SEQ, BLOCK) VALUES(?, ?, ?)");
prep.setLong(1, lobId);
prep.setInt(2, seq);
prep.setLong(3, block);
prep.execute();
}
/**
* An input stream that reads the data from a reader.
*/
......
h2/src/main/org/h2/value/ValueLob2.java
浏览文件 @ 796a5529
......@@ -54,16 +54,10 @@ public class ValueLob2 extends Value {
this.precision = precision;
}
private ValueLob2(int type, byte[] small) {
private ValueLob2(int type, byte[] small, long precision) {
this.type = type;
this.small = small;
if (small != null) {
if (type == Value.BLOB) {
this.precision = small.length;
} else {
this.precision = getString().length();
}
}
this.precision = precision;
}
/**
......@@ -87,8 +81,8 @@ public class ValueLob2 extends Value {
* @param small the byte array
* @return the lob value
*/
public static ValueLob2 createSmallLob(int type, byte[] small) {
return new ValueLob2(type, small);
public static ValueLob2 createSmallLob(int type, byte[] small, long precision) {
return new ValueLob2(type, small, precision);
}
/**
......@@ -384,9 +378,9 @@ public class ValueLob2 extends Value {
}
if (len <= handler.getMaxLengthInplaceLob()) {
byte[] small = StringUtils.utf8Encode(new String(buff, 0, len));
return ValueLob2.createSmallLob(Value.CLOB, small);
return ValueLob2.createSmallLob(Value.CLOB, small, len);
}
ValueLob2 lob = new ValueLob2(Value.CLOB, null);
ValueLob2 lob = new ValueLob2(Value.CLOB, null, 0);
lob.createTempFromReader(buff, len, in, remaining, handler);
return lob;
} catch (IOException e) {
......@@ -421,9 +415,9 @@ public class ValueLob2 extends Value {
if (len <= handler.getMaxLengthInplaceLob()) {
byte[] small = Utils.newBytes(len);
System.arraycopy(buff, 0, small, 0, len);
return ValueLob2.createSmallLob(Value.BLOB, small);
return ValueLob2.createSmallLob(Value.BLOB, small, small.length);
}
ValueLob2 lob = new ValueLob2(Value.BLOB, null);
ValueLob2 lob = new ValueLob2(Value.BLOB, null, 0);
lob.createTempFromStream(buff, len, in, remaining, handler);
return lob;
} catch (IOException e) {
......
h2/src/test/org/h2/test/TestAll.java
浏览文件 @ 796a5529
......@@ -292,11 +292,15 @@ java org.h2.test.TestAll timer
int testing;
// System.setProperty("h2.lobInDatabase", "true");
System.setProperty("h2.lobInDatabase", "true");
/*
new lob storage: test compression
new lob storage: test in-place storage (old and new)
power failure test
power failure test: MULTI_THREADED=TRUE
power failure test: larger binaries and additional index.
power failure test with randomly generating / dropping indexes and tables.
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论