New lob storage.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,8 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>-
+<ul><li>The experimental LOB storage mechanism now supports all features of the
+    old one. To use it, set the system property "h2.lobInDatabase" to "true".
 </li></ul>
 
 <h2>Version 1.2.133 (2010-04-10)</h2>

h2/src/main/org/h2/constant/SysProperties.java

@@ -175,6 +175,14 @@ public class SysProperties {
      */
     public static final int DEFAULT_MAX_LENGTH_INPLACE_LOB = getIntSetting("h2.defaultMaxLengthInplaceLob", 4096);
 
+    /**
+     * System property <code>h2.defaultMaxLengthInplaceLob2</code>
+     * (default: 128).<br />
+     * The default maximum length of an LOB that is stored with the record itself.
+     * Only used if h2.lobInDatabase is enabled.
+     */
+    public static final int DEFAULT_MAX_LENGTH_INPLACE_LOB2 = getIntSetting("h2.defaultMaxLengthInplaceLob2", 128);
+
     /**
      * System property <code>h2.defaultResultSetConcurrency</code> (default:
      * ResultSet.CONCUR_READ_ONLY).<br />

h2/src/main/org/h2/engine/Database.java

@@ -129,7 +129,7 @@ public class Database implements DataHandler {
     private int maxMemoryRows = Constants.DEFAULT_MAX_MEMORY_ROWS;
     private int maxMemoryUndo = SysProperties.DEFAULT_MAX_MEMORY_UNDO;
     private int lockMode = SysProperties.DEFAULT_LOCK_MODE;
-    private int maxLengthInplaceLob = SysProperties.DEFAULT_MAX_LENGTH_INPLACE_LOB;
+    private int maxLengthInplaceLob;
     private int allowLiterals = Constants.ALLOW_LITERALS_ALL;
 
     private int powerOffCount = initialPowerOffCount;
@@ -173,6 +173,8 @@ public class Database implements DataHandler {
         this.filePasswordHash = ci.getFilePasswordHash();
         this.databaseName = name;
         this.databaseShortName = parseDatabaseShortName();
+        this.maxLengthInplaceLob = SysProperties.LOB_IN_DATABASE ?
+                SysProperties.DEFAULT_MAX_LENGTH_INPLACE_LOB2 : SysProperties.DEFAULT_MAX_LENGTH_INPLACE_LOB;
         this.cipher = cipher;
         String lockMethodName = ci.getProperty("FILE_LOCK", null);
         this.accessModeData = ci.getProperty("ACCESS_MODE_DATA", "rw").toLowerCase();

h2/src/main/org/h2/store/LobStorage.java

@@ -16,10 +16,12 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicLong;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.message.DbException;
+import org.h2.tools.CompressTool;
 import org.h2.util.IOUtils;
 import org.h2.util.New;
 import org.h2.util.StringUtils;
@@ -47,8 +49,9 @@ public class LobStorage {
     private static final long UNIQUE = 0xffff;
     private Connection conn;
     private HashMap<String, PreparedStatement> prepared = New.hashMap();
-    private long nextLob;
+    private AtomicLong nextLob = new AtomicLong();
     private long nextBlock;
+    private CompressTool compress = CompressTool.getInstance();
 
     private final DataHandler handler;
     private boolean init;
@@ -76,17 +79,17 @@ public class LobStorage {
             stat.execute("CREATE TABLE IF NOT EXISTS " + LOBS + "(ID BIGINT PRIMARY KEY, LENGTH BIGINT, TABLE INT) HIDDEN");
             stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_MAP + "(LOB BIGINT, SEQ INT, BLOCK BIGINT, PRIMARY KEY(LOB, SEQ)) HIDDEN");
             stat.execute("CREATE INDEX IF NOT EXISTS INFORMATION_SCHEMA.INDEX_LOB_MAP_DATA_LOB ON " + LOB_MAP + "(BLOCK, LOB)");
-            stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, DATA BINARY) HIDDEN");
+            stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, COMPRESSED INT, DATA BINARY) HIDDEN");
             ResultSet rs;
             rs = stat.executeQuery("SELECT MAX(BLOCK) FROM " + LOB_DATA);
             rs.next();
             nextBlock = rs.getLong(1) + 1;
             if (HASH) {
-                nextBlock = Math.max(UNIQUE + 1, nextLob);
+                nextBlock = Math.max(UNIQUE + 1, nextLob.get());
             }
             rs = stat.executeQuery("SELECT MAX(ID) FROM " + LOBS);
             rs.next();
-            nextLob = rs.getLong(1) + 1;
+            nextLob.set(rs.getLong(1) + 1);
         } catch (SQLException e) {
             throw DbException.convert(e);
         }
@@ -125,7 +128,13 @@ public class LobStorage {
      */
     public static Value createSmallLob(int type, byte[] small) {
         if (SysProperties.LOB_IN_DATABASE) {
-            return ValueLob2.createSmallLob(type, small);
+            int precision;
+            if (type == Value.CLOB) {
+                precision = StringUtils.utf8Decode(small).length();
+            } else {
+                precision = small.length;
+            }
+            return ValueLob2.createSmallLob(type, small, precision);
         }
         return ValueLob.createSmallLob(type, small);
     }
@@ -142,6 +151,7 @@ public class LobStorage {
         private long remaining;
         private long lob;
         private int seq;
+        private CompressTool compress;
 
         public LobInputStream(Connection conn, long lob) throws IOException {
             this.conn = conn;
@@ -152,7 +162,7 @@ public class LobStorage {
                 prep.setLong(1, lob);
                 ResultSet rs = prep.executeQuery();
                 if (!rs.next()) {
-                    throw DbException.get(ErrorCode.IO_EXCEPTION_1, "lob: "+ lob + " seq: " + seq).getSQLException();
+                    throw DbException.get(ErrorCode.IO_EXCEPTION_1, "Missing lob: "+ lob).getSQLException();
                 }
                 remaining = rs.getLong(1);
                 rs.close();
@@ -210,7 +220,7 @@ public class LobStorage {
             try {
                 if (prepSelect == null) {
                     prepSelect = conn.prepareStatement(
-                        "SELECT DATA FROM " + LOB_MAP + " M " +
+                        "SELECT COMPRESSED, DATA FROM " + LOB_MAP + " M " +
                         "INNER JOIN " + LOB_DATA + " D ON M.BLOCK = D.BLOCK " +
                         "WHERE M.LOB = ? AND M.SEQ = ?");
                 }
@@ -218,10 +228,17 @@ public class LobStorage {
                 prepSelect.setInt(2, seq);
                 ResultSet rs = prepSelect.executeQuery();
                 if (!rs.next()) {
-                    throw DbException.get(ErrorCode.IO_EXCEPTION_1, "lob: "+ lob + " seq: " + seq).getSQLException();
+                    throw DbException.get(ErrorCode.IO_EXCEPTION_1, "Missing lob entry: "+ lob + "/" + seq).getSQLException();
                 }
                 seq++;
-                buffer = rs.getBytes(1);
+                int compressed = rs.getInt(1);
+                buffer = rs.getBytes(2);
+                if (compressed != 0) {
+                    if (compress == null) {
+                        compress = CompressTool.getInstance();
+                    }
+                    buffer = compress.expand(buffer);
+                }
                 pos = 0;
             } catch (SQLException e) {
                 throw DbException.convertToIOException(e);
@@ -273,14 +290,15 @@ public class LobStorage {
     }
 
     private ValueLob2 addLob(InputStream in, long maxLength, int type) {
-        int todo;
-        // TODO support in-place lobs, and group lobs much smaller than the page size
         byte[] buff = new byte[BLOCK_LENGTH];
         if (maxLength < 0) {
             maxLength = Long.MAX_VALUE;
         }
         long length = 0;
-        long lobId = nextLob++;
+        long lobId;
+        lobId = nextLob.getAndIncrement();
+        int maxLengthInPlaceLob = handler.getMaxLengthInplaceLob();
+        String compressAlgorithm = handler.getLobCompressionAlgorithm(type);
         try {
             try {
                 for (int seq = 0; maxLength > 0; seq++) {
@@ -298,19 +316,47 @@ public class LobStorage {
                     } else {
                         b = buff;
                     }
+                    if (seq == 0 && b.length < BLOCK_LENGTH && b.length <= maxLengthInPlaceLob) {
+                        // CLOB: the precision will be fixed later
+                        ValueLob2 v = ValueLob2.createSmallLob(type, b, b.length);
+                        return v;
+                    }
+                    storeBlock(lobId, seq, b, compressAlgorithm);
+                }
+                PreparedStatement prep = prepare(
+                        "INSERT INTO " + LOBS + "(ID, LENGTH, TABLE) VALUES(?, ?, ?)");
+                prep.setLong(1, lobId);
+                prep.setLong(2, length);
+                prep.setInt(3, TABLE_ID_SESSION_VARIABLE);
+                prep.execute();
+                ValueLob2 v = ValueLob2.create(type, this, null, lobId, length);
+                return v;
+            } catch (IOException e) {
+                deleteLob(lobId);
+                throw DbException.convertIOException(e, "adding blob");
+            }
+        } catch (SQLException e) {
+            throw DbException.convert(e);
+        }
+    }
+
+    synchronized void storeBlock(long lobId, int seq, byte[] b, String compressAlgorithm) throws SQLException {
         long block;
         boolean blockExists = false;
+        if (compressAlgorithm != null) {
+            b = compress.compress(b, compressAlgorithm);
+        }
         if (HASH) {
             block = Arrays.hashCode(b) & UNIQUE;
-                        int todoSynchronize;
             PreparedStatement prep = prepare(
-                                "SELECT DATA FROM " + LOB_DATA +
+                    "SELECT COMPRESSED, DATA FROM " + LOB_DATA +
                     " WHERE BLOCK = ?");
             prep.setLong(1, block);
             ResultSet rs = prep.executeQuery();
             if (rs.next()) {
-                            byte[] compare = rs.getBytes(1);
-                            if (Arrays.equals(b, compare)) {
+                boolean compressed = rs.getInt(1) != 0;
+                byte[] compare = rs.getBytes(2);
+                if (Arrays.equals(b, compare) && compressed == (compressAlgorithm != null)) {
                     blockExists = true;
                 } else {
                     block = nextBlock++;
@@ -321,9 +367,10 @@ public class LobStorage {
         }
         if (!blockExists) {
             PreparedStatement prep = prepare(
-                                "INSERT INTO " + LOB_DATA + "(BLOCK, DATA) VALUES(?, ?)");
+                    "INSERT INTO " + LOB_DATA + "(BLOCK, COMPRESSED, DATA) VALUES(?, ?, ?)");
             prep.setLong(1, block);
-                        prep.setBytes(2, b);
+            prep.setInt(2, compressAlgorithm == null ? 0 : 1);
+            prep.setBytes(3, b);
             prep.execute();
         }
         PreparedStatement prep = prepare(
@@ -333,22 +380,6 @@ public class LobStorage {
         prep.setLong(3, block);
         prep.execute();
     }
-                PreparedStatement prep = prepare(
-                        "INSERT INTO " + LOBS + "(ID, LENGTH, TABLE) VALUES(?, ?, ?)");
-                prep.setLong(1, lobId);
-                prep.setLong(2, length);
-                prep.setInt(3, TABLE_ID_SESSION_VARIABLE);
-                prep.execute();
-                ValueLob2 v = ValueLob2.create(type, this, null, lobId, length);
-                return v;
-            } catch (IOException e) {
-                deleteLob(lobId);
-                throw DbException.convertIOException(e, "adding blob");
-            }
-        } catch (SQLException e) {
-            throw DbException.convert(e);
-        }
-    }
 
     /**
      * An input stream that reads the data from a reader.

h2/src/main/org/h2/value/ValueLob2.java

@@ -54,16 +54,10 @@ public class ValueLob2 extends Value {
         this.precision = precision;
     }
 
-    private ValueLob2(int type, byte[] small) {
+    private ValueLob2(int type, byte[] small, long precision) {
         this.type = type;
         this.small = small;
-        if (small != null) {
-            if (type == Value.BLOB) {
-                this.precision = small.length;
-            } else {
-                this.precision = getString().length();
-            }
-        }
+        this.precision = precision;
     }
 
     /**
@@ -87,8 +81,8 @@ public class ValueLob2 extends Value {
      * @param small the byte array
      * @return the lob value
      */
-    public static ValueLob2 createSmallLob(int type, byte[] small) {
-        return new ValueLob2(type, small);
+    public static ValueLob2 createSmallLob(int type, byte[] small, long precision) {
+        return new ValueLob2(type, small, precision);
     }
 
     /**
@@ -384,9 +378,9 @@ public class ValueLob2 extends Value {
             }
             if (len <= handler.getMaxLengthInplaceLob()) {
                 byte[] small = StringUtils.utf8Encode(new String(buff, 0, len));
-                return ValueLob2.createSmallLob(Value.CLOB, small);
+                return ValueLob2.createSmallLob(Value.CLOB, small, len);
             }
-            ValueLob2 lob = new ValueLob2(Value.CLOB, null);
+            ValueLob2 lob = new ValueLob2(Value.CLOB, null, 0);
             lob.createTempFromReader(buff, len, in, remaining, handler);
             return lob;
         } catch (IOException e) {
@@ -421,9 +415,9 @@ public class ValueLob2 extends Value {
             if (len <= handler.getMaxLengthInplaceLob()) {
                 byte[] small = Utils.newBytes(len);
                 System.arraycopy(buff, 0, small, 0, len);
-                return ValueLob2.createSmallLob(Value.BLOB, small);
+                return ValueLob2.createSmallLob(Value.BLOB, small, small.length);
             }
-            ValueLob2 lob = new ValueLob2(Value.BLOB, null);
+            ValueLob2 lob = new ValueLob2(Value.BLOB, null, 0);
             lob.createTempFromStream(buff, len, in, remaining, handler);
             return lob;
         } catch (IOException e) {

h2/src/test/org/h2/test/TestAll.java

@@ -292,11 +292,15 @@ java org.h2.test.TestAll timer
 
 
         int testing;
-//        System.setProperty("h2.lobInDatabase", "true");
+        System.setProperty("h2.lobInDatabase", "true");
 
 /*
 
+new lob storage: test compression
+new lob storage: test in-place storage (old and new)
+
 power failure test
+power failure test: MULTI_THREADED=TRUE
 power failure test: larger binaries and additional index.
 power failure test with randomly generating / dropping indexes and tables.