New experimental LOB storage.

h2/src/docsrc/html/changelog.html

@@ -18,14 +18,19 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>The Shell tool threw a NumberFormatException on a empty statement.
+<ul><li>A new experimental LOB storage mechanism is available.
+    When enabled, CLOB and BLOB data is stored in hidden tables in the database.
+    To use it, set the system property "h2.lobInDatabase" to "true".
+    Compression is not yet implemented. Duplicate objects are only stored once.
+    Usually writing is slower, but reading is faster (mainly because of caching).
+</li><li>The Shell tool threw a NumberFormatException on a empty statement.
 </li><li>The h2small-*.jar did not support "drop all objects delete files".
     Now it is supported (the DeleteDbFiles tool is now included).
-</li><li>Operations that don't fit in memory are now faster. 
+</li><li>Operations that don't fit in memory are now faster.
     Temporary file handling was changed.
 </li><li>The default maximum log stream size is now 2 MB. This is a good value
     according to some performance tests.
-</li><li>New system property "h2.syncMethod" to configure what method to call when 
+</li><li>New system property "h2.syncMethod" to configure what method to call when
     closing the database, on checkpoint, and on CHECKPOINT SYNC. The default is "sync" which
     calls RandomAccessFile.getFD().sync().
 </li><li>ROWNUM could not be used for pagination. The follow query returned no rows:

h2/src/main/org/h2/command/Parser.java

@@ -4810,6 +4810,9 @@ public class Parser {
             read("PERSISTENT");
             command.setPersistData(false);
         }
+        if (readIf("HIDDEN")) {
+            command.setHidden(true);
+        }
         return command;
     }
 

h2/src/main/org/h2/command/ddl/CreateTable.java

@@ -267,4 +267,8 @@ public class CreateTable extends SchemaCommand {
         data.tableEngine = tableEngine;
     }
 
+    public void setHidden(boolean isHidden) {
+        data.isHidden = isHidden;
+    }
+
 }

h2/src/main/org/h2/command/ddl/CreateTableData.java

@@ -72,4 +72,9 @@ public class CreateTableData {
      */
     public String tableEngine;
 
+    /**
+     * The table is hidden.
+     */
+    public boolean isHidden;
+
 }

h2/src/main/org/h2/command/dml/ScriptCommand.java

@@ -180,6 +180,9 @@ public class ScriptCommand extends ScriptBase {
                 }
             });
             for (Table table : tables) {
+                if (table.isHidden()) {
+                    continue;
+                }
                 table.lock(session, false, false);
                 String sql = table.getCreateSQL();
                 if (sql == null) {
@@ -198,6 +201,9 @@ public class ScriptCommand extends ScriptBase {
                 add(sequence.getCreateSQL(), false);
             }
             for (Table table : tables) {
+                if (table.isHidden()) {
+                    continue;
+                }
                 table.lock(session, false, false);
                 String sql = table.getCreateSQL();
                 if (sql == null) {
@@ -299,6 +305,9 @@ public class ScriptCommand extends ScriptBase {
             });
             for (SchemaObject obj : constraints) {
                 Constraint constraint = (Constraint) obj;
+                if (constraint.getTable().isHidden()) {
+                    continue;
+                }
                 if (!Constraint.PRIMARY_KEY.equals(constraint.getConstraintType())) {
                     add(constraint.getCreateSQLWithoutIndexes(), false);
                 }

h2/src/main/org/h2/constraint/ConstraintCheck.java

@@ -45,7 +45,11 @@ public class ConstraintCheck extends Constraint {
 
     public String getCreateSQLForCopy(Table forTable, String quotedName) {
         StringBuilder buff = new StringBuilder("ALTER TABLE ");
-        buff.append(forTable.getSQL()).append(" ADD CONSTRAINT ").append(quotedName);
+        buff.append(forTable.getSQL()).append(" ADD CONSTRAINT ");
+        if (forTable.isHidden()) {
+            buff.append("IF NOT EXISTS ");
+        }
+        buff.append(quotedName);
         if (comment != null) {
             buff.append(" COMMENT ").append(StringUtils.quoteStringSQL(comment));
         }

h2/src/main/org/h2/constraint/ConstraintReferential.java

@@ -112,7 +112,11 @@ public class ConstraintReferential extends Constraint {
     public String getCreateSQLForCopy(Table forTable, Table forRefTable, String quotedName, boolean internalIndex) {
         StatementBuilder buff = new StatementBuilder("ALTER TABLE ");
         String mainTable = forTable.getSQL();
-        buff.append(mainTable).append(" ADD CONSTRAINT ").append(quotedName);
+        buff.append(mainTable).append(" ADD CONSTRAINT ");
+        if (forTable.isHidden()) {
+            buff.append("IF NOT EXISTS ");
+        }
+        buff.append(quotedName);
         if (comment != null) {
             buff.append(" COMMENT ").append(StringUtils.quoteStringSQL(comment));
         }

h2/src/main/org/h2/constraint/ConstraintUnique.java

@@ -42,7 +42,11 @@ public class ConstraintUnique extends Constraint {
 
     private String getCreateSQLForCopy(Table forTable, String quotedName, boolean internalIndex) {
         StatementBuilder buff = new StatementBuilder("ALTER TABLE ");
-        buff.append(forTable.getSQL()).append(" ADD CONSTRAINT ").append(quotedName);
+        buff.append(forTable.getSQL()).append(" ADD CONSTRAINT ");
+        if (forTable.isHidden()) {
+            buff.append("IF NOT EXISTS ");
+        }
+        buff.append(quotedName);
         if (comment != null) {
             buff.append(" COMMENT ").append(StringUtils.quoteStringSQL(comment));
         }

h2/src/main/org/h2/engine/Database.java

@@ -2233,7 +2233,9 @@ public class Database implements DataHandler {
     public Connection getLobConnection() {
         if (SysProperties.LOB_IN_DATABASE) {
             String url = Constants.CONN_URL_INTERNAL;
-            return new JdbcConnection(systemSession, systemUser.getName(), url);
+            JdbcConnection conn = new JdbcConnection(systemSession, systemUser.getName(), url);
+            conn.setTraceLevel(TraceSystem.OFF);
+            return conn;
         }
         return null;
     }

h2/src/main/org/h2/index/BaseIndex.java

@@ -322,7 +322,11 @@ public abstract class BaseIndex extends SchemaObjectBase implements Index {
     public String getCreateSQLForCopy(Table targetTable, String quotedName) {
         StringBuilder buff = new StringBuilder("CREATE ");
         buff.append(indexType.getSQL());
-        buff.append(' ').append(quotedName);
+        buff.append(' ');
+        if (table.isHidden()) {
+            buff.append("IF NOT EXISTS ");
+        }
+        buff.append(quotedName);
         buff.append(" ON ").append(targetTable.getSQL());
         if (comment != null) {
             buff.append(" COMMENT ").append(StringUtils.quoteStringSQL(comment));

h2/src/main/org/h2/jdbc/JdbcConnection.java

@@ -1700,4 +1700,11 @@ public class JdbcConnection extends TraceObject implements Connection {
         return compareMode;
     }
 
+    /**
+     * INTERNAL
+     */
+    public void setTraceLevel(int level) {
+        trace.setLevel(level);
+    }
+
 }

h2/src/main/org/h2/store/Data.java

@@ -141,16 +141,25 @@ public class Data {
     }
 
     /**
-     * Get the length of a String value.
+     * Get the length of a String. This includes the bytes required to encode
+     * the length.
      *
-     * @param s the value
-     * @return the length
+     * @param s the string
+     * @return the number of bytes required
      */
     public static int getStringLen(String s) {
         int len = s.length();
         return getStringWithoutLengthLen(s, len) + getVarIntLen(len);
     }
 
+    /**
+     * Calculate the length of String, excluding the bytes required to encode
+     * the length.
+     *
+     * @param s the string
+     * @param len the length of the string
+     * @return the number of bytes required
+     */
     public static int getStringWithoutLengthLen(String s, int len) {
         int plus = 0;
         for (int i = 0; i < len; i++) {
@@ -298,7 +307,7 @@ public class Data {
         System.arraycopy(buff, off, data, pos, len);
         pos += len;
     }
-    
+
     /**
      * Append a number of bytes to this buffer.
      *
@@ -320,7 +329,7 @@ public class Data {
         System.arraycopy(data, pos, buff, off, len);
         pos += len;
     }
-    
+
     /**
      * Copy a number of bytes to the given buffer from the current position. The
      * current position is incremented accordingly.
@@ -330,7 +339,7 @@ public class Data {
     public void read(byte[] buff) {
         read(buff, 0, buff.length);
     }
-    
+
     /**
      * Append one single byte.
      *
@@ -745,7 +754,7 @@ public class Data {
     public int getValueLen(Value v) {
         return getValueLen(v, handler);
     }
-    
+
     /**
      * Calculate the number of bytes required to encode the given value.
      *

h2/src/main/org/h2/store/LobStorage.java

@@ -74,10 +74,10 @@ public class LobStorage {
             Statement stat = conn.createStatement();
             // stat.execute("SET UNDO_LOG 0");
             // stat.execute("SET REDO_LOG_BINARY 0");
-            stat.execute("CREATE TABLE IF NOT EXISTS " + LOBS + "(ID BIGINT PRIMARY KEY, LENGTH BIGINT, TABLE INT)");
-            stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_MAP + "(LOB BIGINT, SEQ INT, BLOCK BIGINT, PRIMARY KEY(LOB, SEQ))");
+            stat.execute("CREATE TABLE IF NOT EXISTS " + LOBS + "(ID BIGINT PRIMARY KEY, LENGTH BIGINT, TABLE INT) HIDDEN");
+            stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_MAP + "(LOB BIGINT, SEQ INT, BLOCK BIGINT, PRIMARY KEY(LOB, SEQ)) HIDDEN");
             stat.execute("CREATE INDEX IF NOT EXISTS INFORMATION_SCHEMA.INDEX_LOB_MAP_DATA_LOB ON " + LOB_MAP + "(BLOCK, LOB)");
-            stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, DATA BINARY)");
+            stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, DATA BINARY) HIDDEN");
             ResultSet rs;
             rs = stat.executeQuery("SELECT MAX(BLOCK) FROM " + LOB_DATA);
             rs.next();
@@ -166,7 +166,7 @@ public class LobStorage {
         }
 
         public int read(byte[] buff, int off, int length) throws IOException {
-            return readFully(buff, 0, buff.length);
+            return readFully(buff, off, length);
         }
 
         private int readFully(byte[] buff, int off, int length) throws IOException {
@@ -182,6 +182,7 @@ public class LobStorage {
                 int len = (int) Math.min(length, remaining);
                 len = Math.min(len, buffer.length - pos);
                 System.arraycopy(buffer, pos, buff, off, len);
+                pos += len;
                 read += len;
                 remaining -= len;
                 off += len;
@@ -263,6 +264,8 @@ public class LobStorage {
     }
 
     private ValueLob2 addLob(InputStream in, long maxLength, int type) {
+        int todo;
+        // TODO support in-place lobs, and group lobs much smaller than the page size
         byte[] buff = new byte[BLOCK_LENGTH];
         if (maxLength < 0) {
             maxLength = Long.MAX_VALUE;
@@ -272,7 +275,8 @@ public class LobStorage {
         try {
             try {
                 for (int seq = 0; maxLength > 0; seq++) {
-                    int len = IOUtils.readFully(in, buff, 0, BLOCK_LENGTH);
+                    int len = (int) Math.min(BLOCK_LENGTH, maxLength);
+                    len = IOUtils.readFully(in, buff, 0, len);
                     if (len <= 0) {
                         break;
                     }
@@ -344,16 +348,21 @@ public class LobStorage {
 
         private final Reader reader;
         private long length;
+        private long remaining;
         private int pos;
         private char[] charBuffer = new char[Constants.IO_BUFFER_SIZE];
         private byte[] buffer;
 
-        CountingReaderInputStream(Reader reader) {
+        CountingReaderInputStream(Reader reader, long maxLength) {
             this.reader = reader;
+            this.remaining = maxLength;
             buffer = Utils.EMPTY_BYTES;
         }
 
         public int read(byte[] buff, int offset, int len) throws IOException {
+            if (buffer == null) {
+                return -1;
+            }
             if (pos >= buffer.length) {
                 fillBuffer();
                 if (buffer == null) {
@@ -362,10 +371,14 @@ public class LobStorage {
             }
             len = Math.min(len, buffer.length - pos);
             System.arraycopy(buffer, pos, buff, offset, len);
+            pos += len;
             return len;
         }
 
         public int read() throws IOException {
+            if (buffer == null) {
+                return -1;
+            }
             if (pos >= buffer.length) {
                 fillBuffer();
                 if (buffer == null) {
@@ -376,13 +389,20 @@ public class LobStorage {
         }
 
         private void fillBuffer() throws IOException {
-            int len = reader.read(charBuffer);
+            int len = (int) Math.min(charBuffer.length, remaining);
+            if (len > 0) {
+                len = reader.read(charBuffer, 0, len);
+            } else {
+                len = -1;
+            }
             if (len < 0) {
                 buffer = null;
             } else {
                 buffer = StringUtils.utf8Encode(new String(charBuffer, 0, len));
                 length += len;
+                remaining -= len;
             }
+            pos = 0;
         }
 
         public long getLength() {
@@ -426,8 +446,9 @@ public class LobStorage {
             if (conn == null) {
                 return ValueLob2.createTempClob(reader, maxLength, handler);
             }
-            CountingReaderInputStream in = new CountingReaderInputStream(reader);
-            ValueLob2 lob = addLob(in, maxLength, Value.BLOB);
+            long max = maxLength == -1 ? Long.MAX_VALUE : maxLength;
+            CountingReaderInputStream in = new CountingReaderInputStream(reader, max);
+            ValueLob2 lob = addLob(in, Long.MAX_VALUE, Value.CLOB);
             lob.setPrecision(in.getLength());
             return lob;
         }

h2/src/main/org/h2/store/fs/FileSystem.java

@@ -316,9 +316,9 @@ public abstract class FileSystem {
      * @param inTempDir if the file should be stored in the temporary directory
      * @return the name of the created file
      */
-    public String createTempFile(String name, String suffix, boolean deleteOnExit, boolean inTempDir) throws IOException {
+    public String createTempFile(String prefix, String suffix, boolean deleteOnExit, boolean inTempDir) throws IOException {
         while (true) {
-            String n = name + getNextTempFileNamePart(false) + suffix;
+            String n = prefix + getNextTempFileNamePart(false) + suffix;
             if (exists(n)) {
                 // in theory, the random number could collide
                 getNextTempFileNamePart(true);

h2/src/main/org/h2/table/MetaTable.java

@@ -619,6 +619,9 @@ public class MetaTable extends Table {
                 if (!checkIndex(session, tableName, indexFrom, indexTo)) {
                     continue;
                 }
+                if (table.isHidden()) {
+                    continue;
+                }
                 String storageType;
                 if (table.isTemporary()) {
                     if (table.isGlobalTemporary()) {
@@ -660,6 +663,9 @@ public class MetaTable extends Table {
                 if (!checkIndex(session, tableName, indexFrom, indexTo)) {
                     continue;
                 }
+                if (table.isHidden()) {
+                    continue;
+                }
                 Column[] cols = table.getColumns();
                 String collation = database.getCompareMode().getName();
                 for (int j = 0; j < cols.length; j++) {
@@ -723,6 +729,9 @@ public class MetaTable extends Table {
                 if (!checkIndex(session, tableName, indexFrom, indexTo)) {
                     continue;
                 }
+                if (table.isHidden()) {
+                    continue;
+                }
                 ArrayList<Index> indexes = table.getIndexes();
                 ArrayList<Constraint> constraints = table.getConstraints();
                 for (int j = 0; indexes != null && j < indexes.size(); j++) {
@@ -1175,7 +1184,7 @@ public class MetaTable extends Table {
         case TABLE_PRIVILEGES: {
             for (Right r : database.getAllRights()) {
                 Table table = r.getGrantedTable();
-                if (table == null) {
+                if (table == null || table.isHidden()) {
                     continue;
                 }
                 String tableName = identifier(table.getName());
@@ -1189,7 +1198,7 @@ public class MetaTable extends Table {
         case COLUMN_PRIVILEGES: {
             for (Right r : database.getAllRights()) {
                 Table table = r.getGrantedTable();
-                if (table == null) {
+                if (table == null || table.isHidden()) {
                     continue;
                 }
                 String tableName = identifier(table.getName());
@@ -1321,6 +1330,9 @@ public class MetaTable extends Table {
                 String checkExpression = null;
                 IndexColumn[] indexColumns = null;
                 Table table = constraint.getTable();
+                if (table.isHidden()) {
+                    continue;
+                }
                 Index index = constraint.getUniqueIndex();
                 String uniqueIndexName = null;
                 if (index != null) {

h2/src/main/org/h2/table/RegularTable.java

@@ -67,6 +67,7 @@ public class RegularTable extends TableBase {
 
     public RegularTable(CreateTableData data) {
         super(data);
+        this.isHidden = data.isHidden;
         if (data.persistData && database.isPersistent()) {
             mainIndex = new PageDataIndex(this, data.id, IndexColumn.wrap(getColumns()), IndexType.createScan(data.persistData), data.create, data.session);
             scanIndex = mainIndex;

h2/src/main/org/h2/table/Table.java

@@ -95,6 +95,12 @@ public abstract class Table extends SchemaObjectBase {
      */
     protected CompareMode compareMode;
 
+    /**
+     * Protected tables are not listed in the meta data and are excluded when
+     * using the SCRIPT command.
+     */
+    protected boolean isHidden;
+
     private final HashMap<String, Column> columnMap = New.hashMap();
     private boolean persistIndexes;
     private boolean persistData;
@@ -966,4 +972,14 @@ public abstract class Table extends SchemaObjectBase {
         return column.convert(v);
     }
 
+    /**
+     * Check whether this is a hidden table that doesn't appear in the meta
+     * data and in the script.
+     *
+     * @return true if it is hidden
+     */
+    public boolean isHidden() {
+        return isHidden;
+    }
+
 }

h2/src/main/org/h2/table/TableBase.java

@@ -53,7 +53,11 @@ public abstract class TableBase extends Table {
         } else {
             buff.append("MEMORY ");
         }
-        buff.append("TABLE ").append(getSQL());
+        buff.append("TABLE ");
+        if (isHidden) {
+            buff.append("IF NOT EXISTS ");
+        }
+        buff.append(getSQL());
         if (comment != null) {
             buff.append(" COMMENT ").append(StringUtils.quoteStringSQL(comment));
         }
@@ -71,6 +75,9 @@ public abstract class TableBase extends Table {
         if (!isPersistIndexes() && !isPersistData()) {
             buff.append("\nNOT PERSISTENT");
         }
+        if (isHidden) {
+            buff.append("\nHIDDEN");
+        }
         return buff.toString();
     }
 

h2/src/main/org/h2/value/ValueLob2.java

@@ -102,11 +102,15 @@ public class ValueLob2 extends Value {
         if (t == type) {
             return this;
         } else if (t == Value.CLOB) {
-            Value copy = handler.getLobStorage().createClob(getReader(), -1);
-            return copy;
+            if (lobStorage != null) {
+                Value copy = lobStorage.createClob(getReader(), -1);
+                return copy;
+            }
         } else if (t == Value.BLOB) {
-            Value copy = handler.getLobStorage().createBlob(getInputStream(), -1);
-            return copy;
+            if (lobStorage != null) {
+                Value copy = lobStorage.createBlob(getInputStream(), -1);
+                return copy;
+            }
         }
         return super.convertTo(t);
     }
@@ -474,6 +478,7 @@ public class ValueLob2 extends Value {
     private FileStoreOutputStream initTemp(DataHandler h) {
         this.precision = 0;
         this.handler = h;
+        this.lobStorage = h.getLobStorage();
         this.small = null;
         try {
             String path = h.getDatabasePath();

h2/src/test/org/h2/test/TestAll.java

@@ -290,6 +290,10 @@ java org.h2.test.TestAll timer
         System.setProperty("h2.maxMemoryRowsDistinct", "128");
         System.setProperty("h2.check2", "true");
 
+
+        int testing;
+        System.setProperty("h2.lobInDatabase", "true");
+
 /*
 
 power failure test
@@ -304,8 +308,6 @@ delete from test;
 rename Page* classes
 move classes to the right packages
 
-Fix the default value and documentation for MAX_LOG_SIZE.
-
 // System.setProperty("h2.pageSize", "64");
 test with small freeList pages, page size 64
 
@@ -351,12 +353,12 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         } else {
             test.runTests();
             Profiler prof = new Profiler();
-            prof.depth = 10;
+            prof.depth = 4;
             prof.interval = 1;
             prof.startCollecting();
             TestPerformance.main("-init", "-db", "1");
             prof.stopCollecting();
-            System.out.println(prof.getTop(2));
+            System.out.println(prof.getTop(3));
 //            Recover.execute("data", null);
 //            RunScript.execute("jdbc:h2:data/test2",
 //                 "sa1", "sa1", "data/test.h2.sql", null, false);

h2/src/test/org/h2/test/db/TestLob.java

@@ -138,6 +138,9 @@ public class TestLob extends TestBase {
     }
 
     private void testLobDeleteTemp() throws SQLException {
+        if (SysProperties.LOB_IN_DATABASE) {
+            return;
+        }
         deleteDb("lob");
         Connection conn = getConnection("lob");
         Statement stat = conn.createStatement();
@@ -161,7 +164,7 @@ public class TestLob extends TestBase {
     }
 
     private void testLobDelete() throws SQLException {
-        if (config.memory) {
+        if (config.memory || SysProperties.LOB_IN_DATABASE) {
             return;
         }
         deleteDb("lob");

h2/src/test/org/h2/test/unit/TestTools.java

@@ -367,6 +367,7 @@ public class TestTools extends TestBase {
         assertTrue(trace.exists());
         File newTrace = new File(baseDir + "/test.trace.db");
         newTrace.delete();
+        assertFalse(newTrace.exists());
         assertTrue(trace.renameTo(newTrace));
         deleteDb("toolsConvertTraceFile");
         Player.main(baseDir + "/test.trace.db");