New experimental LOB storage mechanism.

h2/src/main/org/h2/store/Data.java

@@ -32,7 +32,7 @@ import org.h2.value.ValueFloat;
 import org.h2.value.ValueInt;
 import org.h2.value.ValueJavaObject;
 import org.h2.value.ValueLob;
-import org.h2.value.ValueLob2;
+import org.h2.value.ValueLobDb;
 import org.h2.value.ValueLong;
 import org.h2.value.ValueNull;
 import org.h2.value.ValueShort;
@@ -565,10 +565,11 @@ public class Data {
                     write(small, 0, small.length);
                 }
             } else {
-                ValueLob2 lob = (ValueLob2) v;
+                ValueLobDb lob = (ValueLobDb) v;
                 byte[] small = lob.getSmall();
                 if (small == null) {
                     writeVarInt(-3);
+                    writeVarInt(lob.getTableId());
                     writeVarLong(lob.getLobId());
                     writeVarLong(lob.getPrecision());
                 } else {
@@ -697,10 +698,11 @@ public class Data {
                 read(small, 0, smallLen);
                 return LobStorage.createSmallLob(type, small);
             } else if (smallLen == -3) {
+                int tableId = readVarInt();
                 long lobId = readVarLong();
                 long precision = readVarLong();
                 LobStorage lobStorage = handler.getLobStorage();
-                ValueLob2 lob = ValueLob2.create(type, lobStorage, null, lobId, precision);
+                ValueLobDb lob = ValueLobDb.create(type, lobStorage, null, tableId, lobId, precision);
                 return lob;
             } else {
                 int tableId = readVarInt();
@@ -890,10 +892,11 @@ public class Data {
                     len += small.length;
                 }
             } else {
-                ValueLob2 lob = (ValueLob2) v;
+                ValueLobDb lob = (ValueLobDb) v;
                 byte[] small = lob.getSmall();
                 if (small == null) {
                     len += getVarIntLen(-3);
+                    len += getVarIntLen(lob.getTableId());
                     len += getVarLongLen(lob.getLobId());
                     len += getVarLongLen(lob.getPrecision());
                 } else {

h2/src/main/org/h2/store/LobStorage.java

@@ -16,7 +16,6 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.concurrent.atomic.AtomicLong;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
@@ -28,7 +27,7 @@ import org.h2.util.StringUtils;
 import org.h2.util.Utils;
 import org.h2.value.Value;
 import org.h2.value.ValueLob;
-import org.h2.value.ValueLob2;
+import org.h2.value.ValueLobDb;
 
 /**
  * This class stores LOB objects in the database.
@@ -40,6 +39,12 @@ public class LobStorage {
      */
     public static final int TABLE_ID_SESSION_VARIABLE = -1;
 
+    /**
+     * The table id for temporary objects (not assigned to any object).
+     */
+    public static final int TABLE_TEMP = -2;
+
+
     private static final String LOBS = "INFORMATION_SCHEMA.LOBS";
     private static final String LOB_MAP = "INFORMATION_SCHEMA.LOB_MAP";
     private static final String LOB_DATA = "INFORMATION_SCHEMA.LOB_DATA";
@@ -49,7 +54,6 @@ public class LobStorage {
     private static final long UNIQUE = 0xffff;
     private Connection conn;
     private HashMap<String, PreparedStatement> prepared = New.hashMap();
-    private AtomicLong nextLob = new AtomicLong();
     private long nextBlock;
     private CompressTool compress = CompressTool.getInstance();
 
@@ -76,7 +80,7 @@ public class LobStorage {
             Statement stat = conn.createStatement();
             // stat.execute("SET UNDO_LOG 0");
             // stat.execute("SET REDO_LOG_BINARY 0");
-            stat.execute("CREATE TABLE IF NOT EXISTS " + LOBS + "(ID BIGINT PRIMARY KEY, LENGTH BIGINT, TABLE INT) HIDDEN");
+            stat.execute("CREATE TABLE IF NOT EXISTS " + LOBS + "(ID BIGINT PRIMARY KEY, BYTE_COUNT BIGINT, TABLE INT) HIDDEN");
             stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_MAP + "(LOB BIGINT, SEQ INT, BLOCK BIGINT, PRIMARY KEY(LOB, SEQ)) HIDDEN");
             stat.execute("CREATE INDEX IF NOT EXISTS INFORMATION_SCHEMA.INDEX_LOB_MAP_DATA_LOB ON " + LOB_MAP + "(BLOCK, LOB)");
             stat.execute("CREATE TABLE IF NOT EXISTS " + LOB_DATA + "(BLOCK BIGINT PRIMARY KEY, COMPRESSED INT, DATA BINARY) HIDDEN");
@@ -85,16 +89,20 @@ public class LobStorage {
             rs.next();
             nextBlock = rs.getLong(1) + 1;
             if (HASH) {
-                nextBlock = Math.max(UNIQUE + 1, nextLob.get());
+                nextBlock = Math.max(UNIQUE + 1, nextBlock);
             }
-            rs = stat.executeQuery("SELECT MAX(ID) FROM " + LOBS);
-            rs.next();
-            nextLob.set(rs.getLong(1) + 1);
         } catch (SQLException e) {
             throw DbException.convert(e);
         }
     }
 
+    private long getNextLobId() throws SQLException {
+        PreparedStatement prep = prepare("SELECT MAX(ID) FROM " + LOBS);
+        ResultSet rs = prep.executeQuery();
+        rs.next();
+        return rs.getLong(1) + 1;
+    }
+
     /**
      * Remove all LOBs for this table.
      *
@@ -104,7 +112,7 @@ public class LobStorage {
         if (SysProperties.LOB_IN_DATABASE) {
             init();
             try {
-                PreparedStatement prep = prepare("SELECT ID FROM " + LOBS + " WHERE TABLE=?");
+                PreparedStatement prep = prepare("SELECT ID FROM " + LOBS + " WHERE TABLE = ?");
                 prep.setInt(1, tableId);
                 ResultSet rs = prep.executeQuery();
                 while (rs.next()) {
@@ -113,6 +121,9 @@ public class LobStorage {
             } catch (SQLException e) {
                 throw DbException.convert(e);
             }
+            if (tableId == TABLE_ID_SESSION_VARIABLE) {
+                removeAllForTable(TABLE_TEMP);
+            }
             // remove both lobs in the database as well as in the file system
             // (compatibility)
         }
@@ -134,7 +145,7 @@ public class LobStorage {
             } else {
                 precision = small.length;
             }
-            return ValueLob2.createSmallLob(type, small, precision);
+            return ValueLobDb.createSmallLob(type, small, precision);
         }
         return ValueLob.createSmallLob(type, small);
     }
@@ -148,7 +159,7 @@ public class LobStorage {
         private PreparedStatement prepSelect;
         private byte[] buffer;
         private int pos;
-        private long remaining;
+        private long remainingBytes;
         private long lob;
         private int seq;
         private CompressTool compress;
@@ -158,13 +169,13 @@ public class LobStorage {
             try {
                 this.lob = lob;
                 PreparedStatement prep = conn.prepareStatement(
-                        "SELECT LENGTH FROM " + LOBS + " WHERE ID = ?");
+                        "SELECT BYTE_COUNT FROM " + LOBS + " WHERE ID = ?");
                 prep.setLong(1, lob);
                 ResultSet rs = prep.executeQuery();
                 if (!rs.next()) {
                     throw DbException.get(ErrorCode.IO_EXCEPTION_1, "Missing lob: "+ lob).getSQLException();
                 }
-                remaining = rs.getLong(1);
+                remainingBytes = rs.getLong(1);
                 rs.close();
             } catch (SQLException e) {
                 throw DbException.convertToIOException(e);
@@ -173,10 +184,10 @@ public class LobStorage {
 
         public int read() throws IOException {
             fillBuffer();
-            if (remaining <= 0) {
+            if (remainingBytes <= 0) {
                 return -1;
             }
-            remaining--;
+            remainingBytes--;
             return buffer[pos++] & 255;
         }
 
@@ -195,15 +206,15 @@ public class LobStorage {
             int read = 0;
             while (length > 0) {
                 fillBuffer();
-                if (remaining <= 0) {
+                if (remainingBytes <= 0) {
                     break;
                 }
-                int len = (int) Math.min(length, remaining);
+                int len = (int) Math.min(length, remainingBytes);
                 len = Math.min(len, buffer.length - pos);
                 System.arraycopy(buffer, pos, buff, off, len);
                 pos += len;
                 read += len;
-                remaining -= len;
+                remainingBytes -= len;
                 off += len;
                 length -= len;
             }
@@ -214,7 +225,7 @@ public class LobStorage {
             if (buffer != null && pos < buffer.length) {
                 return;
             }
-            if (remaining <= 0) {
+            if (remainingBytes <= 0) {
                 return;
             }
             try {
@@ -289,17 +300,17 @@ public class LobStorage {
         return new LobInputStream(conn, lobId);
     }
 
-    private ValueLob2 addLob(InputStream in, long maxLength, int type) {
+    private ValueLobDb addLob(InputStream in, long maxLength, int type) {
         byte[] buff = new byte[BLOCK_LENGTH];
         if (maxLength < 0) {
             maxLength = Long.MAX_VALUE;
         }
         long length = 0;
         long lobId;
-        lobId = nextLob.getAndIncrement();
         int maxLengthInPlaceLob = handler.getMaxLengthInplaceLob();
         String compressAlgorithm = handler.getLobCompressionAlgorithm(type);
         try {
+            lobId = getNextLobId();
             try {
                 for (int seq = 0; maxLength > 0; seq++) {
                     int len = (int) Math.min(BLOCK_LENGTH, maxLength);
@@ -318,23 +329,63 @@ public class LobStorage {
                     }
                     if (seq == 0 && b.length < BLOCK_LENGTH && b.length <= maxLengthInPlaceLob) {
                         // CLOB: the precision will be fixed later
-                        ValueLob2 v = ValueLob2.createSmallLob(type, b, b.length);
+                        ValueLobDb v = ValueLobDb.createSmallLob(type, b, b.length);
                         return v;
                     }
                     storeBlock(lobId, seq, b, compressAlgorithm);
                 }
+                return registerLob(type, lobId, TABLE_TEMP, length);
+            } catch (IOException e) {
+                deleteLob(lobId);
+                throw DbException.convertIOException(e, "adding blob");
+            }
+        } catch (SQLException e) {
+            throw DbException.convert(e);
+        }
+    }
+
+    private ValueLobDb registerLob(int type, long lobId, int tableId, long byteCount) {
+        try {
             PreparedStatement prep = prepare(
-                        "INSERT INTO " + LOBS + "(ID, LENGTH, TABLE) VALUES(?, ?, ?)");
+                    "INSERT INTO " + LOBS + "(ID, BYTE_COUNT, TABLE) VALUES(?, ?, ?)");
             prep.setLong(1, lobId);
-                prep.setLong(2, length);
-                prep.setInt(3, TABLE_ID_SESSION_VARIABLE);
+            prep.setLong(2, byteCount);
+            prep.setInt(3, tableId);
             prep.execute();
-                ValueLob2 v = ValueLob2.create(type, this, null, lobId, length);
+            ValueLobDb v = ValueLobDb.create(type, this, null, tableId, lobId, byteCount);
             return v;
-            } catch (IOException e) {
-                deleteLob(lobId);
-                throw DbException.convertIOException(e, "adding blob");
+        } catch (SQLException e) {
+            throw DbException.convert(e);
         }
+    }
+
+    /**
+     * Copy a lob.
+     *
+     * @param type the type
+     * @param oldLobId the old lob id
+     * @param tableId the new table id
+     * @param length the length
+     * @return the new lob
+     */
+    public ValueLobDb copyLob(int type, long oldLobId, int tableId, long length) {
+        try {
+            long lobId = getNextLobId();
+            PreparedStatement prep = prepare(
+                    "INSERT INTO " + LOB_MAP + "(LOB, SEQ, BLOCK) " +
+                    "SELECT ?, SEQ, BLOCK FROM " + LOB_MAP + " WHERE LOB = ?");
+            prep.setLong(1, lobId);
+            prep.setLong(2, oldLobId);
+            prep.executeUpdate();
+            prep = prepare(
+                    "INSERT INTO " + LOBS + "(ID, BYTE_COUNT, TABLE) " +
+                    "SELECT ?, BYTE_COUNT, ? FROM " + LOBS + " WHERE ID = ?");
+            prep.setLong(1, lobId);
+            prep.setLong(2, tableId);
+            prep.setLong(3, oldLobId);
+            prep.executeUpdate();
+            ValueLobDb v = ValueLobDb.create(type, this, null, tableId, lobId, length);
+            return v;
         } catch (SQLException e) {
             throw DbException.convert(e);
         }
@@ -474,7 +525,7 @@ public class LobStorage {
         if (SysProperties.LOB_IN_DATABASE) {
             init();
             if (conn == null) {
-                return ValueLob2.createTempBlob(in, maxLength, handler);
+                return ValueLobDb.createTempBlob(in, maxLength, handler);
             }
             return addLob(in, maxLength, Value.BLOB);
         }
@@ -492,11 +543,11 @@ public class LobStorage {
         if (SysProperties.LOB_IN_DATABASE) {
             init();
             if (conn == null) {
-                return ValueLob2.createTempClob(reader, maxLength, handler);
+                return ValueLobDb.createTempClob(reader, maxLength, handler);
             }
             long max = maxLength == -1 ? Long.MAX_VALUE : maxLength;
             CountingReaderInputStream in = new CountingReaderInputStream(reader, max);
-            ValueLob2 lob = addLob(in, Long.MAX_VALUE, Value.CLOB);
+            ValueLobDb lob = addLob(in, Long.MAX_VALUE, Value.CLOB);
             lob.setPrecision(in.getLength());
             return lob;
         }
@@ -516,7 +567,8 @@ public class LobStorage {
             prep.setLong(2, lobId);
             int updateCount = prep.executeUpdate();
             if (updateCount != 1) {
-                throw DbException.throwInternalError("count: " + updateCount);
+                // can be zero when recovering
+                // throw DbException.throwInternalError("count: " + updateCount);
             }
         } catch (SQLException e) {
             throw DbException.convert(e);

h2/src/main/org/h2/tools/Recover.java

@@ -18,6 +18,7 @@ import java.io.Reader;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashMap;
@@ -29,6 +30,7 @@ import org.h2.constant.SysProperties;
 import org.h2.engine.Constants;
 import org.h2.engine.DbObject;
 import org.h2.engine.MetaRecord;
+import org.h2.jdbc.JdbcConnection;
 import org.h2.message.DbException;
 import org.h2.result.Row;
 import org.h2.result.SimpleRow;
@@ -55,7 +57,7 @@ import org.h2.util.Tool;
 import org.h2.util.Utils;
 import org.h2.value.Value;
 import org.h2.value.ValueLob;
-import org.h2.value.ValueLob2;
+import org.h2.value.ValueLobDb;
 import org.h2.value.ValueLong;
 
 /**
@@ -74,15 +76,46 @@ public class Recover extends Tool implements DataHandler {
     private ArrayList<MetaRecord> schema;
     private HashSet<Integer> objectIdSet;
     private HashMap<Integer, String> tableMap;
+    private HashMap<String, String> columnTypeMap;
     private boolean remove;
 
-    private long pageDataEmpty;
-    private int pageDataRows;
-    private int pageDataHead;
     private int pageSize;
     private FileStore store;
     private int[] parents;
 
+    private Stats stat;
+
+    /**
+     * Statistic data
+     */
+    class Stats {
+
+        /**
+         * The empty space in bytes in a data leaf pages.
+         */
+        long pageDataEmpty;
+
+        /**
+         * The number of bytes used for data.
+         */
+        int pageDataRows;
+
+        /**
+         * The number of bytes used for the page headers.
+         */
+        int pageDataHead;
+
+        /**
+         * The count per page type.
+         */
+        int[] pageTypeCount = new int[Page.TYPE_STREAM_DATA + 2];
+
+        /**
+         * The number of free pages.
+         */
+        int free;
+    }
+
     /**
      * Options are case sensitive. Supported options are:
      * <table>
@@ -155,18 +188,21 @@ public class Recover extends Tool implements DataHandler {
     /**
      * INTERNAL
      */
-    public static Reader readClobDb(Connection conn, long lobId) throws IOException {
-        return new BufferedReader(new InputStreamReader(readBlobDb(conn, lobId), "UTF-8"));
+    public static Value.ValueBlob readBlobDb(Connection conn, long lobId, long precision) {
+        DataHandler h = ((JdbcConnection) conn).getSession().getDataHandler();
+        LobStorage lobStorage = h.getLobStorage();
+        return ValueLobDb.create(Value.BLOB, lobStorage, null, LobStorage.TABLE_TEMP, lobId, precision);
     }
 
     /**
      * INTERNAL
      */
-    public static InputStream readBlobDb(Connection conn, long lobId) throws IOException {
-        return new BufferedInputStream(new LobStorage.LobInputStream(conn, lobId));
+    public static Value.ValueClob readClobDb(Connection conn, long lobId, long precision) {
+        DataHandler h = ((JdbcConnection) conn).getSession().getDataHandler();
+        LobStorage lobStorage = h.getLobStorage();
+        return ValueLobDb.create(Value.CLOB, lobStorage, null, LobStorage.TABLE_TEMP, lobId, precision);
     }
 
-
     private void trace(String message) {
         if (trace) {
             out.println(message);
@@ -279,7 +315,7 @@ public class Recover extends Tool implements DataHandler {
         }
     }
 
-    private String getSQL(Value v) {
+    private String getSQL(String column, Value v) {
         if (v instanceof ValueLob) {
             ValueLob lob = (ValueLob) v;
             byte[] small = lob.getSmall();
@@ -290,15 +326,24 @@ public class Recover extends Tool implements DataHandler {
                 }
                 return "READ_CLOB('" + file + ".txt')";
             }
-        } else if (v instanceof ValueLob2) {
-            ValueLob2 lob = (ValueLob2) v;
+        } else if (v instanceof ValueLobDb) {
+            ValueLobDb lob = (ValueLobDb) v;
             byte[] small = lob.getSmall();
             if (small == null) {
+                int type = lob.getType();
                 long id = lob.getLobId();
-                if (lob.getType() == Value.BLOB) {
-                    return "READ_BLOB_DB(" + id + ")";
+                long precision = lob.getPrecision();
+                String m;
+                String columnType;
+                if (type == Value.BLOB) {
+                    columnType = "BLOB";
+                    m = "READ_BLOB_DB";
+                } else {
+                    columnType = "CLOB";
+                    m = "READ_CLOB_DB";
                 }
-                return "READ_CLOB_DB(" + id + ")";
+                columnTypeMap.put(column, columnType);
+                return m + "(" + id + ", " + precision + ")";
             }
         }
         return v.getSQL();
@@ -311,17 +356,13 @@ public class Recover extends Tool implements DataHandler {
     private void dumpPageStore(String fileName) {
         setDatabaseName(fileName.substring(0, fileName.length() - Constants.SUFFIX_PAGE_FILE.length()));
         PrintWriter writer = null;
-        int[] pageTypeCount = new int[Page.TYPE_STREAM_DATA + 2];
-        int emptyPages = 0;
-        pageDataEmpty = 0;
-        pageDataRows = 0;
-        pageDataHead = 0;
+        stat = new Stats();
         try {
             writer = getWriter(fileName, ".sql");
-            writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB FOR \"" + this.getClass().getName() + ".readClob\";");
             writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB FOR \"" + this.getClass().getName() + ".readBlob\";");
-            writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB_DB FOR \"" + this.getClass().getName() + ".readClobDb\";");
+            writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB FOR \"" + this.getClass().getName() + ".readClob\";");
             writer.println("CREATE ALIAS IF NOT EXISTS READ_BLOB_DB FOR \"" + this.getClass().getName() + ".readBlobDb\";");
+            writer.println("CREATE ALIAS IF NOT EXISTS READ_CLOB_DB FOR \"" + this.getClass().getName() + ".readClobDb\";");
             resetSchema();
             store = FileStore.open(null, fileName, remove ? "rw" : "r");
             long length = store.length();
@@ -356,6 +397,7 @@ public class Recover extends Tool implements DataHandler {
                 parents[i] = s.readInt();
             }
             int logKey = 0, logFirstTrunkPage = 0, logFirstDataPage = 0;
+            s = Data.create(this, pageSize);
             for (int i = 1;; i++) {
                 if (i == 3) {
                     break;
@@ -363,7 +405,6 @@ public class Recover extends Tool implements DataHandler {
                 s.reset();
                 store.seek(i * pageSize);
                 store.readFully(s.getBytes(), 0, pageSize);
-
                 CRC32 crc = new CRC32();
                 crc.update(s.getBytes(), 4, pageSize - 4);
                 int expected = (int) crc.getValue();
@@ -386,8 +427,45 @@ public class Recover extends Tool implements DataHandler {
             writer.println("-- firstTrunkPage: " + logFirstTrunkPage +
                     " firstDataPage: " + logFirstDataPage);
 
-            s = Data.create(this, pageSize);
-            int free = 0;
+            int[] pageTypeCount = new int[Page.TYPE_STREAM_DATA + 2];
+            PrintWriter devNull = new PrintWriter(new OutputStream() {
+                public void write(int b) {
+                    // ignore
+                }
+            });
+            dumpPageStore(devNull, pageCount);
+            Arrays.fill(pageTypeCount, 0);
+            stat = new Stats();
+            schema.clear();
+            objectIdSet = New.hashSet();
+            dumpPageStore(writer, pageCount);
+            writeSchema(writer);
+            try {
+                dumpPageLogStream(writer, logKey, logFirstTrunkPage, logFirstDataPage);
+            } catch (EOFException e) {
+                // ignore
+            }
+            writer.println("---- Statistics ----------");
+            writer.println("-- page count: " + pageCount + " free: " + stat.free);
+            writer.println("-- page data head: " + stat.pageDataHead + " empty: " + stat.pageDataEmpty + " rows: " + stat.pageDataRows);
+            for (int i = 0; i < pageTypeCount.length; i++) {
+                int count = pageTypeCount[i];
+                if (count > 0) {
+                    writer.println("-- page count type: " + i + " " + (100 * count / pageCount) + "% count: " + count);
+                }
+            }
+
+            writer.close();
+        } catch (Throwable e) {
+            writeError(writer, e);
+        } finally {
+            IOUtils.closeSilently(writer);
+            closeSilently(store);
+        }
+    }
+
+    private void dumpPageStore(PrintWriter writer, int pageCount) {
+        Data s = Data.create(this, pageSize);
         for (int page = 3; page < pageCount; page++) {
             s = Data.create(this, pageSize);
             store.seek(page * pageSize);
@@ -395,8 +473,7 @@ public class Recover extends Tool implements DataHandler {
             int type = s.readByte();
             switch (type) {
             case Page.TYPE_EMPTY:
-                    pageTypeCount[type]++;
-                    emptyPages++;
+                stat.pageTypeCount[type]++;
                 continue;
             }
             boolean last = (type & Page.FLAG_LAST) != 0;
@@ -408,7 +485,7 @@ public class Recover extends Tool implements DataHandler {
             switch (type) {
             // type 1
             case Page.TYPE_DATA_LEAF: {
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 int parentPageId = s.readInt();
                 setStorage(s.readVarInt());
                 int columnCount = s.readVarInt();
@@ -420,7 +497,7 @@ public class Recover extends Tool implements DataHandler {
             }
             // type 2
             case Page.TYPE_DATA_NODE: {
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 int parentPageId = s.readInt();
                 setStorage(s.readVarInt());
                 int rowCount = s.readInt();
@@ -432,12 +509,12 @@ public class Recover extends Tool implements DataHandler {
             }
             // type 3
             case Page.TYPE_DATA_OVERFLOW:
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 writer.println("-- page " + page + ": data overflow " + (last ? "(last)" : ""));
                 break;
             // type 4
             case Page.TYPE_BTREE_LEAF: {
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 int parentPageId = s.readInt();
                 setStorage(s.readVarInt());
                 int entries = s.readShortInt();
@@ -450,7 +527,7 @@ public class Recover extends Tool implements DataHandler {
             }
             // type 5
             case Page.TYPE_BTREE_NODE:
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 int parentPageId = s.readInt();
                 setStorage(s.readVarInt());
                 writer.println("-- page " + page + ": b-tree node" + (last ? "(last)" : "") +  " parent: " + parentPageId +
@@ -459,18 +536,18 @@ public class Recover extends Tool implements DataHandler {
                 break;
             // type 6
             case Page.TYPE_FREE_LIST:
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 writer.println("-- page " + page + ": free list " + (last ? "(last)" : ""));
-                    free += dumpPageFreeList(writer, s, page, pageCount);
+                stat.free += dumpPageFreeList(writer, s, page, pageCount);
                 break;
             // type 7
             case Page.TYPE_STREAM_TRUNK:
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 writer.println("-- page " + page + ": log trunk");
                 break;
             // type 8
             case Page.TYPE_STREAM_DATA:
-                    pageTypeCount[type]++;
+                stat.pageTypeCount[type]++;
                 writer.println("-- page " + page + ": log data");
                 break;
             default:
@@ -478,28 +555,6 @@ public class Recover extends Tool implements DataHandler {
                 break;
             }
         }
-            writeSchema(writer);
-            try {
-                dumpPageLogStream(writer, logKey, logFirstTrunkPage, logFirstDataPage);
-            } catch (EOFException e) {
-                // ignore
-            }
-            writer.println("---- Statistics ----------");
-            writer.println("-- page count: " + pageCount + " empty: " + emptyPages + " free: " + free);
-            writer.println("-- page data head: " + pageDataHead + " empty: " + pageDataEmpty + " rows: " + pageDataRows);
-            for (int i = 0; i < pageTypeCount.length; i++) {
-                int count = pageTypeCount[i];
-                if (count > 0) {
-                    writer.println("-- page count type: " + i + " " + (100 * count / pageCount) + "% count: " + count);
-                }
-            }
-            writer.close();
-        } catch (Throwable e) {
-            writeError(writer, e);
-        } finally {
-            IOUtils.closeSilently(writer);
-            closeSilently(store);
-        }
     }
 
     private void dumpPageLogStream(PrintWriter writer, int logKey, int logFirstTrunkPage, int logFirstDataPage) throws IOException {
@@ -906,10 +961,10 @@ public class Recover extends Tool implements DataHandler {
             empty = Math.min(off, empty);
             offsets[i] = off;
         }
-        pageDataRows += pageSize - empty;
+        stat.pageDataRows += pageSize - empty;
         empty = empty - s.length();
-        pageDataHead += s.length();
-        pageDataEmpty += empty;
+        stat.pageDataHead += s.length();
+        stat.pageDataEmpty += empty;
         if (trace) {
             writer.println("--   empty: " + empty);
         }
@@ -1020,7 +1075,8 @@ public class Recover extends Tool implements DataHandler {
                 if (valueId > 0) {
                     sb.append(", ");
                 }
-                sb.append(getSQL(v));
+                String columnName = storageName + "." + valueId;
+                sb.append(getSQL(columnName, v));
             } catch (Exception e) {
                 writeDataError(writer, "exception " + e, s.getBytes());
                 continue;
@@ -1051,6 +1107,7 @@ public class Recover extends Tool implements DataHandler {
         schema = New.arrayList();
         objectIdSet = New.hashSet();
         tableMap = New.hashMap();
+        columnTypeMap = New.hashMap();
     }
 
     private void writeSchema(PrintWriter writer) {
@@ -1063,11 +1120,31 @@ public class Recover extends Tool implements DataHandler {
                 writer.println(sql + ";");
             }
         }
+        // first, copy the lob storage (if there is any)
+        // must occur before copying data,
+        // otherwise the lob storage may be overwritten
         for (Map.Entry<Integer, String> entry : tableMap.entrySet()) {
             Integer objectId = entry.getKey();
             String name = entry.getValue();
             if (objectIdSet.contains(objectId)) {
+                if (name.startsWith("INFORMATION_SCHEMA.LOB")) {
                     setStorage(objectId);
+                    writer.println("DELETE FROM " + name + ";");
+                    writer.println("INSERT INTO " + name + " SELECT * FROM " + storageName + ";");
+                    if (name.startsWith("INFORMATION_SCHEMA.LOBS")) {
+                        writer.println("UPDATE " + name + " SET TABLE = " + LobStorage.TABLE_TEMP + ";");
+                    }
+                }
+            }
+        }
+        for (Map.Entry<Integer, String> entry : tableMap.entrySet()) {
+            Integer objectId = entry.getKey();
+            String name = entry.getValue();
+            if (objectIdSet.contains(objectId)) {
+                setStorage(objectId);
+                if (name.startsWith("INFORMATION_SCHEMA.LOB")) {
+                    continue;
+                }
                 writer.println("INSERT INTO " + name + " SELECT * FROM " + storageName + ";");
             }
         }
@@ -1075,10 +1152,11 @@ public class Recover extends Tool implements DataHandler {
             setStorage(objectId);
             writer.println("DROP TABLE " + storageName + ";");
         }
-        writer.println("DROP ALIAS READ_CLOB;");
         writer.println("DROP ALIAS READ_BLOB;");
-        writer.println("DROP ALIAS READ_CLOB_DB;");
+        writer.println("DROP ALIAS READ_CLOB;");
         writer.println("DROP ALIAS READ_BLOB_DB;");
+        writer.println("DROP ALIAS READ_CLOB_DB;");
+        writer.println("DELETE FROM INFORMATION_SCHEMA.LOBS WHERE TABLE = " + LobStorage.TABLE_TEMP + ";");
         for (MetaRecord m : schema) {
             String sql = m.getSQL();
             // everything except create
@@ -1095,7 +1173,13 @@ public class Recover extends Tool implements DataHandler {
             buff.append(storageName).append('(');
             for (int i = 0; i < recordLength; i++) {
                 buff.appendExceptFirst(", ");
-                buff.append('C').append(i).append(" VARCHAR");
+                buff.append('C').append(i).append(' ');
+                String columnType = columnTypeMap.get(storageName + "." + i);
+                if (columnType == null) {
+                    buff.append("VARCHAR");
+                } else {
+                    buff.append(columnType);
+                }
             }
             writer.println(buff.append(");").toString());
             writer.flush();
@@ -1119,6 +1203,9 @@ public class Recover extends Tool implements DataHandler {
         } else {
             return "UNKNOWN";
         }
+        if (sql.startsWith("IF NOT EXISTS ")) {
+            sql = sql.substring("IF NOT EXISTS ".length());
+        }
         boolean ignore = false;
         for (int i = 0; i < sql.length(); i++) {
             char ch = sql.charAt(i);

h2/src/main/org/h2/value/DataType.java

@@ -770,6 +770,10 @@ public class DataType {
         }
         if (ResultSet.class.isAssignableFrom(x)) {
             return Value.RESULT_SET;
+        } else if (Value.ValueBlob.class.isAssignableFrom(x)) {
+            return Value.BLOB;
+        } else if (Value.ValueClob.class.isAssignableFrom(x)) {
+            return Value.CLOB;
         } else if (String.class.isAssignableFrom(x)) {
             return Value.STRING;
         } else if (BigDecimal.class.isAssignableFrom(x)) {
@@ -836,6 +840,12 @@ public class DataType {
         }
         if (x instanceof String) {
             return ValueString.get((String) x);
+        } else if (x instanceof Value) {
+            return (Value) x;
+        } else if (x instanceof Long) {
+            return ValueLong.get(((Long) x).longValue());
+        } else if (x instanceof Integer) {
+            return ValueInt.get(((Integer) x).intValue());
         } else if (x instanceof BigDecimal) {
             return ValueDecimal.get((BigDecimal) x);
         } else if (x instanceof Boolean) {
@@ -844,10 +854,6 @@ public class DataType {
             return ValueByte.get(((Byte) x).byteValue());
         } else if (x instanceof Short) {
             return ValueShort.get(((Short) x).shortValue());
-        } else if (x instanceof Integer) {
-            return ValueInt.get(((Integer) x).intValue());
-        } else if (x instanceof Long) {
-            return ValueLong.get(((Long) x).longValue());
         } else if (x instanceof Float) {
             return ValueFloat.get(((Float) x).floatValue());
         } else if (x instanceof Double) {

h2/src/main/org/h2/value/Value.java

@@ -1007,4 +1007,18 @@ public abstract class Value {
         return this;
     }
 
+    /**
+     * A "binary large object".
+     */
+    public interface ValueClob {
+        // this is a marker interface
+    }
+
+    /**
+     * A "character large object".
+     */
+    public interface ValueBlob {
+        // this is a marker interface
+    }
+
 }

h2/src/main/org/h2/value/ValueLob2.java → h2/src/main/org/h2/value/ValueLobDb.java

@@ -29,7 +29,7 @@ import org.h2.util.Utils;
 /**
  * An alternate LOB implementation.
  */
-public class ValueLob2 extends Value {
+public class ValueLobDb extends Value implements Value.ValueClob, Value.ValueBlob {
 
     private final int type;
     private long precision;
@@ -45,7 +45,7 @@ public class ValueLob2 extends Value {
     private FileStore tempFile;
     private String fileName;
 
-    private ValueLob2(int type, LobStorage lobStorage, String fileName, int tableId, long lobId, long precision) {
+    private ValueLobDb(int type, LobStorage lobStorage, String fileName, int tableId, long lobId, long precision) {
         this.type = type;
         this.lobStorage = lobStorage;
         this.fileName = fileName;
@@ -54,7 +54,7 @@ public class ValueLob2 extends Value {
         this.precision = precision;
     }
 
-    private ValueLob2(int type, byte[] small, long precision) {
+    private ValueLobDb(int type, byte[] small, long precision) {
         this.type = type;
         this.small = small;
         this.precision = precision;
@@ -66,12 +66,13 @@ public class ValueLob2 extends Value {
      * @param type the type
      * @param lobStorage the storage
      * @param fileName the file name (may be null)
+     * @param tableId the table id
      * @param id the lob id
      * @param precision the precision (number of bytes / characters)
      * @return the value
      */
-    public static ValueLob2 create(int type, LobStorage lobStorage, String fileName, long id, long precision) {
-        return new ValueLob2(type, lobStorage, fileName, LobStorage.TABLE_ID_SESSION_VARIABLE, id, precision);
+    public static ValueLobDb create(int type, LobStorage lobStorage, String fileName, int tableId, long id, long precision) {
+        return new ValueLobDb(type, lobStorage, fileName, tableId, id, precision);
     }
 
     /**
@@ -82,8 +83,8 @@ public class ValueLob2 extends Value {
      * @param precision the precision
      * @return the lob value
      */
-    public static ValueLob2 createSmallLob(int type, byte[] small, long precision) {
-        return new ValueLob2(type, small, precision);
+    public static ValueLobDb createSmallLob(int type, byte[] small, long precision) {
+        return new ValueLobDb(type, small, precision);
     }
 
     /**
@@ -141,12 +142,21 @@ public class ValueLob2 extends Value {
     public Value link(DataHandler h, int tabId) {
         if (small == null) {
             if (tabId != tableId) {
-                if (tableId != LobStorage.TABLE_ID_SESSION_VARIABLE) {
-                    throw DbException.throwInternalError();
+                if (tableId != LobStorage.TABLE_TEMP) {
+                    return lobStorage.copyLob(type, lobId, tabId, getPrecision());
                 }
                 lobStorage.setTable(lobId, tabId);
                 this.tableId = tabId;
             }
+        } else if (small.length > h.getMaxLengthInplaceLob()) {
+            LobStorage s = h.getLobStorage();
+            Value v;
+            if (type == Value.BLOB) {
+                v = s.createBlob(getInputStream(), getPrecision());
+            } else {
+                v = s.createClob(getReader(), getPrecision());
+            }
+            return v.link(h, tabId);
         }
         return this;
     }
@@ -335,7 +345,7 @@ public class ValueLob2 extends Value {
      *
      * @return the value
      */
-    public ValueLob2 copyToTemp() {
+    public ValueLobDb copyToTemp() {
         return this;
     }
 
@@ -359,7 +369,7 @@ public class ValueLob2 extends Value {
      * @param handler the data handler
      * @return the lob value
      */
-    public static ValueLob2 createTempClob(Reader in, long length, DataHandler handler) {
+    public static ValueLobDb createTempClob(Reader in, long length, DataHandler handler) {
         try {
             boolean compress = handler.getLobCompressionAlgorithm(Value.CLOB) != null;
             long remaining = Long.MAX_VALUE;
@@ -379,9 +389,9 @@ public class ValueLob2 extends Value {
             }
             if (len <= handler.getMaxLengthInplaceLob()) {
                 byte[] small = StringUtils.utf8Encode(new String(buff, 0, len));
-                return ValueLob2.createSmallLob(Value.CLOB, small, len);
+                return ValueLobDb.createSmallLob(Value.CLOB, small, len);
             }
-            ValueLob2 lob = new ValueLob2(Value.CLOB, null, 0);
+            ValueLobDb lob = new ValueLobDb(Value.CLOB, null, 0);
             lob.createTempFromReader(buff, len, in, remaining, handler);
             return lob;
         } catch (IOException e) {
@@ -397,7 +407,7 @@ public class ValueLob2 extends Value {
      * @param handler the data handler
      * @return the lob value
      */
-    public static ValueLob2 createTempBlob(InputStream in, long length, DataHandler handler) {
+    public static ValueLobDb createTempBlob(InputStream in, long length, DataHandler handler) {
         try {
             long remaining = Long.MAX_VALUE;
             boolean compress = handler.getLobCompressionAlgorithm(Value.BLOB) != null;
@@ -416,9 +426,9 @@ public class ValueLob2 extends Value {
             if (len <= handler.getMaxLengthInplaceLob()) {
                 byte[] small = Utils.newBytes(len);
                 System.arraycopy(buff, 0, small, 0, len);
-                return ValueLob2.createSmallLob(Value.BLOB, small, small.length);
+                return ValueLobDb.createSmallLob(Value.BLOB, small, small.length);
             }
-            ValueLob2 lob = new ValueLob2(Value.BLOB, null, 0);
+            ValueLobDb lob = new ValueLobDb(Value.BLOB, null, 0);
             lob.createTempFromStream(buff, len, in, remaining, handler);
             return lob;
         } catch (IOException e) {