Merge remote-tracking branch 'upstream/master' into testing_pagestore

h2/src/main/org/h2/command/dml/Insert.java

@@ -23,6 +23,7 @@ import org.h2.expression.Expression;
 import org.h2.expression.ExpressionColumn;
 import org.h2.expression.Parameter;
 import org.h2.expression.SequenceValue;
+import org.h2.expression.ValueExpression;
 import org.h2.index.Index;
 import org.h2.index.PageDataIndex;
 import org.h2.message.DbException;
@@ -183,7 +184,7 @@ public class Insert extends Prepared implements ResultTarget {
                     try {
                         table.addRow(session, newRow);
                     } catch (DbException de) {
-                        if (handleOnDuplicate(de)) {
+                        if (handleOnDuplicate(de, null)) {
                             // MySQL returns 2 for updated row
                             // TODO: detect no-op change
                             rowNumber++;
@@ -207,9 +208,20 @@ public class Insert extends Prepared implements ResultTarget {
                 while (rows.next()) {
                     generatedKeys.nextRow();
                     Value[] r = rows.currentRow();
-                    Row newRow = addRowImpl(r);
-                    if (newRow != null) {
-                        generatedKeys.confirmRow(newRow);
+                    try {
+                        Row newRow = addRowImpl(r);
+                        if (newRow != null) {
+                            generatedKeys.confirmRow(newRow);
+                        }
+                    } catch (DbException de) {
+                        if (handleOnDuplicate(de, r)) {
+                            // MySQL returns 2 for updated row
+                            // TODO: detect no-op change
+                            rowNumber++;
+                        } else {
+                            // INSERT IGNORE case
+                            rowNumber--;
+                        }
                     }
                 }
                 rows.close();
@@ -365,9 +377,10 @@ public class Insert extends Prepared implements ResultTarget {
 
     /**
      * @param de duplicate key exception
+     * @param currentRow current row values (optional)
      * @return {@code true} if row was updated, {@code false} if row was ignored
      */
-    private boolean handleOnDuplicate(DbException de) {
+    private boolean handleOnDuplicate(DbException de, Value[] currentRow) {
         if (de.getErrorCode() != ErrorCode.DUPLICATE_KEY_1) {
             throw de;
         }
@@ -381,13 +394,20 @@ public class Insert extends Prepared implements ResultTarget {
 
         ArrayList<String> variableNames = new ArrayList<>(
                 duplicateKeyAssignmentMap.size());
-        Expression[] row = list.get(getCurrentRowNumber() - 1);
+        Expression[] row = (currentRow == null) ? list.get(getCurrentRowNumber() - 1) 
+                : new Expression[columns.length];
         for (int i = 0; i < columns.length; i++) {
             String key = table.getSchema().getName() + "." +
                     table.getName() + "." + columns[i].getName();
             variableNames.add(key);
-            session.setVariable(key,
-                    row[i].getValue(session));
+            Value value;
+            if (currentRow != null) {
+                value = currentRow[i];
+                row[i] = ValueExpression.get(value);
+            } else {
+                value = row[i].getValue(session);
+            }
+            session.setVariable(key, value);
         }
 
         StatementBuilder buff = new StatementBuilder("UPDATE ");
@@ -403,7 +423,7 @@ public class Insert extends Prepared implements ResultTarget {
             throw DbException.getUnsupportedException(
                     "Unable to apply ON DUPLICATE KEY UPDATE, no index found!");
         }
-        buff.append(prepareUpdateCondition(foundIndex).getSQL());
+        buff.append(prepareUpdateCondition(foundIndex, row).getSQL());
         String sql = buff.toString();
         Update command = (Update) session.prepare(sql);
         command.setUpdateToCurrentValuesReturnsZero(true);
@@ -418,7 +438,7 @@ public class Insert extends Prepared implements ResultTarget {
         return result;
     }
 
-    private Expression prepareUpdateCondition(Index foundIndex) {
+    private Expression prepareUpdateCondition(Index foundIndex, Expression[] row) {
         // MVPrimaryIndex is playing fast and loose with it's implementation of
         // the Index interface.
         // It returns all of the columns in the table when we call
@@ -440,7 +460,6 @@ public class Insert extends Prepared implements ResultTarget {
             indexedColumns = foundIndex.getColumns();
         }
 
-        Expression[] row = list.get(getCurrentRowNumber() - 1);
         Expression condition = null;
         for (Column column : indexedColumns) {
             ExpressionColumn expr = new ExpressionColumn(session.getDatabase(),

h2/src/main/org/h2/mvstore/db/MVPlainTempResult.java

+/*
+ * Copyright 2004-2018 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.mvstore.db;
+
+import org.h2.engine.Database;
+import org.h2.expression.Expression;
+import org.h2.message.DbException;
+import org.h2.mvstore.Cursor;
+import org.h2.mvstore.MVMap;
+import org.h2.mvstore.MVMap.Builder;
+import org.h2.result.ResultExternal;
+import org.h2.value.Value;
+import org.h2.value.ValueArray;
+import org.h2.value.ValueLong;
+
+/**
+ * Plain temporary result.
+ */
+class MVPlainTempResult extends MVTempResult {
+
+    /**
+     * The type of the values in the main map and keys in the index.
+     */
+    private final ValueDataType valueType;
+
+    /**
+     * Map with identities of rows as keys rows as values.
+     */
+    private final MVMap<ValueLong, ValueArray> map;
+
+    /**
+     * Counter for the identities of rows. A separate counter is used instead of
+     * {@link #rowCount} because rows due to presence of {@link #removeRow(Value[])}
+     * method to ensure that each row will have an own identity.
+     */
+    private long counter;
+
+    /**
+     * Optional index. This index is created only if {@link #contains(Value[])}
+     * method is invoked. Only the root result should have an index if required.
+     */
+    private MVMap<ValueArray, Boolean> index;
+
+    /**
+     * Cursor for the {@link #next()} method.
+     */
+    private Cursor<ValueLong, ValueArray> cursor;
+
+    /**
+     * Creates a shallow copy of the result.
+     *
+     * @param parent
+     *                   parent result
+     */
+    private MVPlainTempResult(MVPlainTempResult parent) {
+        super(parent);
+        this.valueType = null;
+        this.map = parent.map;
+    }
+
+    /**
+     * Creates a new plain temporary result.
+     *
+     * @param database
+     *                        database
+     * @param expressions
+     *                        column expressions
+     */
+    MVPlainTempResult(Database database, Expression[] expressions) {
+        super(database);
+        ValueDataType keyType = new ValueDataType(null, null, null);
+        valueType = new ValueDataType(database.getCompareMode(), database, new int[expressions.length]);
+        Builder<ValueLong, ValueArray> builder = new MVMap.Builder<ValueLong, ValueArray>().keyType(keyType)
+                .valueType(valueType);
+        map = store.openMap("tmp", builder);
+    }
+
+    @Override
+    public int addRow(Value[] values) {
+        assert parent == null && index == null;
+        map.put(ValueLong.get(counter++), ValueArray.get(values));
+        return ++rowCount;
+    }
+
+    @Override
+    public boolean contains(Value[] values) {
+        // Only parent result maintains the index
+        if (parent != null) {
+            return parent.contains(values);
+        }
+        if (index == null) {
+            createIndex();
+        }
+        return index.containsKey(ValueArray.get(values));
+    }
+
+    private void createIndex() {
+        Builder<ValueArray, Boolean> builder = new MVMap.Builder<ValueArray, Boolean>().keyType(valueType);
+        index = store.openMap("idx", builder);
+        Cursor<ValueLong, ValueArray> c = map.cursor(null);
+        while (c.hasNext()) {
+            c.next();
+            index.putIfAbsent(c.getValue(), true);
+        }
+    }
+
+    @Override
+    public synchronized ResultExternal createShallowCopy() {
+        if (parent != null) {
+            return parent.createShallowCopy();
+        }
+        if (closed) {
+            return null;
+        }
+        childCount++;
+        return new MVPlainTempResult(this);
+    }
+
+    @Override
+    public Value[] next() {
+        if (cursor == null) {
+            cursor = map.cursor(null);
+        }
+        if (!cursor.hasNext()) {
+            return null;
+        }
+        cursor.next();
+        return cursor.getValue().getList();
+    }
+
+    @Override
+    public int removeRow(Value[] values) {
+        throw DbException.getUnsupportedException("removeRow()");
+    }
+
+    @Override
+    public void reset() {
+        cursor = null;
+    }
+
+}

h2/src/main/org/h2/mvstore/db/MVSortedTempResult.java

+/*
+ * Copyright 2004-2018 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.mvstore.db;
+
+import java.util.BitSet;
+
+import org.h2.engine.Database;
+import org.h2.expression.Expression;
+import org.h2.mvstore.Cursor;
+import org.h2.mvstore.MVMap;
+import org.h2.mvstore.MVMap.Builder;
+import org.h2.result.ResultExternal;
+import org.h2.result.SortOrder;
+import org.h2.value.Value;
+import org.h2.value.ValueArray;
+
+/**
+ * Sorted temporary result.
+ *
+ * <p>
+ * This result is used for distinct and/or sorted results.
+ * </p>
+ */
+class MVSortedTempResult extends MVTempResult {
+
+    /**
+     * Whether this result is distinct.
+     */
+    private final boolean distinct;
+
+    /**
+     * Mapping of indexes of columns to its positions in the store, or {@code null}
+     * if columns are not reordered.
+     */
+    private final int[] indexes;
+
+    /**
+     * Map with rows as keys and counts of duplicate rows as values. If this map is
+     * distinct all values are 1.
+     */
+    private final MVMap<ValueArray, Long> map;
+
+    /**
+     * Cursor for the {@link #next()} method.
+     */
+    private Cursor<ValueArray, Long> cursor;
+
+    /**
+     * Current value for the {@link #next()} method. Used in non-distinct results
+     * with duplicate rows.
+     */
+    private Value[] current;
+
+    /**
+     * Count of remaining duplicate rows for the {@link #next()} method. Used in
+     * non-distinct results.
+     */
+    private long valueCount;
+
+    /**
+     * Creates a shallow copy of the result.
+     *
+     * @param parent
+     *                   parent result
+     */
+    private MVSortedTempResult(MVSortedTempResult parent) {
+        super(parent);
+        this.distinct = parent.distinct;
+        this.indexes = parent.indexes;
+        this.map = parent.map;
+        this.rowCount = parent.rowCount;
+    }
+
+    /**
+     * Creates a new sorted temporary result.
+     *
+     * @param database
+     *                        database
+     * @param expressions
+     *                        column expressions
+     * @param distinct
+     *                        whether this result should be distinct
+     * @param sort
+     *                        sort order, or {@code null} if this result does not
+     *                        need any sorting
+     */
+    MVSortedTempResult(Database database, Expression[] expressions, boolean distinct, SortOrder sort) {
+        super(database);
+        this.distinct = distinct;
+        int length = expressions.length;
+        int[] sortTypes = new int[length];
+        int[] indexes;
+        if (sort != null) {
+            /*
+             * If sorting is specified we need to reorder columns in requested order and set
+             * sort types (ASC, DESC etc) for them properly.
+             */
+            indexes = new int[length];
+            int[] colIndex = sort.getQueryColumnIndexes();
+            int len = colIndex.length;
+            // This set is used to remember columns that are already included
+            BitSet used = new BitSet();
+            for (int i = 0; i < len; i++) {
+                int idx = colIndex[i];
+                assert !used.get(idx);
+                used.set(idx);
+                indexes[i] = idx;
+                sortTypes[i] = sort.getSortTypes()[i];
+            }
+            /*
+             * Because this result may have more columns than specified in sorting we need
+             * to add all remaining columns to the mapping of columns. A default sorting
+             * order (ASC / 0) will be used for them.
+             */
+            int idx = 0;
+            for (int i = len; i < length; i++) {
+                idx = used.nextClearBit(idx);
+                indexes[i] = idx;
+                idx++;
+            }
+            /*
+             * Sometimes columns may be not reordered. Because reordering of columns
+             * slightly slows down other methods we check whether columns are really
+             * reordered or have the same order.
+             */
+            sameOrder: {
+                for (int i = 0; i < length; i++) {
+                    if (indexes[i] != i) {
+                        // Columns are reordered
+                        break sameOrder;
+                    }
+                }
+                /*
+                 * Columns are not reordered, set this field to null to disable reordering in
+                 * other methods.
+                 */
+                indexes = null;
+            }
+        } else {
+            // Columns are not reordered if sort order is not specified
+            indexes = null;
+        }
+        this.indexes = indexes;
+        ValueDataType keyType = new ValueDataType(database.getCompareMode(), database, sortTypes);
+        Builder<ValueArray, Long> builder = new MVMap.Builder<ValueArray, Long>().keyType(keyType);
+        map = store.openMap("tmp", builder);
+    }
+
+    @Override
+    public int addRow(Value[] values) {
+        assert parent == null;
+        ValueArray key = getKey(values);
+        if (distinct) {
+            // Add a row and increment the counter only if row does not exist
+            if (map.putIfAbsent(key, 1L) == null) {
+                rowCount++;
+            }
+        } else {
+            // Try to set counter to 1 first if such row does not exist yet
+            Long old = map.putIfAbsent(key, 1L);
+            if (old != null) {
+                // This rows is already in the map, increment its own counter
+                map.put(key, old + 1);
+            }
+            rowCount++;
+        }
+        return rowCount;
+    }
+
+    @Override
+    public boolean contains(Value[] values) {
+        return map.containsKey(getKey(values));
+    }
+
+    @Override
+    public synchronized ResultExternal createShallowCopy() {
+        if (parent != null) {
+            return parent.createShallowCopy();
+        }
+        if (closed) {
+            return null;
+        }
+        childCount++;
+        return new MVSortedTempResult(this);
+    }
+
+    /**
+     * Reorder values if required and convert them into {@link ValueArray}.
+     *
+     * @param values
+     *                   values
+     * @return ValueArray for maps
+     */
+    private ValueArray getKey(Value[] values) {
+        if (indexes != null) {
+            Value[] r = new Value[indexes.length];
+            for (int i = 0; i < indexes.length; i++) {
+                r[indexes[i]] = values[i];
+            }
+            values = r;
+        }
+        return ValueArray.get(values);
+    }
+
+    /**
+     * Reorder values back if required.
+     *
+     * @param key
+     *                reordered values
+     * @return original values
+     */
+    private Value[] getValue(Value[] key) {
+        if (indexes != null) {
+            Value[] r = new Value[indexes.length];
+            for (int i = 0; i < indexes.length; i++) {
+                r[i] = key[indexes[i]];
+            }
+            key = r;
+        }
+        return key;
+    }
+
+    @Override
+    public Value[] next() {
+        if (cursor == null) {
+            cursor = map.cursor(null);
+            current = null;
+            valueCount = 0L;
+        }
+        // If we have multiple rows with the same values return them all
+        if (--valueCount > 0) {
+            /*
+             * Underflow in valueCount is hypothetically possible after a lot of invocations
+             * (not really possible in practice), but current will be null anyway.
+             */
+            return current;
+        }
+        if (!cursor.hasNext()) {
+            // Set current to null to be sure
+            current = null;
+            return null;
+        }
+        // Read the next row
+        current = getValue(cursor.next().getList());
+        /*
+         * If valueCount is greater than 1 that is possible for non-distinct results the
+         * following invocations of next() will use this.current and this.valueCount.
+         */
+        valueCount = cursor.getValue();
+        return current;
+    }
+
+    @Override
+    public int removeRow(Value[] values) {
+        assert parent == null;
+        ValueArray key = getKey(values);
+        if (distinct) {
+            // If an entry was removed decrement the counter
+            if (map.remove(key) != null) {
+                rowCount--;
+            }
+        } else {
+            Long old = map.remove(key);
+            if (old != null) {
+                long l = old;
+                if (l > 1) {
+                    /*
+                     * We have more than one such row. Decrement its counter by 1 and put this row
+                     * back into map.
+                     */
+                    map.put(key, l - 1);
+                }
+                rowCount--;
+            }
+        }
+        return rowCount;
+    }
+
+    @Override
+    public void reset() {
+        cursor = null;
+        current = null;
+        valueCount = 0L;
+    }
+
+}

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -75,12 +75,7 @@ public class MVTableEngine implements TableEngine {
             }
             if (key != null) {
                 encrypted = true;
-                char[] password = new char[key.length / 2];
-                for (int i = 0; i < password.length; i++) {
-                    password[i] = (char) (((key[i + i] & 255) << 16) |
-                            ((key[i + i + 1]) & 255));
-                }
-                builder.encryptionKey(password);
+                builder.encryptionKey(decodePassword(key));
             }
             if (db.getSettings().compressData) {
                 builder.compress();
@@ -101,6 +96,15 @@ public class MVTableEngine implements TableEngine {
         return store;
     }
 
+    static char[] decodePassword(byte[] key) {
+        char[] password = new char[key.length / 2];
+        for (int i = 0; i < password.length; i++) {
+            password[i] = (char) (((key[i + i] & 255) << 16) |
+                    ((key[i + i + 1]) & 255));
+        }
+        return password;
+    }
+
     @Override
     public TableBase createTable(CreateTableData data) {
         Database db = data.session.getDatabase();

h2/src/main/org/h2/mvstore/db/MVTempResult.java

+/*
+ * Copyright 2004-2018 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.mvstore.db;
+
+import java.io.IOException;
+import java.lang.ref.Reference;
+import java.util.ArrayList;
+
+import org.h2.engine.Constants;
+import org.h2.engine.Database;
+import org.h2.expression.Expression;
+import org.h2.message.DbException;
+import org.h2.mvstore.MVStore;
+import org.h2.mvstore.MVStore.Builder;
+import org.h2.result.ResultExternal;
+import org.h2.result.SortOrder;
+import org.h2.store.fs.FileUtils;
+import org.h2.util.TempFileDeleter;
+import org.h2.value.Value;
+
+/**
+ * Temporary result.
+ *
+ * <p>
+ * A separate MVStore in a temporary file is used for each result. The file is
+ * removed when this result and all its copies are closed.
+ * {@link TempFileDeleter} is also used to delete this file if results are not
+ * closed properly.
+ * </p>
+ */
+public abstract class MVTempResult implements ResultExternal {
+
+    private static final class CloseImpl implements AutoCloseable {
+        /**
+         * MVStore.
+         */
+        private final MVStore store;
+
+        /**
+         * File name.
+         */
+        private final String fileName;
+
+        CloseImpl(MVStore store, String fileName) {
+            this.store = store;
+            this.fileName = fileName;
+        }
+
+        @Override
+        public void close() throws Exception {
+            store.closeImmediately();
+            FileUtils.tryDelete(fileName);
+        }
+
+    }
+
+    /**
+     * Creates MVStore-based temporary result.
+     *
+     * @param database
+     *                        database
+     * @param expressions
+     *                        expressions
+     * @param distinct
+     *                        is output distinct
+     * @param sort
+     *                        sort order, or {@code null}
+     * @return temporary result
+     */
+    public static ResultExternal of(Database database, Expression[] expressions, boolean distinct, SortOrder sort) {
+        return distinct || sort != null ? new MVSortedTempResult(database, expressions, distinct, sort)
+                : new MVPlainTempResult(database, expressions);
+    }
+
+    /**
+     * MVStore.
+     */
+    final MVStore store;
+
+    /**
+     * Count of rows. Used only in a root results, copies always have 0 value.
+     */
+    int rowCount;
+
+    /**
+     * Parent store for copies. If {@code null} this result is a root result.
+     */
+    final MVTempResult parent;
+
+    /**
+     * Count of child results.
+     */
+    int childCount;
+
+    /**
+     * Whether this result is closed.
+     */
+    boolean closed;
+
+    /**
+     * Temporary file deleter.
+     */
+    private final TempFileDeleter tempFileDeleter;
+
+    /**
+     * Closeable to close the storage.
+     */
+    private final CloseImpl closeable;
+
+    /**
+     * Reference to the record in the temporary file deleter.
+     */
+    private final Reference<?> fileRef;
+
+    /**
+     * Creates a shallow copy of the result.
+     *
+     * @param parent
+     *                   parent result
+     */
+    MVTempResult(MVTempResult parent) {
+        this.parent = parent;
+        this.store = parent.store;
+        this.tempFileDeleter = null;
+        this.closeable = null;
+        this.fileRef = null;
+    }
+
+    /**
+     * Creates a new temporary result.
+     *
+     * @param database
+     *                     database
+     */
+    MVTempResult(Database database) {
+        try {
+            String fileName = FileUtils.createTempFile("h2tmp", Constants.SUFFIX_TEMP_FILE, false, true);
+            Builder builder = new MVStore.Builder().fileName(fileName);
+            byte[] key = database.getFileEncryptionKey();
+            if (key != null) {
+                builder.encryptionKey(MVTableEngine.decodePassword(key));
+            }
+            store = builder.open();
+            tempFileDeleter = database.getTempFileDeleter();
+            closeable = new CloseImpl(store, fileName);
+            fileRef = tempFileDeleter.addFile(closeable, this);
+        } catch (IOException e) {
+            throw DbException.convert(e);
+        }
+        parent = null;
+    }
+
+    @Override
+    public int addRows(ArrayList<Value[]> rows) {
+        for (Value[] row : rows) {
+            addRow(row);
+        }
+        return rowCount;
+    }
+
+    @Override
+    public synchronized void close() {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        if (parent != null) {
+            parent.closeChild();
+        } else {
+            if (childCount == 0) {
+                delete();
+            }
+        }
+    }
+
+    private synchronized void closeChild() {
+        if (--childCount == 0 && closed) {
+            delete();
+        }
+    }
+
+    private void delete() {
+        tempFileDeleter.deleteFile(fileRef, closeable);
+    }
+
+    @Override
+    public void done() {
+        // Do nothing
+    }
+
+}

h2/src/main/org/h2/result/LocalResult.java

@@ -15,6 +15,7 @@ import org.h2.engine.Session;
 import org.h2.engine.SessionInterface;
 import org.h2.expression.Expression;
 import org.h2.message.DbException;
+import org.h2.mvstore.db.MVTempResult;
 import org.h2.util.Utils;
 import org.h2.util.ValueHashMap;
 import org.h2.value.DataType;
@@ -289,6 +290,13 @@ public class LocalResult implements ResultInterface, ResultTarget {
         return ValueArray.get(values);
     }
 
+    private void createExternalResult() {
+        Database database = session.getDatabase();
+        external = database.getMvStore() != null
+                ? MVTempResult.of(database, expressions, distinct, sort)
+                        : new ResultTempTable(session, expressions, distinct, sort);
+    }
+
     /**
      * Add a row to this object.
      *
@@ -303,7 +311,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
                 distinctRows.put(array, values);
                 rowCount = distinctRows.size();
                 if (rowCount > maxMemoryRows) {
-                    external = new ResultTempTable(session, expressions, true, sort);
+                    createExternalResult();
                     rowCount = external.addRows(distinctRows.values());
                     distinctRows = null;
                 }
@@ -316,7 +324,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
         rowCount++;
         if (rows.size() > maxMemoryRows) {
             if (external == null) {
-                external = new ResultTempTable(session, expressions, false, sort);
+                createExternalResult();
             }
             addRowsToDisk();
         }
@@ -353,7 +361,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
                             break;
                         }
                         if (external == null) {
-                            external = new ResultTempTable(session, expressions, true, sort);
+                            createExternalResult();
                         }
                         rows.add(list);
                         if (rows.size() > maxMemoryRows) {

h2/src/main/org/h2/result/ResultTempTable.java

@@ -7,6 +7,8 @@ package org.h2.result;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
+
 import org.h2.command.ddl.CreateTableData;
 import org.h2.engine.Constants;
 import org.h2.engine.Database;
@@ -88,26 +90,38 @@ public class ResultTempTable implements ResultExternal {
     }
 
     private void createIndex() {
-        IndexColumn[] indexCols = null;
-        // If we need to do distinct, the distinct columns may not match the
-        // sort columns. So we need to disregard the sort. Not ideal.
-        if (sort != null && !distinct) {
+        IndexColumn[] indexCols;
+        if (sort != null) {
             int[] colIndex = sort.getQueryColumnIndexes();
-            indexCols = new IndexColumn[colIndex.length];
-            for (int i = 0; i < colIndex.length; i++) {
-                IndexColumn indexColumn = new IndexColumn();
-                indexColumn.column = table.getColumn(colIndex[i]);
-                indexColumn.sortType = sort.getSortTypes()[i];
-                indexColumn.columnName = COLUMN_NAME + i;
-                indexCols[i] = indexColumn;
+            int len = colIndex.length;
+            if (distinct) {
+                BitSet used = new BitSet();
+                indexCols = new IndexColumn[columnCount];
+                for (int i = 0; i < len; i++) {
+                    int idx = colIndex[i];
+                    used.set(idx);
+                    IndexColumn indexColumn = createIndexColumn(idx);
+                    indexColumn.sortType = sort.getSortTypes()[i];
+                    indexCols[i] = indexColumn;
+                }
+                int idx = 0;
+                for (int i = len; i < columnCount; i++) {
+                    idx = used.nextClearBit(idx);
+                    indexCols[i] = createIndexColumn(idx);
+                    idx++;
+                }
+            } else {
+                indexCols = new IndexColumn[len];
+                for (int i = 0; i < len; i++) {
+                    IndexColumn indexColumn = createIndexColumn(colIndex[i]);
+                    indexColumn.sortType = sort.getSortTypes()[i];
+                    indexCols[i] = indexColumn;
+                }
             }
         } else {
             indexCols = new IndexColumn[columnCount];
             for (int i = 0; i < columnCount; i++) {
-                IndexColumn indexColumn = new IndexColumn();
-                indexColumn.column = table.getColumn(i);
-                indexColumn.columnName = COLUMN_NAME + i;
-                indexCols[i] = indexColumn;
+                indexCols[i] = createIndexColumn(i);
             }
         }
         String indexName = table.getSchema().getUniqueIndexName(session,
@@ -118,6 +132,13 @@ public class ResultTempTable implements ResultExternal {
                 indexType, true, null);
     }
 
+    private IndexColumn createIndexColumn(int index) {
+        IndexColumn indexColumn = new IndexColumn();
+        indexColumn.column = table.getColumn(index);
+        indexColumn.columnName = COLUMN_NAME + index;
+        return indexColumn;
+    }
+
     @Override
     public synchronized ResultExternal createShallowCopy() {
         if (parent != null) {
@@ -257,19 +278,7 @@ public class ResultTempTable implements ResultExternal {
             } else {
                 idx = table.getScanIndex(session);
             }
-            if (session.getDatabase().getMvStore() != null) {
-                // sometimes the transaction is already committed,
-                // in which case we can't use the session
-                if (idx.getRowCount(session) == 0 && rowCount > 0) {
-                    // this means querying is not transactional
-                    resultCursor = idx.find((Session) null, null, null);
-                } else {
-                    // the transaction is still open
-                    resultCursor = idx.find(session, null, null);
-                }
-            } else {
-                resultCursor = idx.find(session, null, null);
-            }
+            resultCursor = idx.find(session, null, null);
         }
         if (!resultCursor.next()) {
             return null;

h2/src/main/org/h2/util/TempFileDeleter.java

@@ -21,7 +21,7 @@ import org.h2.store.fs.FileUtils;
 public class TempFileDeleter {
 
     private final ReferenceQueue<Object> queue = new ReferenceQueue<>();
-    private final HashMap<PhantomReference<?>, String> refMap = new HashMap<>();
+    private final HashMap<PhantomReference<?>, Object> refMap = new HashMap<>();
 
     private TempFileDeleter() {
         // utility class
@@ -32,43 +32,59 @@ public class TempFileDeleter {
     }
 
     /**
-     * Add a file to the list of temp files to delete. The file is deleted once
-     * the file object is garbage collected.
+     * Add a file or a closeable to the list of temporary objects to delete. The
+     * file is deleted once the file object is garbage collected.
      *
-     * @param fileName the file name
-     * @param file the object to monitor
-     * @return the reference that can be used to stop deleting the file
+     * @param resource the file name or the closeable
+     * @param monitor the object to monitor
+     * @return the reference that can be used to stop deleting the file or closing the closeable
      */
-    public synchronized Reference<?> addFile(String fileName, Object file) {
-        IOUtils.trace("TempFileDeleter.addFile", fileName, file);
-        PhantomReference<?> ref = new PhantomReference<>(file, queue);
-        refMap.put(ref, fileName);
+    public synchronized Reference<?> addFile(Object resource, Object monitor) {
+        if (!(resource instanceof String) && !(resource instanceof AutoCloseable)) {
+            throw DbException.getUnsupportedException("Unsupported resource " + resource);
+        }
+        IOUtils.trace("TempFileDeleter.addFile",
+                resource instanceof String ? (String) resource : "-", monitor);
+        PhantomReference<?> ref = new PhantomReference<>(monitor, queue);
+        refMap.put(ref, resource);
         deleteUnused();
         return ref;
     }
 
     /**
-     * Delete the given file now. This will remove the reference from the list.
+     * Delete the given file or close the closeable now. This will remove the
+     * reference from the list.
      *
      * @param ref the reference as returned by addFile
-     * @param fileName the file name
+     * @param resource the file name or closeable
      */
-    public synchronized void deleteFile(Reference<?> ref, String fileName) {
+    public synchronized void deleteFile(Reference<?> ref, Object resource) {
         if (ref != null) {
-            String f2 = refMap.remove(ref);
+            Object f2 = refMap.remove(ref);
             if (f2 != null) {
                 if (SysProperties.CHECK) {
-                    if (fileName != null && !f2.equals(fileName)) {
-                        DbException.throwInternalError("f2:" + f2 + " f:" + fileName);
+                    if (resource != null && !f2.equals(resource)) {
+                        DbException.throwInternalError("f2:" + f2 + " f:" + resource);
                     }
                 }
-                fileName = f2;
+                resource = f2;
             }
         }
-        if (fileName != null && FileUtils.exists(fileName)) {
+        if (resource instanceof String) {
+            String fileName = (String) resource;
+            if (FileUtils.exists(fileName)) {
+                try {
+                    IOUtils.trace("TempFileDeleter.deleteFile", fileName, null);
+                    FileUtils.tryDelete(fileName);
+                } catch (Exception e) {
+                    // TODO log such errors?
+                }
+            }
+        } else if (resource instanceof AutoCloseable) {
+            AutoCloseable closeable = (AutoCloseable) resource;
             try {
-                IOUtils.trace("TempFileDeleter.deleteFile", fileName, null);
-                FileUtils.tryDelete(fileName);
+                IOUtils.trace("TempFileDeleter.deleteCloseable", "-", null);
+                closeable.close();
             } catch (Exception e) {
                 // TODO log such errors?
             }
@@ -76,17 +92,17 @@ public class TempFileDeleter {
     }
 
     /**
-     * Delete all registered temp files.
+     * Delete all registered temp resources.
      */
     public void deleteAll() {
-        for (String tempFile : new ArrayList<>(refMap.values())) {
-            deleteFile(null, tempFile);
+        for (Object resource : new ArrayList<>(refMap.values())) {
+            deleteFile(null, resource);
         }
         deleteUnused();
     }
 
     /**
-     * Delete all unused files now.
+     * Delete all unused resources now.
      */
     public void deleteUnused() {
         while (queue != null) {
@@ -99,20 +115,21 @@ public class TempFileDeleter {
     }
 
     /**
-     * This method is called if a file should no longer be deleted if the object
-     * is garbage collected.
+     * This method is called if a file should no longer be deleted or a resource
+     * should no longer be closed if the object is garbage collected.
      *
      * @param ref the reference as returned by addFile
-     * @param fileName the file name
+     * @param resource file name or closeable
      */
-    public void stopAutoDelete(Reference<?> ref, String fileName) {
-        IOUtils.trace("TempFileDeleter.stopAutoDelete", fileName, ref);
+    public void stopAutoDelete(Reference<?> ref, Object resource) {
+        IOUtils.trace("TempFileDeleter.stopAutoDelete",
+                resource instanceof String ? (String) resource : "-", ref);
         if (ref != null) {
-            String f2 = refMap.remove(ref);
+            Object f2 = refMap.remove(ref);
             if (SysProperties.CHECK) {
-                if (f2 == null || !f2.equals(fileName)) {
+                if (f2 == null || !f2.equals(resource)) {
                     DbException.throwInternalError("f2:" + f2 +
-                            " " + (f2 == null ? "" : f2) + " f:" + fileName);
+                            " " + (f2 == null ? "" : f2) + " f:" + resource);
                 }
             }
         }

h2/src/test/org/h2/test/db/TestBigResult.java

@@ -5,12 +5,17 @@
  */
 package org.h2.test.db;
 
+import java.sql.Blob;
+import java.sql.Clob;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.sql.Types;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
 
 import org.h2.store.FileLister;
 import org.h2.test.TestBase;
@@ -35,6 +40,8 @@ public class TestBigResult extends TestBase {
             return;
         }
         testLargeSubquery();
+        testSortingAndDistinct();
+        testLOB();
         testLargeUpdateDelete();
         testCloseConnectionDelete();
         testOrderGroup();
@@ -66,6 +73,286 @@ public class TestBigResult extends TestBase {
         conn.close();
     }
 
+    private void testSortingAndDistinct() throws SQLException {
+        deleteDb("bigResult");
+        Connection conn = getConnection("bigResult");
+        Statement stat = conn.createStatement();
+        int count = getSize(1000, 4000);
+        stat.execute("CREATE TABLE TEST(ID INT PRIMARY KEY, VALUE INT NOT NULL)");
+        PreparedStatement ps = conn.prepareStatement("INSERT INTO TEST VALUES (?, ?)");
+        for (int i = 0; i < count; i++) {
+            ps.setInt(1, i);
+            ps.setInt(2, count - i);
+            ps.executeUpdate();
+        }
+        // local result
+        testSortintAndDistinct1(stat, count, count);
+        // external result
+        testSortintAndDistinct1(stat, 10, count);
+        stat.execute("DROP TABLE TEST");
+        stat.execute("CREATE TABLE TEST(ID INT PRIMARY KEY, VALUE1 INT NOT NULL, VALUE2 INT NOT NULL)");
+        ps = conn.prepareStatement("INSERT INTO TEST VALUES (?, ?, ?)");
+        int partCount = count / 10;
+        for (int i = 0; i < count; i++) {
+            ps.setInt(1, i);
+            int a = i / 10;
+            int b = i % 10;
+            ps.setInt(2, partCount - a);
+            ps.setInt(3, 10 - b);
+            ps.executeUpdate();
+        }
+        String sql;
+        /*
+         * Sorting only
+         */
+        sql = "SELECT VALUE2, VALUE1 FROM (SELECT ID, VALUE2, VALUE1 FROM TEST ORDER BY VALUE2)";
+        // local result
+        testSortingAndDistinct2(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct2(stat, sql, 10, partCount);
+        /*
+         * Distinct only
+         */
+        sql = "SELECT VALUE2, VALUE1 FROM (SELECT DISTINCT ID, VALUE2, VALUE1 FROM TEST)";
+        // local result
+        testSortingAndDistinct2DistinctOnly(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct2DistinctOnly(stat, sql, 10, partCount);
+        /*
+         * Sorting and distinct
+         */
+        sql = "SELECT VALUE2, VALUE1 FROM (SELECT DISTINCT ID, VALUE2, VALUE1 FROM TEST ORDER BY VALUE2)";
+        // local result
+        testSortingAndDistinct2(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct2(stat, sql, 10, partCount);
+        /*
+         * One more distinct only
+         */
+        sql = "SELECT VALUE1 FROM (SELECT DISTINCT VALUE1 FROM TEST)";
+        // local result
+        testSortingAndDistinct3DistinctOnly(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct3DistinctOnly(stat, sql, 1, partCount);
+        /*
+         * One more sorting and distinct
+         */
+        sql = "SELECT VALUE1 FROM (SELECT DISTINCT VALUE1 FROM TEST ORDER BY VALUE1)";
+        // local result
+        testSortingAndDistinct3(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct3(stat, sql, 1, partCount);
+        stat.execute("DROP TABLE TEST");
+        stat.execute("CREATE TABLE TEST(ID INT PRIMARY KEY, VALUE INT)");
+        ps = conn.prepareStatement("INSERT INTO TEST VALUES (?, ?)");
+        for (int i = 0; i < count; i++) {
+            ps.setInt(1, i);
+            int j = i / 10;
+            if (j == 0) {
+                ps.setNull(2, Types.INTEGER);
+            } else {
+                ps.setInt(2, j);
+            }
+            ps.executeUpdate();
+        }
+        /*
+         * Sorting and distinct
+         */
+        sql = "SELECT DISTINCT VALUE FROM TEST ORDER BY VALUE";
+        // local result
+        testSortingAndDistinct4(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct4(stat, sql, 1, partCount);
+        /*
+         * Distinct only
+         */
+        sql = "SELECT DISTINCT VALUE FROM TEST";
+        // local result
+        testSortingAndDistinct4DistinctOnly(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct4DistinctOnly(stat, sql, 1, partCount);
+        /*
+         * Sorting only
+         */
+        sql = "SELECT VALUE FROM TEST ORDER BY VALUE";
+        // local result
+        testSortingAndDistinct4SortingOnly(stat, sql, count, partCount);
+        // external result
+        testSortingAndDistinct4SortingOnly(stat, sql, 1, partCount);
+        conn.close();
+    }
+
+    private void testSortintAndDistinct1(Statement stat, int maxRows, int count) throws SQLException {
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        ResultSet rs = stat.executeQuery("SELECT VALUE FROM (SELECT DISTINCT ID, VALUE FROM TEST ORDER BY VALUE)");
+        for (int i = 1; i <= count; i++) {
+            assertTrue(rs.next());
+            assertEquals(rs.getInt(1), i);
+        }
+        assertFalse(rs.next());
+    }
+
+    private void testSortingAndDistinct2(Statement stat, String sql, int maxRows, int partCount) throws SQLException {
+        ResultSet rs;
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        rs = stat.executeQuery(sql);
+        BitSet set = new BitSet(partCount);
+        for (int i = 1; i <= 10; i++) {
+            set.clear();
+            for (int j = 1; j <= partCount; j++) {
+                assertTrue(rs.next());
+                assertEquals(i, rs.getInt(1));
+                set.set(rs.getInt(2));
+            }
+            assertEquals(partCount + 1, set.nextClearBit(1));
+        }
+        assertFalse(rs.next());
+    }
+
+    private void testSortingAndDistinct2DistinctOnly(Statement stat, String sql, int maxRows, int partCount) throws SQLException {
+        ResultSet rs;
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        rs = stat.executeQuery(sql);
+        BitSet set = new BitSet(partCount * 10);
+        for (int i = 1; i <= 10; i++) {
+            for (int j = 1; j <= partCount; j++) {
+                assertTrue(rs.next());
+                set.set(rs.getInt(1) * partCount + rs.getInt(2));
+            }
+        }
+        assertEquals(partCount * 11 + 1, set.nextClearBit(partCount + 1));
+        assertFalse(rs.next());
+    }
+
+    private void testSortingAndDistinct3(Statement stat, String sql, int maxRows, int partCount) throws SQLException {
+        ResultSet rs;
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        rs = stat.executeQuery(sql);
+        for (int i = 1; i <= partCount; i++) {
+            assertTrue(rs.next());
+            assertEquals(i, rs.getInt(1));
+        }
+        assertFalse(rs.next());
+    }
+
+    private void testSortingAndDistinct3DistinctOnly(Statement stat, String sql, int maxRows, int partCount) throws SQLException {
+        ResultSet rs;
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        rs = stat.executeQuery(sql);
+        BitSet set = new BitSet(partCount);
+        for (int i = 1; i <= partCount; i++) {
+            assertTrue(rs.next());
+            set.set(rs.getInt(1));
+        }
+        assertEquals(partCount + 1, set.nextClearBit(1));
+        assertFalse(rs.next());
+    }
+
+    private void testSortingAndDistinct4(Statement stat, String sql, int maxRows, int count) throws SQLException {
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        ResultSet rs = stat.executeQuery(sql);
+        for (int i = 0; i < count; i++) {
+            assertTrue(rs.next());
+            assertEquals(i, rs.getInt(1));
+            if (i == 0) {
+                assertTrue(rs.wasNull());
+            }
+        }
+        assertFalse(rs.next());
+    }
+
+    private void testSortingAndDistinct4DistinctOnly(Statement stat, String sql, int maxRows, int count) throws SQLException {
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        ResultSet rs = stat.executeQuery(sql);
+        BitSet set = new BitSet();
+        for (int i = 0; i < count; i++) {
+            assertTrue(rs.next());
+            int v = rs.getInt(1);
+            if (v == 0) {
+                assertTrue(rs.wasNull());
+            }
+            assertFalse(set.get(v));
+            set.set(v);
+        }
+        assertFalse(rs.next());
+        assertEquals(count, set.nextClearBit(0));
+    }
+
+    private void testSortingAndDistinct4SortingOnly(Statement stat, String sql, int maxRows, int count) throws SQLException {
+        stat.execute("SET MAX_MEMORY_ROWS " + maxRows);
+        ResultSet rs = stat.executeQuery(sql);
+        for (int i = 0; i < count; i++) {
+            for (int j = 0; j < 10; j++) {
+                assertTrue(rs.next());
+                assertEquals(i, rs.getInt(1));
+                if (i == 0) {
+                    assertTrue(rs.wasNull());
+                }
+            }
+        }
+        assertFalse(rs.next());
+    }
+
+    private void testLOB() throws SQLException {
+        deleteDb("bigResult");
+        Connection conn = getConnection("bigResult");
+        Statement stat = conn.createStatement();
+        stat.execute("SET MAX_MEMORY_ROWS " + 1);
+        stat.execute("CREATE TABLE TEST(ID INT PRIMARY KEY, VALUE BLOB NOT NULL)");
+        PreparedStatement ps = conn.prepareStatement("INSERT INTO TEST VALUES (?, ?)");
+        int length = 1_000_000;
+        byte[] data = new byte[length];
+        for (int i = 1; i <= 10; i++) {
+            ps.setInt(1, i);
+            Arrays.fill(data, (byte) i);
+            ps.setBytes(2, data);
+            ps.executeUpdate();
+        }
+        Blob[] blobs = new Blob[10];
+        ResultSet rs = stat.executeQuery("SELECT * FROM TEST");
+        for (int i = 1; i <= 10; i++) {
+            assertTrue(rs.next());
+            assertEquals(i, rs.getInt(1));
+            blobs[i - 1] = rs.getBlob(2);
+        }
+        assertFalse(rs.next());
+        rs.close();
+        for (int i = 1; i <= 10; i++) {
+            Blob b = blobs[i - 1];
+            byte[] bytes = b.getBytes(1, (int) b.length());
+            Arrays.fill(data, (byte) i);
+            assertEquals(data, bytes);
+            b.free();
+        }
+        stat.execute("DROP TABLE TEST");
+        stat.execute("CREATE TABLE TEST(ID INT PRIMARY KEY, VALUE CLOB NOT NULL)");
+        ps = conn.prepareStatement("INSERT INTO TEST VALUES (?, ?)");
+        char[] cdata = new char[length];
+        for (int i = 1; i <= 10; i++) {
+            ps.setInt(1, i);
+            Arrays.fill(cdata, (char) i);
+            ps.setString(2, new String(cdata));
+            ps.executeUpdate();
+        }
+        Clob[] clobs = new Clob[10];
+        rs = stat.executeQuery("SELECT * FROM TEST");
+        for (int i = 1; i <= 10; i++) {
+            assertTrue(rs.next());
+            assertEquals(i, rs.getInt(1));
+            clobs[i - 1] = rs.getClob(2);
+        }
+        assertFalse(rs.next());
+        rs.close();
+        for (int i = 1; i <= 10; i++) {
+            Clob c = clobs[i - 1];
+            String string = c.getSubString(1, (int) c.length());
+            Arrays.fill(cdata, (char) i);
+            assertEquals(new String(cdata), string);
+            c.free();
+        }
+        conn.close();
+    }
+
     private void testLargeUpdateDelete() throws SQLException {
         deleteDb("bigResult");
         Connection conn = getConnection("bigResult");

h2/src/test/org/h2/test/scripts/dml/insertIgnore.sql

@@ -39,3 +39,60 @@ SELECT * FROM TEST ORDER BY ID;
 > 4  40
 > 5  52
 > rows (ordered): 5
+
+CREATE TABLE TESTREF(ID BIGINT PRIMARY KEY, VALUE INT NOT NULL);
+> ok
+
+INSERT INTO TESTREF VALUES (1, 11), (2, 21), (6, 61), (7, 71);
+> update count: 4
+
+INSERT INTO TEST (ID, VALUE) SELECT ID, VALUE FROM TESTREF;
+> exception DUPLICATE_KEY_1
+
+SELECT * FROM TEST ORDER BY ID;
+> ID VALUE
+> -- -----
+> 1  10
+> 2  20
+> 3  30
+> 4  40
+> 5  52
+> rows (ordered): 5
+
+INSERT IGNORE INTO TEST (ID, VALUE) SELECT ID, VALUE FROM TESTREF;
+> update count: 2
+
+INSERT IGNORE INTO TEST (ID, VALUE) SELECT ID, VALUE FROM TESTREF;
+> ok
+
+SELECT * FROM TEST ORDER BY ID;
+> ID VALUE
+> -- -----
+> 1  10
+> 2  20
+> 3  30
+> 4  40
+> 5  52
+> 6  61
+> 7  71
+> rows (ordered): 7
+
+INSERT INTO TESTREF VALUES (8, 81), (9, 91);
+> update count: 2
+
+INSERT INTO TEST (ID, VALUE) SELECT ID, VALUE FROM TESTREF ON DUPLICATE KEY UPDATE VALUE=83;
+> update count: 10
+
+SELECT * FROM TEST ORDER BY ID;
+> ID VALUE
+> -- -----
+> 1  83
+> 2  83
+> 3  30
+> 4  40
+> 5  52
+> 6  83
+> 7  83
+> 8  81
+> 9  91
+> rows (ordered): 9