New experimental feature to speed up CREATE TABLE ... AS SELECT.

h2/src/main/org/h2/command/ddl/CreateTable.java

@@ -171,6 +171,7 @@ public class CreateTable extends SchemaCommand {
                     insert.setSortedInsertMode(sortedInsertMode);
                     insert.setQuery(asQuery);
                     insert.setTable(table);
+                    insert.setInsertFromSelect(true);
                     insert.prepare();
                     insert.update();
                 } finally {

h2/src/main/org/h2/command/dml/Insert.java

@@ -21,6 +21,7 @@ import org.h2.expression.Parameter;
 import org.h2.index.PageIndex;
 import org.h2.message.DbException;
 import org.h2.result.ResultInterface;
+import org.h2.result.ResultTarget;
 import org.h2.result.Row;
 import org.h2.table.Column;
 import org.h2.table.Table;
@@ -32,13 +33,15 @@ import org.h2.value.Value;
  * This class represents the statement
  * INSERT
  */
-public class Insert extends Prepared {
+public class Insert extends Prepared implements ResultTarget {
 
     private Table table;
     private Column[] columns;
     private ArrayList<Expression[]> list = New.arrayList();
     private Query query;
     private boolean sortedInsertMode;
+    private int rowNumber;
+    private boolean insertFromSelect;
 
     public Insert(Session session) {
         super(session);
@@ -89,17 +92,18 @@ public class Insert extends Prepared {
     }
 
     private int insertRows() {
-        int count;
         session.getUser().checkRight(table, Right.INSERT);
         setCurrentRowNumber(0);
         table.fire(session, Trigger.INSERT, true);
-        if (list.size() > 0) {
-            count = 0;
-            for (int x = 0; x < list.size(); x++) {
-                Expression[] expr = list.get(x);
+        rowNumber = 0;
+        int listSize = list.size();
+        if (listSize > 0) {
+            int columnLen = columns.length;
+            for (int x = 0; x < listSize; x++) {
                 Row newRow = table.getTemplateRow();
+                Expression[] expr = list.get(x);
                 setCurrentRowNumber(x + 1);
-                for (int i = 0; i < columns.length; i++) {
+                for (int i = 0; i < columnLen; i++) {
                     Column c = columns[i];
                     int index = c.getColumnId();
                     Expression e = expr[i];
@@ -114,6 +118,7 @@ public class Insert extends Prepared {
                         }
                     }
                 }
+                rowNumber++;
                 table.validateConvertUpdateSequence(session, newRow);
                 boolean done = table.fireBeforeRow(session, null, newRow);
                 if (!done) {
@@ -122,25 +127,35 @@ public class Insert extends Prepared {
                     session.log(table, UndoLogRecord.INSERT, newRow);
                     table.fireAfterRow(session, null, newRow, false);
                 }
-                count++;
             }
         } else {
-            ResultInterface rows = query.query(0);
-            count = 0;
             table.lock(session, true, false);
+            if (insertFromSelect) {
+                query.query(0, this);
+            } else {
+                ResultInterface rows = query.query(0);
                 while (rows.next()) {
-                count++;
                     Value[] r = rows.currentRow();
+                    addRow(r);
+                }
+                rows.close();
+            }
+        }
+        table.fire(session, Trigger.INSERT, false);
+        return rowNumber;
+    }
+
+    public void addRow(Value[] values) {
         Row newRow = table.getTemplateRow();
-                setCurrentRowNumber(count);
-                for (int j = 0; j < columns.length; j++) {
+        setCurrentRowNumber(++rowNumber);
+        for (int j = 0, len = columns.length; j < len; j++) {
             Column c = columns[j];
             int index = c.getColumnId();
             try {
-                        Value v = c.convert(r[j]);
+                Value v = c.convert(values[j]);
                 newRow.setValue(index, v);
             } catch (DbException ex) {
-                        throw setRow(ex, count, getSQL(r));
+                throw setRow(ex, rowNumber, getSQL(values));
             }
         }
         table.validateConvertUpdateSequence(session, newRow);
@@ -151,11 +166,6 @@ public class Insert extends Prepared {
             table.fireAfterRow(session, null, newRow, false);
         }
     }
-            rows.close();
-        }
-        table.fire(session, Trigger.INSERT, false);
-        return count;
-    }
 
     public String getPlanSQL() {
         StatementBuilder buff = new StatementBuilder("INSERT INTO ");
@@ -204,7 +214,7 @@ public class Insert extends Prepared {
                 if (expr.length != columns.length) {
                     throw DbException.get(ErrorCode.COLUMN_COUNT_DOES_NOT_MATCH);
                 }
-                for (int i = 0; i < expr.length; i++) {
+                for (int i = 0, len = expr.length; i < len; i++) {
                     Expression e = expr[i];
                     if (e != null) {
                         e = e.optimize(session);
@@ -240,4 +250,8 @@ public class Insert extends Prepared {
         return CommandInterface.INSERT;
     }
 
+    public void setInsertFromSelect(boolean value) {
+        this.insertFromSelect = value;
+    }
+
 }

h2/src/main/org/h2/command/dml/Query.java

@@ -21,6 +21,7 @@ import org.h2.expression.ValueExpression;
 import org.h2.message.DbException;
 import org.h2.result.LocalResult;
 import org.h2.result.ResultInterface;
+import org.h2.result.ResultTarget;
 import org.h2.result.SortOrder;
 import org.h2.table.ColumnResolver;
 import org.h2.table.Table;
@@ -62,12 +63,15 @@ public abstract class Query extends Prepared {
     }
 
     /**
-     * Execute the query without checking the cache.
+     * Execute the query without checking the cache. If a target is specified,
+     * the results are written to it, and the method returns null. If no target
+     * is specified, a new LocalResult is created and returned.
      *
      * @param limit the limit as specified in the JDBC method call
+     * @param target the target to write results to
      * @return the result
      */
-    protected abstract LocalResult queryWithoutCache(int limit);
+    protected abstract LocalResult queryWithoutCache(int limit, ResultTarget target);
 
     /**
      * Initialize the query.
@@ -218,10 +222,21 @@ public abstract class Query extends Prepared {
         return params;
     }
 
-    public ResultInterface query(int limit) {
+    public ResultInterface query(int maxrows) {
+        return query(maxrows, null);
+    }
+
+    /**
+     * Execute the query, writing the result to the target result.
+     *
+     * @param maxrows the maximum number of rows to return
+     * @param target the target result (null will return the result)
+     * @return the result set (if the target is not set).
+     */
+    ResultInterface query(int limit, ResultTarget target) {
         fireBeforeSelectTriggers();
         if (!session.getDatabase().getOptimizeReuseResults()) {
-            return queryWithoutCache(limit);
+            return queryWithoutCache(limit, target);
         }
         Value[] params = getParameterValues();
         long now = session.getDatabase().getModificationDataId();
@@ -238,7 +253,7 @@ public abstract class Query extends Prepared {
         }
         lastParameters = params;
         closeLastResult();
-        lastResult = queryWithoutCache(limit);
+        lastResult = queryWithoutCache(limit, target);
         this.lastEvaluated = now;
         lastLimit = limit;
         return lastResult;

h2/src/main/org/h2/command/dml/Select.java

@@ -29,6 +29,7 @@ import org.h2.index.IndexType;
 import org.h2.message.DbException;
 import org.h2.result.LocalResult;
 import org.h2.result.ResultInterface;
+import org.h2.result.ResultTarget;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
 import org.h2.result.SortOrder;
@@ -151,7 +152,7 @@ public class Select extends Query {
         }
     }
 
-    private void queryGroupSorted(int columnCount, LocalResult result) {
+    private void queryGroupSorted(int columnCount, ResultTarget result) {
         int rowNumber = 0;
         setCurrentRowNumber(0);
         Value[] previousKeyValues = null;
@@ -190,7 +191,7 @@ public class Select extends Query {
         }
     }
 
-    private void addGroupSortedRow(Value[] keyValues, int columnCount, LocalResult result) {
+    private void addGroupSortedRow(Value[] keyValues, int columnCount, ResultTarget result) {
         Value[] row = new Value[columnCount];
         for (int j = 0; groupIndex != null && j < groupIndex.length; j++) {
             row[groupIndex[j]] = keyValues[j];
@@ -443,7 +444,7 @@ public class Select extends Query {
         return null;
     }
 
-    private void queryDistinct(LocalResult result, long limitRows) {
+    private void queryDistinct(ResultTarget result, long limitRows) {
         if (limitRows != 0 && offsetExpr != null) {
             // limitRows must be long, otherwise we get an int overflow
             // if limitRows is at or near Integer.MAX_VALUE
@@ -469,7 +470,7 @@ public class Select extends Query {
             Value[] row = { value };
             result.addRow(row);
             rowNumber++;
-            if ((sort == null || sortUsingIndex) && limitRows != 0 && result.getRowCount() >= limitRows) {
+            if ((sort == null || sortUsingIndex) && limitRows != 0 && rowNumber >= limitRows) {
                 break;
             }
             if (sampleSize > 0 && rowNumber >= sampleSize) {
@@ -478,7 +479,7 @@ public class Select extends Query {
         }
     }
 
-    private void queryFlat(int columnCount, LocalResult result, long limitRows) {
+    private void queryFlat(int columnCount, ResultTarget result, long limitRows) {
         if (limitRows != 0 && offsetExpr != null) {
             // limitRows must be long, otherwise we get an int overflow
             // if limitRows is at or near Integer.MAX_VALUE
@@ -503,7 +504,7 @@ public class Select extends Query {
                 }
                 result.addRow(row);
                 rowNumber++;
-                if ((sort == null || sortUsingIndex) && limitRows != 0 && result.getRowCount() >= limitRows) {
+                if ((sort == null || sortUsingIndex) && limitRows != 0 && rowNumber >= limitRows) {
                     break;
                 }
                 if (sampleSize > 0 && rowNumber >= sampleSize) {
@@ -516,7 +517,7 @@ public class Select extends Query {
         }
     }
 
-    private void queryQuick(int columnCount, LocalResult result) {
+    private void queryQuick(int columnCount, ResultTarget result) {
         Value[] row = new Value[columnCount];
         for (int i = 0; i < columnCount; i++) {
             Expression expr = expressions.get(i);
@@ -531,7 +532,7 @@ public class Select extends Query {
         return result;
     }
 
-    protected LocalResult queryWithoutCache(int maxRows) {
+    protected LocalResult queryWithoutCache(int maxRows, ResultTarget target) {
         int limitRows = maxRows;
         if (limitExpr != null) {
             int l = limitExpr.getValue(session).getInt();
@@ -542,13 +543,24 @@ public class Select extends Query {
             }
         }
         int columnCount = expressions.size();
-        LocalResult result = new LocalResult(session, expressionArray, visibleColumnCount);
-        if (!sortUsingIndex || distinct) {
+        LocalResult result = null;
+        if (!SysProperties.optimizeInsertFromSelect || target == null) {
+            result = createLocalResult(result);
+        }
+        if (sort != null && (!sortUsingIndex || distinct)) {
+            result = createLocalResult(result);
             result.setSortOrder(sort);
         }
         if (distinct && !isDistinctQuery) {
+            result = createLocalResult(result);
             result.setDistinct();
         }
+        if (isGroupQuery && !isGroupSortedQuery) {
+            result = createLocalResult(result);
+        }
+        if (limitRows != 0 || offsetExpr != null) {
+            result = createLocalResult(result);
+        }
         topTableFilter.startQuery(session);
         topTableFilter.reset();
         boolean exclusive = isForUpdate && !isForUpdateMvcc;
@@ -566,18 +578,19 @@ public class Select extends Query {
                 throw DbException.getUnsupportedException("FOR UPDATE && JOIN");
             }
         }
+        ResultTarget to = result != null ? result : target;
         if (isQuickAggregateQuery) {
-            queryQuick(columnCount, result);
+            queryQuick(columnCount, to);
         } else if (isGroupQuery) {
             if (isGroupSortedQuery) {
-                queryGroupSorted(columnCount, result);
+                queryGroupSorted(columnCount, to);
             } else {
                 queryGroup(columnCount, result);
             }
         } else if (isDistinctQuery) {
-            queryDistinct(result, limitRows);
+            queryDistinct(to, limitRows);
         } else {
-            queryFlat(columnCount, result, limitRows);
+            queryFlat(columnCount, to, limitRows);
         }
         if (offsetExpr != null) {
             result.setOffset(offsetExpr.getValue(session).getInt());
@@ -585,9 +598,23 @@ public class Select extends Query {
         if (limitRows != 0) {
             result.setLimit(limitRows);
         }
+        if (result != null) {
             result.done();
+            if (target != null) {
+                while (result.next()) {
+                    target.addRow(result.currentRow());
+                }
+                result.close();
+                return null;
+            }
             return result;
         }
+        return null;
+    }
+
+    private LocalResult createLocalResult(LocalResult old) {
+        return old != null ? old : new LocalResult(session, expressionArray, visibleColumnCount);
+    }
 
     private void expandColumnList() {
         for (int i = 0; i < expressions.size(); i++) {

h2/src/main/org/h2/command/dml/SelectUnion.java

@@ -20,6 +20,7 @@ import org.h2.expression.ValueExpression;
 import org.h2.message.DbException;
 import org.h2.result.LocalResult;
 import org.h2.result.ResultInterface;
+import org.h2.result.ResultTarget;
 import org.h2.result.SortOrder;
 import org.h2.table.Column;
 import org.h2.table.ColumnResolver;
@@ -118,16 +119,27 @@ public class SelectUnion extends Query {
         return new LocalResult(session, expressionArray, columnCount);
     }
 
-    protected LocalResult queryWithoutCache(int maxrows) {
+    protected LocalResult queryWithoutCache(int maxrows, ResultTarget target) {
         if (maxrows != 0) {
             if (limitExpr != null) {
                 maxrows = Math.min(limitExpr.getValue(session).getInt(), maxrows);
             }
             limitExpr = ValueExpression.get(ValueInt.get(maxrows));
         }
+        if (SysProperties.optimizeInsertFromSelect) {
+            if (unionType == UNION_ALL && target != null) {
+                if (sort == null && !distinct && maxrows == 0 && offsetExpr == null && limitExpr == null) {
+                    left.query(0, target);
+                    right.query(0, target);
+                    return null;
+                }
+            }
+        }
         int columnCount = left.getColumnCount();
         LocalResult result = new LocalResult(session, expressionArray, columnCount);
+        if (sort != null) {
             result.setSortOrder(sort);
+        }
         if (distinct) {
             left.setDistinct(true);
             right.setDistinct(true);
@@ -197,6 +209,13 @@ public class SelectUnion extends Query {
             result.setLimit(limitExpr.getValue(session).getInt());
         }
         result.done();
+        if (target != null) {
+            while (result.next()) {
+                target.addRow(result.currentRow());
+            }
+            result.close();
+            return null;
+        }
         return result;
     }
 
@@ -350,9 +369,9 @@ public class SelectUnion extends Query {
         return buff.toString();
     }
 
-    public ResultInterface query(int limit) {
+    public ResultInterface query(int limit, ResultTarget target) {
         // union doesn't always know the parameter list of the left and right queries
-        return queryWithoutCache(limit);
+        return queryWithoutCache(limit, target);
     }
 
     public boolean isEverything(ExpressionVisitor visitor) {

h2/src/main/org/h2/constant/SysProperties.java

@@ -480,6 +480,14 @@ public class SysProperties {
      */
     public static final int OBJECT_CACHE_SIZE = MathUtils.nextPowerOf2(getIntSetting("h2.objectCacheSize", 1024));
 
+    /**
+     * System property <code>h2.optimizeInsertFromSelect</code>
+     * (default: false).<br />
+     * Insert into table from query directly bypassing temporary disk storage.
+     * This also applies to create table as select.
+     */
+    public static boolean optimizeInsertFromSelect = getBooleanSetting("h2.optimizeInsertFromSelect", false);
+
     /**
      * System property <code>h2.optimizeDistinct</code> (default: true).<br />
      * Improve the performance of simple DISTINCT queries if an index is

h2/src/main/org/h2/result/LocalResult.java

@@ -25,7 +25,7 @@ import org.h2.value.ValueArray;
  * and it is also used directly by the ResultSet class in the embedded mode.
  * If the result does not fit in memory, it is written to a temporary file.
  */
-public class LocalResult implements ResultInterface {
+public class LocalResult implements ResultInterface, ResultTarget {
 
     private int maxMemoryRows;
     private Session session;
@@ -78,10 +78,10 @@ public class LocalResult implements ResultInterface {
      * @return the local result set
      */
     public static LocalResult read(Session session, ResultSet rs, int maxrows) {
-        try {
         Expression[] cols = Expression.getExpressionColumns(session, rs);
         int columnCount = cols.length;
         LocalResult result = new LocalResult(session, cols, columnCount);
+        try {
             for (int i = 0; (maxrows == 0 || i < maxrows) && rs.next(); i++) {
                 Value[] list = new Value[columnCount];
                 for (int j = 0; j < columnCount; j++) {
@@ -90,11 +90,11 @@ public class LocalResult implements ResultInterface {
                 }
                 result.addRow(list);
             }
-            result.done();
-            return result;
         } catch (SQLException e) {
             throw DbException.convert(e);
         }
+        result.done();
+        return result;
     }
 
     /**

h2/src/main/org/h2/result/ResultTarget.java

+/*
+ * Copyright 2004-2010 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.result;
+
+import org.h2.value.Value;
+
+/**
+ * A object where rows are written to.
+ */
+public interface ResultTarget {
+
+    /**
+     * Add the row to the result set.
+     *
+     * @param values the values
+     */
+    void addRow(Value[] values);
+
+}

h2/src/test/org/h2/samples/DirectInsert.java

+/*
+ * Copyright 2004-2010 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.samples;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.h2.constant.SysProperties;
+import org.h2.tools.DeleteDbFiles;
+
+/**
+ * Demonstrates the benefit of using the CREATE TABLE ... AS SELECT
+ * optimization.
+ */
+public class DirectInsert {
+
+    /**
+     * This method is called when executing this sample application from the
+     * command line.
+     *
+     * @param args the command line parameters
+     */
+    public static void main(String... args) throws Exception {
+        Class.forName("org.h2.Driver");
+        DeleteDbFiles.execute("~", "test", true);
+        Connection conn = DriverManager.getConnection("jdbc:h2:~/test;LOG=0", "sa", "");
+        Statement stat = conn.createStatement();
+        initialInsert(conn, stat, 200000);
+        for (int i = 0; i < 3; i++) {
+            createAsSelect(stat, true);
+            createAsSelect(stat, false);
+        }
+    }
+
+    private static void initialInsert(Connection conn, Statement stat, int len) throws SQLException {
+        stat.execute("DROP TABLE IF EXISTS TEST");
+        stat.execute("CREATE TABLE TEST(ID INT PRIMARY KEY, NAME VARCHAR)");
+        PreparedStatement prep = conn.prepareStatement("INSERT INTO TEST VALUES(?, 'Test' || SPACE(100))");
+        long time = System.currentTimeMillis();
+        for (int i = 0; i < len; i++) {
+            long now = System.currentTimeMillis();
+            if (now > time + 1000) {
+                time = now;
+                System.out.println("Inserting " + (100L * i / len) + "%");
+            }
+            prep.setInt(1, i);
+            prep.execute();
+        }
+        conn.commit();
+    }
+
+    private static void createAsSelect(Statement stat, boolean optimize) throws SQLException {
+        SysProperties.optimizeInsertFromSelect = optimize;
+        stat.execute("DROP TABLE IF EXISTS TEST2");
+        System.out.println("CREATE TABLE ... AS SELECT " + (optimize ? "(optimized)" : ""));
+        long time = System.currentTimeMillis();
+        stat.execute("CREATE TABLE TEST2 AS SELECT * FROM TEST");
+        System.out.printf("%.3f sec.\n", (System.currentTimeMillis() - time) / 1000.0);
+        stat.execute("INSERT INTO TEST2 SELECT * FROM TEST2");
+    }
+
+}