The condition "in(select ...)" did not work correctly if the subquery could not…

The condition "in(select ...)" did not work correctly if the subquery could not be converted to a "distinct" query, as in: "select * from dual where x in (select x from dual group by x order by max(x))".

h2/src/main/org/h2/result/LocalResult.java

@@ -255,7 +255,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
                 Database db = session.getDatabase();
                 if (rowCount > db.getSettings().maxMemoryRowsDistinct && db.isPersistent()) {
                     external = new ResultTempTable(session, sort);
-                    external.addRows(distinctRows.values());
+                    rowCount = external.addRows(distinctRows.values());
                     distinctRows = null;
                 }
             } else {
@@ -278,7 +278,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
     }
 
     private void addRowsToDisk() {
-        external.addRows(rows);
+        rowCount = external.addRows(rows);
         rows.clear();
     }
 
@@ -315,7 +315,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
                         }
                         rows.add(list);
                         if (rows.size() > maxMemoryRows) {
-                            external.addRows(rows);
+                            rowCount = external.addRows(rows);
                             rows.clear();
                         }
                     }
@@ -452,7 +452,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
     }
 
     public String toString() {
-        return "columns: " + visibleColumnCount + " rows: " + rowCount + " pos: " + rowId;
+        return super.toString() + " columns: " + visibleColumnCount + " rows: " + rowCount + " pos: " + rowId;
     }
 
     /**

h2/src/main/org/h2/result/ResultDiskBuffer.java

@@ -33,6 +33,7 @@ class ResultDiskBuffer implements ResultExternal {
     private final int maxBufferSize;
 
     private FileStore file;
+    private int rowCount;
 
     private final ResultDiskBuffer parent;
     private boolean closed;
@@ -122,7 +123,7 @@ class ResultDiskBuffer implements ResultExternal {
         return new ResultDiskBuffer(this);
     }
 
-    public void addRows(ArrayList<Value[]> rows) {
+    public int addRows(ArrayList<Value[]> rows) {
         if (sort != null) {
             sort.sort(rows);
         }
@@ -166,6 +167,8 @@ class ResultDiskBuffer implements ResultExternal {
         } else {
             mainTape.end = file.getFilePointer();
         }
+        rowCount += rows.size();
+        return rowCount;
     }
 
     public void done() {

h2/src/main/org/h2/result/ResultExternal.java

@@ -27,12 +27,21 @@ public interface ResultExternal {
      */
     Value[] next();
 
+    /**
+     * Add a row to this object.
+     *
+     * @param values the row to add
+     * @return the new number of rows in this object
+     */
+    int addRow(Value[] values);
+
     /**
      * Add a number of rows to the result.
      *
      * @param rows the list of rows to add
+     * @return the new number of rows in this object
      */
-    void addRows(ArrayList<Value[]> rows);
+    int addRows(ArrayList<Value[]> rows);
 
     /**
      * This method is called after all rows have been added.
@@ -61,14 +70,6 @@ public interface ResultExternal {
      */
     boolean contains(Value[] values);
 
-    /**
-     * Add a row to this object.
-     *
-     * @param values the row to add
-     * @return the new number of rows in this object
-     */
-    int addRow(Value[] values);
-
     /**
      * Create a shallow copy of this object if possible.
      *

h2/src/main/org/h2/result/ResultTempTable.java

@@ -108,13 +108,14 @@ public class ResultTempTable implements ResultExternal {
         return (int) table.getRowCount(session);
     }
 
-    public void addRows(ArrayList<Value[]> rows) {
+    public int addRows(ArrayList<Value[]> rows) {
         if (sort != null) {
             sort.sort(rows);
         }
         for (Value[] values : rows) {
             addRow(values);
         }
+        return (int) table.getRowCount(session);
     }
 
     private synchronized void closeChild() {

h2/src/test/org/h2/test/synth/TestFuzzOptimizations.java

@@ -81,6 +81,16 @@ public class TestFuzzOptimizations extends TestBase {
         db.execute("update test0 set b = null where b = 9");
         db.execute("update test0 set c = null where c = 9");
         db.execute("insert into test1 select * from test0");
+
+        // this failed at some point
+        Prepared p = db.prepare("select * from test0 where b in(" +
+                "select a from test1 where a <? and a not in(" +
+                "select c from test1 where b <=10 and a in(" +
+                "select a from test1 where b =1 or b =2 and b not in(2))) or c <>a) " +
+                "and c in(0, 10) and c in(10, 0, 0) order by 1, 2, 3");
+        p.set(1);
+        p.execute();
+
         Random seedGenerator = new Random();
         String[] columns = new String[] { "a", "b", "c" };
         String[] values = new String[] { null, "0", "0", "1", "2", "10", "a", "?" };