Add window frame exclusion clause implementation

h2/src/docsrc/help/help.csv

@@ -2535,17 +2535,10 @@ SELECT * FROM (VALUES(1, 'Hello'), (2, 'World')) AS V;
 
 "Other Grammar","Window specification","
 ([PARTITION BY expression [,...]] [ORDER BY order [,...]]
-[RANGE BETWEEN {
-    UNBOUNDED PRECEDING AND CURRENT ROW
-    |CURRENT ROW AND UNBOUNDED FOLLOWING
-    |UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
-    }])
+[windowFrame])
 ","
 A window specification for a window function or aggregate.
 
-Window frame clause (RANGE) is currently supported only in aggregates and
-FIRST_VALUE(), LAST_VALUE(), and NTH_VALUE() window functions.
-
 Window functions are currently experimental in H2 and should be used with caution.
 They also may require a lot of memory for large queries.
 ","
@@ -2553,6 +2546,21 @@ They also may require a lot of memory for large queries.
 (ORDER BY ID)
 (PARTITION BY CATEGORY)
 (PARTITION BY CATEGORY ORDER BY NAME, ID)
+(ORDER BY Y RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE TIES)
+"
+
+"Other Grammar","Window frame","
+[RANGE BETWEEN {
+    UNBOUNDED PRECEDING AND CURRENT ROW
+    |CURRENT ROW AND UNBOUNDED FOLLOWING
+    |UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+    }]
+[EXCLUDE {CURRENT ROW|GROUP|TIES|NO OTHERS}]
+","
+A window frame clause.
+Is currently supported only in aggregates and FIRST_VALUE(), LAST_VALUE(), and NTH_VALUE() window functions.
+","
+RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE GROUP
 "
 
 "Other Grammar","Term","

h2/src/main/org/h2/command/Parser.java

@@ -177,6 +177,7 @@ import org.h2.expression.aggregate.JavaAggregate;
 import org.h2.expression.aggregate.Window;
 import org.h2.expression.aggregate.WindowFrame;
 import org.h2.expression.aggregate.WindowFrame.SimpleExtent;
+import org.h2.expression.aggregate.WindowFrame.WindowFrameExclusion;
 import org.h2.expression.aggregate.WindowFunction;
 import org.h2.expression.aggregate.WindowFunction.WindowFunctionType;
 import org.h2.index.Index;
@@ -3073,7 +3074,8 @@ public class Parser {
                     frame = readWindowFrame();
                     break;
                 default:
-                    frame = new WindowFrame(SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_CURRENT_ROW);
+                    frame = new WindowFrame(SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_CURRENT_ROW,
+                            WindowFrameExclusion.EXCLUDE_NO_OTHERS);
                 }
             } else {
                 frame = readWindowFrame();
@@ -3091,6 +3093,7 @@ public class Parser {
 
     private WindowFrame readWindowFrame() {
         SimpleExtent extent;
+        WindowFrameExclusion exclusion = WindowFrameExclusion.EXCLUDE_NO_OTHERS;
         if (readIf("RANGE")) {
             read("BETWEEN");
             if (readIf("UNBOUNDED")) {
@@ -3112,10 +3115,23 @@ public class Parser {
                 read("FOLLOWING");
                 extent = SimpleExtent.RANGE_BETWEEN_CURRENT_ROW_AND_UNBOUNDED_FOLLOWING;
             }
+            if (readIf("EXCLUDE")) {
+                if (readIf("CURRENT")) {
+                    read("ROW");
+                    exclusion = WindowFrameExclusion.EXCLUDE_CURRENT_ROW;
+                } else if (readIf(GROUP)) {
+                    exclusion = WindowFrameExclusion.EXCLUDE_GROUP;
+                } else if (readIf("TIES")) {
+                    exclusion = WindowFrameExclusion.EXCLUDE_TIES;
+                } else {
+                    read("NO");
+                    read("OTHERS");
+                }
+            }
         } else {
             extent = SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_CURRENT_ROW;
         }
-        return new WindowFrame(extent);
+        return new WindowFrame(extent, exclusion);
     }
 
     private AggregateType getAggregateType(String name) {

h2/src/main/org/h2/expression/aggregate/AbstractAggregate.java

@@ -8,6 +8,7 @@ package org.h2.expression.aggregate;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 
 import org.h2.api.ErrorCode;
 import org.h2.command.dml.Select;
@@ -441,12 +442,12 @@ public abstract class AbstractAggregate extends Expression {
                 result.put(row[rowIdColumn].getInt(), value);
             }
         } else {
-            // TODO optimize unordered aggregates
             int size = ordered.size();
             for (int i = 0; i < size; i++) {
                 Object aggregateData = createAggregateData();
-                for (int j = i; j < size; j++) {
-                    updateFromExpressions(session, aggregateData, ordered.get(j));
+                for (Iterator<Value[]> iter = frame.iterator(ordered, getOverOrderBySort(), i, false); iter
+                        .hasNext();) {
+                    updateFromExpressions(session, aggregateData, iter.next());
                 }
                 result.put(ordered.get(i)[rowIdColumn].getInt(), getAggregatedValue(session, aggregateData));
             }

h2/src/main/org/h2/expression/aggregate/Window.java

@@ -36,7 +36,10 @@ public final class Window {
      */
     static void appendOrderBy(StringBuilder builder, ArrayList<SelectOrderBy> orderBy) {
         if (orderBy != null && !orderBy.isEmpty()) {
-            builder.append(" ORDER BY ");
+            if (builder.charAt(builder.length() - 1) != '(') {
+                builder.append(' ');
+            }
+            builder.append("ORDER BY ");
             for (int i = 0; i < orderBy.size(); i++) {
                 SelectOrderBy o = orderBy.get(i);
                 if (i > 0) {
@@ -187,10 +190,10 @@ public final class Window {
                 builder.append(StringUtils.unEnclose(partitionBy.get(i).getSQL()));
             }
         }
+        appendOrderBy(builder, orderBy);
         if (!frame.isDefault()) {
             builder.append(' ').append(frame.getSQL());
         }
-        appendOrderBy(builder, orderBy);
         return builder.append(')').toString();
     }
 

h2/src/main/org/h2/expression/aggregate/WindowFrame.java

@@ -6,10 +6,13 @@
 package org.h2.expression.aggregate;
 
 import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
 import org.h2.message.DbException;
+import org.h2.result.SortOrder;
 import org.h2.value.Value;
 
 /**
@@ -59,68 +62,76 @@ public final class WindowFrame {
 
     }
 
-    private final SimpleExtent extent;
+    /**
+     * Window frame exclusion clause.
+     */
+    public enum WindowFrameExclusion {
+        /**
+         * EXCLUDE CURRENT ROW exclusion clause.
+         */
+        EXCLUDE_CURRENT_ROW("EXCLUDE CURRENT ROW"),
 
         /**
-     * Creates new instance of window frame clause.
-     *
-     * @param extent
-     *            window frame extent
+         * EXCLUDE GROUP exclusion clause.
          */
-    public WindowFrame(SimpleExtent extent) {
-        this.extent = extent;
-    }
+        EXCLUDE_GROUP("EXCLUDE GROUP"),
 
         /**
-     * Returns whether window frame specification can be omitted.
-     *
-     * @return whether window frame specification can be omitted
+         * EXCLUDE TIES exclusion clause.
          */
-    public boolean isDefault() {
-        return extent == SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_CURRENT_ROW;
-    }
+        EXCLUDE_TIES("EXCLUDE TIES"),
 
         /**
-     * Returns whether window frame specification contains all rows in
-     * partition.
-     *
-     * @return whether window frame specification contains all rows in partition
+         * EXCLUDE NO OTHERS exclusion clause.
          */
-    public boolean isFullPartition() {
-        return extent == SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_UNBOUNDED_FOLLOWING;
+        EXCLUDE_NO_OTHERS("EXCLUDE NO OTHERS"),
+
+        ;
+
+        private final String sql;
+
+        private WindowFrameExclusion(String sql) {
+            this.sql = sql;
         }
 
         /**
-     * Returns iterator.
+         * Returns SQL representation.
          *
-     * @param orderedRows
-     *            ordered rows
-     * @param currentRow
-     *            index of the current row
-     * @return iterator
+         * @return SQL representation.
+         * @see org.h2.expression.Expression#getSQL()
          */
-    public Iterator<Value[]> iterator(final ArrayList<Value[]> orderedRows, int currentRow) {
-        int size = orderedRows.size();
-        final int startIndex, endIndex;
-        switch (extent) {
-        case RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_CURRENT_ROW:
-            startIndex = 0;
-            endIndex = currentRow;
-            break;
-        case RANGE_BETWEEN_CURRENT_ROW_AND_UNBOUNDED_FOLLOWING:
-            startIndex = currentRow;
-            endIndex = size - 1;
-            break;
-        case RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_UNBOUNDED_FOLLOWING:
-            startIndex = 0;
-            endIndex = size - 1;
-            break;
-        default:
-            throw DbException.getUnsupportedException("window frame extent =" + extent);
+        public String getSQL() {
+            return sql;
+        }
+
+    }
+
+    private abstract class Itr implements Iterator<Value[]> {
+
+        final ArrayList<Value[]> orderedRows;
+
+        Itr(ArrayList<Value[]> orderedRows) {
+            this.orderedRows = orderedRows;
+        }
+
+        @Override
+        public final void remove() {
+            throw new UnsupportedOperationException();
+        }
+
     }
-        return new Iterator<Value[]>() {
 
-            private int cursor = startIndex;
+    private final class PlainItr extends Itr {
+
+        private final int endIndex;
+
+        private int cursor;
+
+        PlainItr(ArrayList<Value[]> orderedRows, int startIndex, int endIndex) {
+            super(orderedRows);
+            this.endIndex = endIndex;
+            cursor = startIndex;
+        }
 
         @Override
         public boolean hasNext() {
@@ -135,24 +146,144 @@ public final class WindowFrame {
             return orderedRows.get(cursor++);
         }
 
+    }
+
+    private final class PlainReverseItr extends Itr {
+
+        private final int startIndex;
+
+        private int cursor;
+
+        PlainReverseItr(ArrayList<Value[]> orderedRows, int startIndex, int endIndex) {
+            super(orderedRows);
+            this.startIndex = startIndex;
+            cursor = endIndex;
+        }
+
         @Override
-            public void remove() {
-                throw new UnsupportedOperationException();
+        public boolean hasNext() {
+            return cursor >= startIndex;
         }
 
-        };
+        @Override
+        public Value[] next() {
+            if (cursor < startIndex) {
+                throw new NoSuchElementException();
+            }
+            return orderedRows.get(cursor--);
         }
 
+    }
+
+    private abstract class AbstractBitSetItr extends Itr {
+
+        final BitSet set;
+
+        int cursor;
+
+        AbstractBitSetItr(ArrayList<Value[]> orderedRows, BitSet set) {
+            super(orderedRows);
+            this.set = set;
+        }
+
+        @Override
+        public final boolean hasNext() {
+            return cursor >= 0;
+        }
+
+    }
+
+    private final class BitSetItr extends AbstractBitSetItr {
+
+        BitSetItr(ArrayList<Value[]> orderedRows, BitSet set) {
+            super(orderedRows, set);
+            cursor = set.nextSetBit(0);
+        }
+
+        @Override
+        public Value[] next() {
+            if (cursor < 0) {
+                throw new NoSuchElementException();
+            }
+            Value[] result = orderedRows.get(cursor);
+            cursor = set.nextSetBit(cursor + 1);
+            return result;
+        }
+
+    }
+
+    private final class BitSetReverseItr extends AbstractBitSetItr {
+
+        BitSetReverseItr(ArrayList<Value[]> orderedRows, BitSet set) {
+            super(orderedRows, set);
+            cursor = set.length() - 1;
+        }
+
+        @Override
+        public Value[] next() {
+            if (cursor < 0) {
+                throw new NoSuchElementException();
+            }
+            Value[] result = orderedRows.get(cursor);
+            cursor = set.previousSetBit(cursor - 1);
+            return result;
+        }
+
+    }
+
+    private final SimpleExtent extent;
+
+    private final WindowFrameExclusion exclusion;
+
     /**
-     * Returns iterator in descending order.
+     * Creates new instance of window frame clause.
+     *
+     * @param extent
+     *            window frame extent
+     * @param exclusion
+     *            window frame exclusion
+     */
+    public WindowFrame(SimpleExtent extent, WindowFrameExclusion exclusion) {
+        this.extent = extent;
+        this.exclusion = exclusion;
+    }
+
+    /**
+     * Returns whether window frame specification can be omitted.
+     *
+     * @return whether window frame specification can be omitted
+     */
+    public boolean isDefault() {
+        return extent == SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_CURRENT_ROW
+                && exclusion == WindowFrameExclusion.EXCLUDE_NO_OTHERS;
+    }
+
+    /**
+     * Returns whether window frame specification contains all rows in
+     * partition.
+     *
+     * @return whether window frame specification contains all rows in partition
+     */
+    public boolean isFullPartition() {
+        return extent == SimpleExtent.RANGE_BETWEEN_UNBOUNDED_PRECEDING_AND_UNBOUNDED_FOLLOWING
+                && exclusion == WindowFrameExclusion.EXCLUDE_NO_OTHERS;
+    }
+
+    /**
+     * Returns iterator.
      *
      * @param orderedRows
      *            ordered rows
+     * @param sortOrder
+     *            sort order
      * @param currentRow
      *            index of the current row
-     * @return iterator in descending order
+     * @param reverse
+     *            whether iterator should iterate in reverse order
+     * @return iterator
      */
-    public Iterator<Value[]> reverseIterator(final ArrayList<Value[]> orderedRows, int currentRow) {
+    public Iterator<Value[]> iterator(ArrayList<Value[]> orderedRows, SortOrder sortOrder, int currentRow,
+            boolean reverse) {
         int size = orderedRows.size();
         final int startIndex, endIndex;
         switch (extent) {
@@ -171,29 +302,45 @@ public final class WindowFrame {
         default:
             throw DbException.getUnsupportedException("window frame extent =" + extent);
         }
-        return new Iterator<Value[]>() {
-
-            private int cursor = endIndex;
-
-            @Override
-            public boolean hasNext() {
-                return cursor >= startIndex;
+        if (exclusion != WindowFrameExclusion.EXCLUDE_NO_OTHERS) {
+            return complexIterator(orderedRows, sortOrder, currentRow, startIndex, endIndex, reverse);
+        }
+        return reverse ? new PlainReverseItr(orderedRows, startIndex, endIndex)
+                : new PlainItr(orderedRows, startIndex, endIndex);
     }
 
-            @Override
-            public Value[] next() {
-                if (cursor < startIndex) {
-                    throw new NoSuchElementException();
+    private Iterator<Value[]> complexIterator(ArrayList<Value[]> orderedRows, SortOrder sortOrder, int currentRow,
+            int startIndex, int endIndex, boolean reverse) {
+        int size = orderedRows.size();
+        BitSet set = new BitSet(size);
+        set.set(startIndex, endIndex + 1);
+        switch (exclusion) {
+        case EXCLUDE_CURRENT_ROW:
+            set.clear(currentRow);
+            break;
+        case EXCLUDE_GROUP:
+        case EXCLUDE_TIES: {
+            int exStart = currentRow;
+            Value[] row = orderedRows.get(currentRow);
+            while (exStart > startIndex && sortOrder.compare(row, orderedRows.get(exStart - 1)) == 0) {
+                exStart--;
             }
-                return orderedRows.get(cursor--);
+            int exEnd = currentRow;
+            while (exEnd < endIndex && sortOrder.compare(row, orderedRows.get(exEnd + 1)) == 0) {
+                exEnd++;
             }
-
-            @Override
-            public void remove() {
-                throw new UnsupportedOperationException();
+            set.clear(exStart, exEnd + 1);
+            if (exclusion == WindowFrameExclusion.EXCLUDE_TIES) {
+                set.set(currentRow);
             }
-
-        };
+        }
+        //$FALL-THROUGH$
+        default:
+        }
+        if (set.isEmpty()) {
+            return Collections.emptyIterator();
+        }
+        return reverse ? new BitSetReverseItr(orderedRows, set) : new BitSetItr(orderedRows, set);
     }
 
     /**
@@ -203,7 +350,11 @@ public final class WindowFrame {
      * @see org.h2.expression.Expression#getSQL()
      */
     public String getSQL() {
-        return extent.getSQL();
+        String sql = extent.getSQL();
+        if (exclusion != WindowFrameExclusion.EXCLUDE_NO_OTHERS) {
+            sql = sql + ' ' + exclusion.getSQL();
+        }
+        return sql;
     }
 
 }

h2/src/main/org/h2/expression/aggregate/WindowFunction.java

@@ -321,11 +321,11 @@ public class WindowFunction extends AbstractAggregate {
             Value v;
             switch (type) {
             case FIRST_VALUE: {
-                v = getNthValue(frame.iterator(ordered, i), 0, ignoreNulls);
+                v = getNthValue(frame.iterator(ordered, getOverOrderBySort(), i, false), 0, ignoreNulls);
                 break;
             }
             case LAST_VALUE:
-                v = getNthValue(frame.reverseIterator(ordered, i), 0, ignoreNulls);
+                v = getNthValue(frame.iterator(ordered, getOverOrderBySort(), i, true), 0, ignoreNulls);
                 break;
             case NTH_VALUE: {
                 int n = row[1].getInt();
@@ -333,7 +333,7 @@ public class WindowFunction extends AbstractAggregate {
                     throw DbException.getInvalidValueException("nth row", n);
                 }
                 n--;
-                Iterator<Value[]> iter = fromLast ? frame.reverseIterator(ordered, i) : frame.iterator(ordered, i);
+                Iterator<Value[]> iter = frame.iterator(ordered, getOverOrderBySort(), i, fromLast);
                 v = getNthValue(iter, n, ignoreNulls);
                 break;
             }

h2/src/test/org/h2/test/scripts/functions/aggregate/array-agg.sql

@@ -233,8 +233,8 @@ SELECT
 > rows: 6
 
 SELECT ARRAY_AGG(SUM(ID)) OVER(ORDER /**/ BY ID) FROM TEST GROUP BY ID;
-> ARRAY_AGG(SUM(ID)) OVER ( ORDER BY ID)
-> --------------------------------------
+> ARRAY_AGG(SUM(ID)) OVER (ORDER BY ID)
+> -------------------------------------
 > (1)
 > (1, 2)
 > (1, 2, 3)
@@ -245,3 +245,33 @@ SELECT ARRAY_AGG(SUM(ID)) OVER(ORDER /**/ BY ID) FROM TEST GROUP BY ID;
 
 DROP TABLE TEST;
 > ok
+
+CREATE TABLE TEST(ID INT, G INT);
+> ok
+
+INSERT INTO TEST VALUES
+    (1, 1),
+    (2, 2),
+    (3, 2),
+    (4, 2),
+    (5, 3);
+> update count: 5
+
+SELECT
+    ARRAY_AGG(ID) OVER (ORDER BY G RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) D,
+    ARRAY_AGG(ID) OVER (ORDER BY G RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE CURRENT ROW) R,
+    ARRAY_AGG(ID) OVER (ORDER BY G RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE GROUP) G,
+    ARRAY_AGG(ID) OVER (ORDER BY G RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE TIES) T,
+    ARRAY_AGG(ID) OVER (ORDER BY G RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING EXCLUDE NO OTHERS) N
+    FROM TEST;
+> D               R            G            T               N
+> --------------- ------------ ------------ --------------- ---------------
+> (1, 2, 3, 4, 5) (2, 3, 4, 5) (2, 3, 4, 5) (1, 2, 3, 4, 5) (1, 2, 3, 4, 5)
+> (1, 2, 3, 4, 5) (1, 3, 4, 5) (1, 5)       (1, 2, 5)       (1, 2, 3, 4, 5)
+> (1, 2, 3, 4, 5) (1, 2, 4, 5) (1, 5)       (1, 3, 5)       (1, 2, 3, 4, 5)
+> (1, 2, 3, 4, 5) (1, 2, 3, 5) (1, 5)       (1, 4, 5)       (1, 2, 3, 4, 5)
+> (1, 2, 3, 4, 5) (1, 2, 3, 4) (1, 2, 3, 4) (1, 2, 3, 4, 5) (1, 2, 3, 4, 5)
+> rows (ordered): 5
+
+DROP TABLE TEST;
+> ok

h2/src/test/org/h2/test/scripts/functions/window/nth_value.sql

@@ -135,5 +135,16 @@ SELECT *,
 > 13 4        null  41    null null
 > rows (ordered): 13
 
+SELECT ID, CATEGORY,
+    NTH_VALUE(CATEGORY, 2) OVER (ORDER BY CATEGORY RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) C,
+    NTH_VALUE(CATEGORY, 2) OVER (ORDER BY CATEGORY RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW)
+    FROM TEST FETCH FIRST 3 ROWS ONLY;
+> ID CATEGORY C    NTH_VALUE(CATEGORY, 2) OVER (ORDER BY CATEGORY RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW)
+> -- -------- ---- ---------------------------------------------------------------------------------------------------------------------
+> 1  1        null null
+> 2  1        1    null
+> 3  1        1    1
+> rows (ordered): 3
+
 DROP TABLE TEST;
 > ok

h2/src/tools/org/h2/build/doc/dictionary.txt

@@ -796,4 +796,4 @@ interior envelopes multilinestring multipoint packed exterior normalization awkw
 xym normalizes coord setz xyzm geometrycollection multipolygon mixup rings polygons rejection finite
 pointzm pointz pointm dimensionality redefine forum measures
 mpg casted pzm mls constrained subtypes complains
-ranks rno dro rko precede cume reopens preceding unbounded rightly
+ranks rno dro rko precede cume reopens preceding unbounded rightly itr