#1097 reduce memory overhead of the group by data structures

h2/src/main/org/h2/command/dml/Select.java

@@ -81,9 +81,21 @@ public class Select extends Query {
     boolean[] groupByExpression;
 
     /**
-     * The current group-by values.
+     * The array of current group-by expression data e.g. AggregateData.
      */
-    HashMap<Expression, Object> currentGroup;
+    Object[] currentGroupByExprData;
+    /**
+     * Maps an expression object to an index, to use in accessing the Object[] pointed to by groupByData.
+     */
+    private final HashMap<Expression,Integer> exprToIndexInGroupByData = new HashMap<>();
+    /**
+     * Map of group-by key to group-by expression data e.g. AggregateData
+     */
+    private ValueHashMap<Object[]> groupByData;
+    /**
+     * Key into groupByData that produces currentGroupByExprData. Not used in lazy mode.
+     */
+    private ValueArray currentGroupsKey;
 
     private int havingIndex;
     private boolean isGroupQuery, isGroupSortedQuery;
@@ -151,8 +163,47 @@ public class Select extends Query {
         return group;
     }
 
-    public HashMap<Expression, Object> getCurrentGroup() {
-        return currentGroup;
+    /**
+     * Is there currently a group-by active
+     */
+    public boolean isCurrentGroup() {
+        return currentGroupByExprData != null;
+    }
+
+    /**
+     * Get the group-by data for the current group and the passed in expression.
+     */
+    public Object getCurrentGroupExprData(Expression expr) {
+        Integer index = exprToIndexInGroupByData.get(expr);
+        if (index == null) {
+            return null;
+        }
+        return currentGroupByExprData[index];
+    }
+
+    /**
+     * Set the group-by data for the current group and the passed in expression.
+     */
+    public void setCurrentGroupExprData(Expression expr, Object obj) {
+        Integer index = exprToIndexInGroupByData.get(expr);
+        if (index != null) {
+            if (currentGroupByExprData[index] != null) {
+                throw DbException.throwInternalError();
+            }
+            currentGroupByExprData[index] = obj;
+            return;
+        }
+        index = exprToIndexInGroupByData.size();
+        exprToIndexInGroupByData.put(expr, index);
+        if (index >= currentGroupByExprData.length) {
+            currentGroupByExprData = Arrays.copyOf(currentGroupByExprData, currentGroupByExprData.length * 2);
+            // this can be null in lazy mode
+            if (currentGroupsKey != null) {
+                // since we changed the size of the array, update the object in the groups map
+                groupByData.put(currentGroupsKey, currentGroupByExprData);
+            }
+        }
+        currentGroupByExprData[index] = obj;
     }
 
     public int getCurrentGroupRowId() {
@@ -313,20 +364,20 @@ public class Select extends Query {
     }
 
     private void queryGroup(int columnCount, LocalResult result) {
-        ValueHashMap<HashMap<Expression, Object>> groups =
-                ValueHashMap.newInstance();
+        groupByData = ValueHashMap.newInstance();
+        currentGroupByExprData = null;
+        currentGroupsKey = null;
+        exprToIndexInGroupByData.clear();
         int rowNumber = 0;
         setCurrentRowNumber(0);
-        currentGroup = null;
         ValueArray defaultGroup = ValueArray.get(new Value[0]);
         int sampleSize = getSampleSizeValue(session);
         while (topTableFilter.next()) {
             setCurrentRowNumber(rowNumber + 1);
             if (isConditionMet()) {
-                Value key;
                 rowNumber++;
                 if (groupIndex == null) {
-                    key = defaultGroup;
+                    currentGroupsKey = defaultGroup;
                 } else {
                     Value[] keyValues = new Value[groupIndex.length];
                     // update group
@@ -335,14 +386,14 @@ public class Select extends Query {
                         Expression expr = expressions.get(idx);
                         keyValues[i] = expr.getValue(session);
                     }
-                    key = ValueArray.get(keyValues);
+                    currentGroupsKey = ValueArray.get(keyValues);
                 }
-                HashMap<Expression, Object> values = groups.get(key);
+                Object[] values = groupByData.get(currentGroupsKey);
                 if (values == null) {
-                    values = new HashMap<>();
-                    groups.put(key, values);
+                    values = new Object[Math.max(exprToIndexInGroupByData.size(), expressions.size())];
+                    groupByData.put(currentGroupsKey, values);
                 }
-                currentGroup = values;
+                currentGroupByExprData = values;
                 currentGroupRowId++;
                 for (int i = 0; i < columnCount; i++) {
                     if (groupByExpression == null || !groupByExpression[i]) {
@@ -355,14 +406,14 @@ public class Select extends Query {
                 }
             }
         }
-        if (groupIndex == null && groups.size() == 0) {
-            groups.put(defaultGroup, new HashMap<Expression, Object>());
+        if (groupIndex == null && groupByData.size() == 0) {
+            groupByData.put(defaultGroup, new Object[Math.max(exprToIndexInGroupByData.size(), expressions.size())]);
         }
-        ArrayList<Value> keys = groups.keys();
+        ArrayList<Value> keys = groupByData.keys();
         for (Value v : keys) {
-            ValueArray key = (ValueArray) v;
-            currentGroup = groups.get(key);
-            Value[] keyValues = key.getList();
+            currentGroupsKey = (ValueArray) v;
+            currentGroupByExprData = groupByData.get(currentGroupsKey);
+            Value[] keyValues = currentGroupsKey.getList();
             Value[] row = new Value[columnCount];
             for (int j = 0; groupIndex != null && j < groupIndex.length; j++) {
                 row[groupIndex[j]] = keyValues[j];
@@ -380,6 +431,10 @@ public class Select extends Query {
             row = keepOnlyDistinct(row, columnCount);
             result.addRow(row);
         }
+        groupByData = null;
+        currentGroupsKey = null;
+        currentGroupByExprData = null;
+        exprToIndexInGroupByData.clear();
     }
 
     /**
@@ -1475,13 +1530,15 @@ public class Select extends Query {
 
         LazyResultGroupSorted(Expression[] expressions, int columnCount) {
             super(expressions, columnCount);
-            currentGroup = null;
+            currentGroupByExprData = null;
+            currentGroupsKey = null;
         }
 
         @Override
         public void reset() {
             super.reset();
-            currentGroup = null;
+            currentGroupByExprData = null;
+            currentGroupsKey = null;
         }
 
         @Override
@@ -1501,11 +1558,11 @@ public class Select extends Query {
                     Value[] row = null;
                     if (previousKeyValues == null) {
                         previousKeyValues = keyValues;
-                        currentGroup = new HashMap<>();
+                        currentGroupByExprData = new Object[Math.max(exprToIndexInGroupByData.size(), expressions.size())];
                     } else if (!Arrays.equals(previousKeyValues, keyValues)) {
                         row = createGroupSortedRow(previousKeyValues, columnCount);
                         previousKeyValues = keyValues;
-                        currentGroup = new HashMap<>();
+                        currentGroupByExprData = new Object[Math.max(exprToIndexInGroupByData.size(), expressions.size())];
                     }
                     currentGroupRowId++;
 

h2/src/main/org/h2/expression/Aggregate.java

@@ -284,8 +284,7 @@ public class Aggregate extends Expression {
         // if (on != null) {
         // on.updateAggregate();
         // }
-        HashMap<Expression, Object> group = select.getCurrentGroup();
-        if (group == null) {
+        if (!select.isCurrentGroup()) {
             // this is a different level (the enclosing query)
             return;
         }
@@ -297,10 +296,10 @@ public class Aggregate extends Expression {
         }
         lastGroupRowId = groupRowId;
 
-        AggregateData data = (AggregateData) group.get(this);
+        AggregateData data = (AggregateData) select.getCurrentGroupExprData(this);
         if (data == null) {
             data = AggregateData.create(type);
-            group.put(this, data);
+            select.setCurrentGroupExprData(this, data);
         }
         Value v = on == null ? null : on.getValue(session);
         if (type == AggregateType.GROUP_CONCAT) {
@@ -373,13 +372,10 @@ public class Aggregate extends Expression {
                 DbException.throwInternalError("type=" + type);
             }
         }
-        HashMap<Expression, Object> group = select.getCurrentGroup();
-        if (group == null) {
-            throw DbException.get(ErrorCode.INVALID_USE_OF_AGGREGATE_FUNCTION_1, getSQL());
-        }
-        AggregateData data = (AggregateData) group.get(this);
+        AggregateData data = (AggregateData)select.getCurrentGroupExprData(this);
         if (data == null) {
             data = AggregateData.create(type);
+            select.setCurrentGroupExprData(this, data);
         }
         if (type == AggregateType.GROUP_CONCAT) {
             Value[] array = ((AggregateDataCollecting) data).getArray();

h2/src/main/org/h2/expression/ExpressionColumn.java

@@ -159,14 +159,13 @@ public class ExpressionColumn extends Expression {
         if (select == null) {
             throw DbException.get(ErrorCode.MUST_GROUP_BY_COLUMN_1, getSQL());
         }
-        HashMap<Expression, Object> values = select.getCurrentGroup();
-        if (values == null) {
+        if (!select.isCurrentGroup()) {
             // this is a different level (the enclosing query)
             return;
         }
-        Value v = (Value) values.get(this);
+        Value v = (Value) select.getCurrentGroupExprData(this);
         if (v == null) {
-            values.put(this, now);
+            select.setCurrentGroupExprData(this, now);
         } else {
             if (!database.areEqual(now, v)) {
                 throw DbException.get(ErrorCode.MUST_GROUP_BY_COLUMN_1, getSQL());
@@ -178,9 +177,8 @@ public class ExpressionColumn extends Expression {
     public Value getValue(Session session) {
         Select select = columnResolver.getSelect();
         if (select != null) {
-            HashMap<Expression, Object> values = select.getCurrentGroup();
-            if (values != null) {
-                Value v = (Value) values.get(this);
+            if (select.isCurrentGroup()) {
+                Value v = (Value) select.getCurrentGroupExprData(this);
                 if (v != null) {
                     return v;
                 }

h2/src/main/org/h2/expression/JavaAggregate.java

@@ -167,15 +167,14 @@ public class JavaAggregate extends Expression {
 
     @Override
     public Value getValue(Session session) {
-        HashMap<Expression, Object> group = select.getCurrentGroup();
-        if (group == null) {
+        if (!select.isCurrentGroup()) {
             throw DbException.get(ErrorCode.INVALID_USE_OF_AGGREGATE_FUNCTION_1, getSQL());
         }
         try {
             Aggregate agg;
             if (distinct) {
                 agg = getInstance();
-                AggregateDataCollecting data = (AggregateDataCollecting) group.get(this);
+                AggregateDataCollecting data = (AggregateDataCollecting) select.getCurrentGroupExprData(this);
                 if (data != null) {
                     for (Value value : data.values) {
                         if (args.length == 1) {
@@ -191,7 +190,7 @@ public class JavaAggregate extends Expression {
                     }
                 }
             } else {
-                agg = (Aggregate) group.get(this);
+                agg = (Aggregate) select.getCurrentGroupExprData(this);
                 if (agg == null) {
                     agg = getInstance();
                 }
@@ -208,8 +207,7 @@ public class JavaAggregate extends Expression {
 
     @Override
     public void updateAggregate(Session session) {
-        HashMap<Expression, Object> group = select.getCurrentGroup();
-        if (group == null) {
+        if (!select.isCurrentGroup()) {
             // this is a different level (the enclosing query)
             return;
         }
@@ -229,10 +227,10 @@ public class JavaAggregate extends Expression {
 
         try {
             if (distinct) {
-                AggregateDataCollecting data = (AggregateDataCollecting) group.get(this);
+                AggregateDataCollecting data = (AggregateDataCollecting) select.getCurrentGroupExprData(this);
                 if (data == null) {
                     data = new AggregateDataCollecting();
-                    group.put(this, data);
+                    select.setCurrentGroupExprData(this, data);
                 }
                 Value[] argValues = new Value[args.length];
                 Value arg = null;
@@ -243,10 +241,10 @@ public class JavaAggregate extends Expression {
                 }
                 data.add(session.getDatabase(), dataType, true, args.length == 1 ? arg : ValueArray.get(argValues));
             } else {
-                Aggregate agg = (Aggregate) group.get(this);
+                Aggregate agg = (Aggregate) select.getCurrentGroupExprData(this);
                 if (agg == null) {
                     agg = getInstance();
-                    group.put(this, agg);
+                    select.setCurrentGroupExprData(this, agg);
                 }
                 Object[] argValues = new Object[args.length];
                 Object arg = null;