You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by kg...@apache.org on 2024/06/24 13:52:47 UTC

(druid) branch master updated: Make use group iterator for all window frames & support for same bound kinds (#16603)

This is an automated email from the ASF dual-hosted git repository.

kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/master by this push:
     new 990fd5f5fbb Make use group iterator for all window frames & support for same bound kinds (#16603)
990fd5f5fbb is described below

commit 990fd5f5fbbc0140c8ff7f17af84216ce45b53f0
Author: Sree Charan Manamala <sr...@imply.io>
AuthorDate: Mon Jun 24 19:22:41 2024 +0530

    Make use group iterator for all window frames & support for same bound kinds (#16603)
    
    
    Fixes apache/druid#15739
---
 .../druid/query/operator/window/WindowFrame.java   |   4 +-
 .../semantic/DefaultFramedOnHeapAggregatable.java  | 510 ++-------------------
 .../semantic/FramedOnHeapAggregatableTest.java     |  22 +-
 .../sql/calcite/planner/DruidSqlValidator.java     |  11 -
 .../apache/druid/sql/calcite/rel/Windowing.java    |   4 +-
 .../apache/druid/sql/calcite/CalciteQueryTest.java |  20 -
 .../druid/sql/calcite/CalciteWindowQueryTest.java  |  46 ++
 .../wikipediaAggregationsMultipleOrdering.sqlTest  |   2 +-
 .../window/wikipediaFramedAggregations.sqlTest     |   2 +-
 9 files changed, 101 insertions(+), 520 deletions(-)

diff --git a/processing/src/main/java/org/apache/druid/query/operator/window/WindowFrame.java b/processing/src/main/java/org/apache/druid/query/operator/window/WindowFrame.java
index b70df2c5203..fca50c25b28 100644
--- a/processing/src/main/java/org/apache/druid/query/operator/window/WindowFrame.java
+++ b/processing/src/main/java/org/apache/druid/query/operator/window/WindowFrame.java
@@ -161,9 +161,9 @@ public class WindowFrame
   public int getLowerOffsetClamped(int maxRows)
   {
     if (lowerUnbounded) {
-      return maxRows;
+      return -maxRows;
     }
-    return Math.min(maxRows, lowerOffset);
+    return Math.max(-maxRows, lowerOffset);
   }
 
   /**
diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/semantic/DefaultFramedOnHeapAggregatable.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/semantic/DefaultFramedOnHeapAggregatable.java
index 106fa9674a0..19bc6170c12 100644
--- a/processing/src/main/java/org/apache/druid/query/rowsandcols/semantic/DefaultFramedOnHeapAggregatable.java
+++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/semantic/DefaultFramedOnHeapAggregatable.java
@@ -29,7 +29,6 @@ import org.apache.druid.query.dimension.DimensionSpec;
 import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
 import org.apache.druid.query.operator.window.WindowFrame;
 import org.apache.druid.query.rowsandcols.RowsAndColumns;
-import org.apache.druid.query.rowsandcols.column.ConstantObjectColumn;
 import org.apache.druid.query.rowsandcols.column.ObjectArrayColumn;
 import org.apache.druid.segment.ColumnSelectorFactory;
 import org.apache.druid.segment.ColumnValueSelector;
@@ -61,34 +60,15 @@ public class DefaultFramedOnHeapAggregatable implements FramedOnHeapAggregatable
       AggregatorFactory[] aggFactories
   )
   {
-    if (frame.isLowerUnbounded() && frame.isUpperUnbounded()) {
-      return computeUnboundedAggregates(aggFactories);
-    }
-
-    if (frame.getPeerType() == WindowFrame.PeerType.ROWS) {
-      if (frame.isLowerUnbounded()) {
-        return computeCumulativeAggregates(aggFactories, frame.getUpperOffset());
-      } else if (frame.isUpperUnbounded()) {
-        return computeReverseCumulativeAggregates(aggFactories, frame.getLowerOffset());
-      } else {
-        final int numRows = rac.numRows();
-        int lowerOffset = frame.getLowerOffset();
-        int upperOffset = frame.getUpperOffset();
-
-        if (numRows < lowerOffset + upperOffset + 1) {
-          // In this case, there are not enough rows to completely build up the full window aperture before it needs to
-          // also start contracting the aperture because of the upper offset. So we use a method that specifically
-          // handles checks for both expanding and reducing the aperture on every iteration.
-          return aggregateWindowApertureInFlux(aggFactories, lowerOffset, upperOffset);
-        } else {
-          // In this case, there are 3 distinct phases that allow us to loop with less
-          // branches, so we have a method that specifically does that.
-          return aggregateWindowApertureWellBehaved(aggFactories, lowerOffset, upperOffset);
-        }
-      }
-    } else {
-      return computeGroupAggregates(aggFactories, frame);
+    Iterable<AggInterval> groupIterator = buildIteratorFor(rac, frame);
+    ResultPopulator resultRac = new ResultPopulator(aggFactories, rac.numRows());
+    AggIntervalCursor aggCursor = new AggIntervalCursor(rac, aggFactories);
+    for (AggInterval aggInterval : groupIterator) {
+      aggCursor.moveTo(aggInterval.inputRows);
+      resultRac.write(aggInterval.outputRows, aggCursor);
     }
+    resultRac.appendTo(rac);
+    return rac;
   }
 
   /**
@@ -124,22 +104,34 @@ public class DefaultFramedOnHeapAggregatable implements FramedOnHeapAggregatable
     }
   }
 
-  private RowsAndColumns computeGroupAggregates(
-      AggregatorFactory[] aggFactories,
-      WindowFrame frame)
+  public static Iterable<AggInterval> buildIteratorFor(AppendableRowsAndColumns rac, WindowFrame frame)
   {
-    Iterable<AggInterval> groupIterator = buildGroupIteratorFor(rac, frame);
-    ResultPopulator resultRac = new ResultPopulator(aggFactories, rac.numRows());
-    AggIntervalCursor aggCursor = new AggIntervalCursor(rac, aggFactories);
-    for (AggInterval aggInterval : groupIterator) {
-      aggCursor.moveTo(aggInterval.inputRows);
-      resultRac.write(aggInterval.outputRows, aggCursor);
+    int numRows = rac.numRows();
+    if (frame.getLowerOffsetClamped(numRows) == -numRows && frame.getUpperOffsetClamped(numRows) == numRows) {
+      return buildUnboundedIteratorFor(rac, frame);
+    } else if (frame.getPeerType() == WindowFrame.PeerType.RANGE) {
+      return buildGroupIteratorFor(rac, frame);
+    } else {
+      return buildRowIteratorFor(rac, frame);
     }
-    resultRac.appendTo(rac);
-    return rac;
   }
 
-  public static Iterable<AggInterval> buildGroupIteratorFor(AppendableRowsAndColumns rac, WindowFrame frame)
+  private static Iterable<AggInterval> buildUnboundedIteratorFor(AppendableRowsAndColumns rac, WindowFrame frame)
+  {
+    int[] groupBoundaries = new int[]{0, rac.numRows()};
+    return new GroupIteratorForWindowFrame(frame, groupBoundaries);
+  }
+
+  private static Iterable<AggInterval> buildRowIteratorFor(AppendableRowsAndColumns rac, WindowFrame frame)
+  {
+    int[] groupBoundaries = new int[rac.numRows() + 1];
+    for (int j = 0; j < groupBoundaries.length; j++) {
+      groupBoundaries[j] = j;
+    }
+    return new GroupIteratorForWindowFrame(frame, groupBoundaries);
+  }
+
+  private static Iterable<AggInterval> buildGroupIteratorFor(AppendableRowsAndColumns rac, WindowFrame frame)
   {
     int[] groupBoundaries = ClusteredGroupPartitioner.fromRAC(rac).computeBoundaries(frame.getOrderByColNames());
     return new GroupIteratorForWindowFrame(frame, groupBoundaries);
@@ -187,7 +179,7 @@ public class DefaultFramedOnHeapAggregatable implements FramedOnHeapAggregatable
                   groupToRowIndex(relativeGroupId(1))
               ),
               Interval.of(
-                  groupToRowIndex(relativeGroupId(-lowerOffset)),
+                  groupToRowIndex(relativeGroupId(lowerOffset)),
                   groupToRowIndex(relativeGroupId(upperOffset))
               )
           );
@@ -365,6 +357,10 @@ public class DefaultFramedOnHeapAggregatable implements FramedOnHeapAggregatable
         for (int i = currentRows.b; i < newRows.b; i++) {
           aggregate(i);
         }
+      } else if (currentRows.a > newRows.a && currentRows.b == newRows.b) {
+        for (int i = newRows.a; i < currentRows.a; i++) {
+          aggregate(i);
+        }
       } else {
         newAggregators();
         for (int i : newRows) {
@@ -390,428 +386,6 @@ public class DefaultFramedOnHeapAggregatable implements FramedOnHeapAggregatable
     }
   }
 
-  private AppendableRowsAndColumns computeUnboundedAggregates(AggregatorFactory[] aggFactories)
-  {
-    Aggregator[] aggs = new Aggregator[aggFactories.length];
-
-
-    AtomicInteger currRow = new AtomicInteger(0);
-    final ColumnSelectorFactory columnSelectorFactory = ColumnSelectorFactoryMaker.fromRAC(rac).make(currRow);
-
-    for (int i = 0; i < aggFactories.length; i++) {
-      aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-    }
-
-    int numRows = rac.numRows();
-    int rowId = currRow.get();
-    while (rowId < numRows) {
-      for (Aggregator agg : aggs) {
-        agg.aggregate();
-      }
-      rowId = currRow.incrementAndGet();
-    }
-
-    for (int i = 0; i < aggFactories.length; ++i) {
-      rac.addColumn(
-          aggFactories[i].getName(),
-          new ConstantObjectColumn(aggs[i].get(), numRows, aggFactories[i].getIntermediateType())
-      );
-      aggs[i].close();
-    }
-    return rac;
-  }
-
-  private AppendableRowsAndColumns computeCumulativeAggregates(AggregatorFactory[] aggFactories, int upperOffset)
-  {
-    int numRows = rac.numRows();
-    if (upperOffset > numRows) {
-      return computeUnboundedAggregates(aggFactories);
-    }
-
-
-    // We store the results in an Object array for convenience.  This is definitely sub-par from a memory management
-    // point of view as we should use native arrays when possible.  This will be fine for now, but it probably makes
-    // sense to look at optimizing this in the future.  That said, such an optimization might best come by having
-    // a specialized implementation of this interface against, say, a Frame object that can deal with arrays instead
-    // of trying to optimize this generic implementation.
-    Object[][] results = new Object[aggFactories.length][numRows];
-    int resultStorageIndex = 0;
-
-    AtomicInteger rowIdProvider = new AtomicInteger(0);
-    final ColumnSelectorFactory columnSelectorFactory = ColumnSelectorFactoryMaker.fromRAC(rac).make(rowIdProvider);
-
-    AggregatorFactory[] combiningFactories = new AggregatorFactory[aggFactories.length];
-    Aggregator[] aggs = new Aggregator[aggFactories.length];
-    for (int i = 0; i < aggFactories.length; i++) {
-      combiningFactories[i] = aggFactories[i].getCombiningFactory();
-      aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-    }
-
-    // If there is an upper offset, we accumulate those aggregations before starting to generate results
-    for (int i = 0; i < upperOffset; ++i) {
-      for (Aggregator agg : aggs) {
-        agg.aggregate();
-      }
-      rowIdProvider.incrementAndGet();
-    }
-
-    // Prime the results
-    if (rowIdProvider.get() < numRows) {
-      for (int i = 0; i < aggs.length; i++) {
-        aggs[i].aggregate();
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggs[i].get());
-        aggs[i].close();
-        aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-      }
-
-      ++resultStorageIndex;
-      rowIdProvider.incrementAndGet();
-    }
-
-    // From here out, we want to aggregate, peel off a row of results and then accumulate the aggregation
-    for (int rowId = rowIdProvider.get(); rowId < numRows; ++rowId) {
-      for (int i = 0; i < aggs.length; i++) {
-        aggs[i].aggregate();
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggs[i].get());
-        aggs[i].close();
-
-        // Use a combining aggregator to combine the result we just got with the result from the previous row
-        // This is a lot of hoops to jump through just to combine two values, but AggregatorFactory.combine
-        // allows for mutation of either of the arguments passed in, so it cannot be meaningfully used in this
-        // context.  Instead, we have to jump through these hoops to make sure that we are generating a new object.
-        // It would've been nice if the AggregatorFactory interface had methods that were more usable for this,
-        // but it doesn't so :shrug:
-        final CumulativeColumnSelectorFactory combiningFactory = new CumulativeColumnSelectorFactory(
-            aggFactories[i],
-            results[i],
-            resultStorageIndex - 1
-        );
-        final Aggregator combiningAgg = combiningFactories[i].factorize(combiningFactory);
-        combiningAgg.aggregate();
-        combiningFactory.increment();
-        combiningAgg.aggregate();
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(combiningAgg.get());
-        combiningAgg.close();
-
-        aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-      }
-
-      ++resultStorageIndex;
-      rowIdProvider.incrementAndGet();
-    }
-
-    // If we haven't filled up all of the results yet, there are no more rows, so just point the rest of the results
-    // at the last result that we generated
-    for (Object[] resultArr : results) {
-      Arrays.fill(resultArr, resultStorageIndex, resultArr.length, resultArr[resultStorageIndex - 1]);
-    }
-
-    return makeReturnRAC(aggFactories, results);
-  }
-
-  private AppendableRowsAndColumns computeReverseCumulativeAggregates(AggregatorFactory[] aggFactories, int lowerOffset)
-  {
-    int numRows = rac.numRows();
-    if (lowerOffset > numRows) {
-      return computeUnboundedAggregates(aggFactories);
-    }
-
-    // We store the results in an Object array for convenience.  This is definitely sub-par from a memory management
-    // point of view as we should use native arrays when possible.  This will be fine for now, but it probably makes
-    // sense to look at optimizing this in the future.  That said, such an optimization might best come by having
-    // a specialized implementation of this interface against, say, a Frame object that can deal with arrays instead
-    // of trying to optimize this generic implementation.
-    Object[][] results = new Object[aggFactories.length][numRows];
-    int resultStorageIndex = numRows - 1;
-
-    AtomicInteger rowIdProvider = new AtomicInteger(numRows - 1);
-    final ColumnSelectorFactory columnSelectorFactory = ColumnSelectorFactoryMaker.fromRAC(rac).make(rowIdProvider);
-
-    AggregatorFactory[] combiningFactories = new AggregatorFactory[aggFactories.length];
-    Aggregator[] aggs = new Aggregator[aggFactories.length];
-    for (int i = 0; i < aggFactories.length; i++) {
-      combiningFactories[i] = aggFactories[i].getCombiningFactory();
-      aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-    }
-
-    // If there is a lower offset, we accumulate those aggregations before starting to generate results
-    for (int i = 0; i < lowerOffset; ++i) {
-      for (Aggregator agg : aggs) {
-        agg.aggregate();
-      }
-      rowIdProvider.decrementAndGet();
-    }
-
-    // Prime the results
-    if (rowIdProvider.get() >= 0) {
-      for (int i = 0; i < aggs.length; i++) {
-        aggs[i].aggregate();
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggs[i].get());
-        aggs[i].close();
-        aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-      }
-
-      --resultStorageIndex;
-      rowIdProvider.decrementAndGet();
-    }
-
-    // From here out, we want to aggregate, peel off a row of results and then accumulate the aggregation
-    for (int rowId = rowIdProvider.get(); rowId >= 0; --rowId) {
-      for (int i = 0; i < aggs.length; i++) {
-        aggs[i].aggregate();
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggs[i].get());
-        aggs[i].close();
-
-        // Use a combining aggregator to combine the result we just got with the result from the previous row
-        // This is a lot of hoops to jump through just to combine two values, but AggregatorFactory.combine
-        // allows for mutation of either of the arguments passed in, so it cannot be meaningfully used in this
-        // context.  Instead, we have to jump through these hoops to make sure that we are generating a new object.
-        // It would've been nice if the AggregatorFactory interface had methods that were more usable for this,
-        // but it doesn't so :shrug:
-        final CumulativeColumnSelectorFactory combiningFactory = new CumulativeColumnSelectorFactory(
-            aggFactories[i],
-            results[i],
-            resultStorageIndex + 1
-        );
-        final Aggregator combiningAgg = combiningFactories[i].factorize(combiningFactory);
-        combiningAgg.aggregate();
-        combiningFactory.decrement();
-        combiningAgg.aggregate();
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(combiningAgg.get());
-        combiningAgg.close();
-
-        aggs[i] = aggFactories[i].factorize(columnSelectorFactory);
-      }
-
-      --resultStorageIndex;
-      rowIdProvider.decrementAndGet();
-    }
-
-    // If we haven't filled up all of the results yet, there are no more rows, so just point the rest of the results
-    // at the last result that we generated
-    for (Object[] resultArr : results) {
-      Arrays.fill(resultArr, 0, resultStorageIndex + 1, resultArr[resultStorageIndex + 1]);
-    }
-
-    return makeReturnRAC(aggFactories, results);
-  }
-
-  private AppendableRowsAndColumns aggregateWindowApertureWellBehaved(
-      AggregatorFactory[] aggFactories,
-      int lowerOffset,
-      int upperOffset
-  )
-  {
-    // There are 3 different phases of operation when we have more rows than our window size
-    // 1. Our window is not full, as we walk the rows we build up towards filling it
-    // 2. Our window is full, as we walk the rows we take a value off and add a new aggregation
-    // 3. We are nearing the end of the rows, we need to start shrinking the window aperture
-
-    int numRows = rac.numRows();
-    int windowSize = lowerOffset + upperOffset + 1;
-
-    // We store the results in an Object array for convenience.  This is definitely sub-par from a memory management
-    // point of view as we should use native arrays when possible.  This will be fine for now, but it probably makes
-    // sense to look at optimizing this in the future.  That said, such an optimization might best come by having
-    // a specialized implementation of this interface against, say, a Frame object that can deal with arrays instead
-    // of trying to optimize this generic implementation.
-    Object[][] results = new Object[aggFactories.length][numRows];
-    int resultStorageIndex = 0;
-
-    AtomicInteger rowIdProvider = new AtomicInteger(0);
-    final ColumnSelectorFactory columnSelectorFactory = ColumnSelectorFactoryMaker.fromRAC(rac).make(rowIdProvider);
-
-    // This is the number of aggregators to actually aggregate for the current row.
-    // Which also doubles as the nextIndex to roll through as we roll things in and out of the window
-    int nextIndex = lowerOffset + 1;
-
-    Aggregator[][] aggregators = new Aggregator[aggFactories.length][windowSize];
-    for (int i = 0; i < aggregators.length; i++) {
-      final AggregatorFactory aggFactory = aggFactories[i];
-      // instantiate the aggregators that need to be read on the first row.
-      for (int j = 0; j < nextIndex; j++) {
-        aggregators[i][j] = aggFactory.factorize(columnSelectorFactory);
-      }
-    }
-
-    // The first few rows will slowly build out the window to consume the upper-offset.  The window will not
-    // be full until we have walked upperOffset number of rows, so phase 1 runs until we have consumed
-    // upperOffset number of rows.
-    for (int upperIndex = 0; upperIndex < upperOffset; ++upperIndex) {
-      for (Aggregator[] aggregator : aggregators) {
-        for (int j = 0; j < nextIndex; ++j) {
-          aggregator[j].aggregate();
-        }
-      }
-
-      for (int i = 0; i < aggFactories.length; ++i) {
-        aggregators[i][nextIndex] = aggFactories[i].factorize(columnSelectorFactory);
-      }
-      ++nextIndex;
-      rowIdProvider.incrementAndGet();
-    }
-
-    // End Phase 1, Enter Phase 2.  At this point, nextIndex == windowSize, rowIdProvider is the same as
-    // upperOffset and the aggregators matrix is entirely non-null.  We need to iterate until our window has all of
-    // the aggregators in it to fill up the final result set.
-    int endResultStorageIndex = numRows - windowSize;
-    for (; resultStorageIndex < endResultStorageIndex; ++resultStorageIndex) {
-      for (Aggregator[] aggregator : aggregators) {
-        for (Aggregator value : aggregator) {
-          value.aggregate();
-        }
-      }
-
-      if (nextIndex == windowSize) {
-        // Wrap back around and start pruning from the beginning of the window
-        nextIndex = 0;
-      }
-
-      for (int i = 0; i < aggFactories.length; ++i) {
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggregators[i][nextIndex].get());
-        aggregators[i][nextIndex].close();
-        aggregators[i][nextIndex] = aggFactories[i].factorize(columnSelectorFactory);
-      }
-
-      ++nextIndex;
-      rowIdProvider.incrementAndGet();
-    }
-
-    if (nextIndex == windowSize) {
-      nextIndex = 0;
-    }
-
-    // End Phase 2, enter Phase 3.  At this point, our window has enough aggregators in it to fill up our final
-    // result set.  This means that for each new row that we complete, the window will "shrink" until we hit numRows,
-    // at which point we will collect anything yet remaining and be done.
-
-    if (nextIndex != 0) {
-      // Start by organizing the aggregators so that we are 0-indexed from nextIndex.  This trades off creating
-      // a new array of references in exchange for removing branches inside of the loop.  It also makes the logic
-      // simpler to understand.
-
-      Aggregator[][] reorganizedAggs = new Aggregator[aggFactories.length][windowSize];
-      for (int i = 0; i < aggFactories.length; i++) {
-        System.arraycopy(aggregators[i], nextIndex, reorganizedAggs[i], 0, windowSize - nextIndex);
-        System.arraycopy(aggregators[i], 0, reorganizedAggs[i], windowSize - nextIndex, nextIndex);
-      }
-      aggregators = reorganizedAggs;
-      nextIndex = 0;
-    }
-
-    for (int rowId = rowIdProvider.get(); rowId < numRows; ++rowId) {
-      for (Aggregator[] aggregator : aggregators) {
-        for (int j = nextIndex; j < aggregator.length; ++j) {
-          aggregator[j].aggregate();
-        }
-      }
-
-      for (int i = 0; i < aggFactories.length; ++i) {
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggregators[i][nextIndex].get());
-        aggregators[i][nextIndex].close();
-        aggregators[i][nextIndex] = null;
-      }
-
-      ++nextIndex;
-      ++resultStorageIndex;
-      rowIdProvider.incrementAndGet();
-    }
-
-    // End Phase 3, anything left in the window needs to be collected and put into our results
-    for (; nextIndex < windowSize; ++nextIndex) {
-      for (int i = 0; i < aggFactories.length; ++i) {
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggregators[i][nextIndex].get());
-        aggregators[i][nextIndex].close();
-        aggregators[i][nextIndex] = null;
-      }
-      ++resultStorageIndex;
-    }
-
-    return makeReturnRAC(aggFactories, results);
-  }
-
-  private AppendableRowsAndColumns aggregateWindowApertureInFlux(
-      AggregatorFactory[] aggFactories,
-      int lowerOffset,
-      int upperOffset
-  )
-  {
-    // In this case, we need to store a value for all items, so our windowSize is equivalent to the number of rows
-    // from the RowsAndColumns object that we are using.
-    int windowSize = rac.numRows();
-
-    // We store the results in an Object array for convenience.  This is definitely sub-par from a memory management
-    // point of view as we should use native arrays when possible.  This will be fine for now, but it probably makes
-    // sense to look at optimizing this in the future.  That said, such an optimization might best come by having
-    // a specialized implementation of this interface against, say, a Frame object that can deal with arrays instead
-    // of trying to optimize this generic implementation.
-    Object[][] results = new Object[aggFactories.length][windowSize];
-    int resultStorageIndex = 0;
-
-    AtomicInteger rowIdProvider = new AtomicInteger(0);
-    final ColumnSelectorFactory columnSelectorFactory = ColumnSelectorFactoryMaker.fromRAC(rac).make(rowIdProvider);
-
-    Aggregator[][] aggregators = new Aggregator[aggFactories.length][windowSize];
-    for (int i = 0; i < aggregators.length; i++) {
-      final AggregatorFactory aggFactory = aggFactories[i];
-      for (int j = 0; j < aggregators[i].length; j++) {
-        aggregators[i][j] = aggFactory.factorize(columnSelectorFactory);
-      }
-    }
-
-    // This is the index to stop at for the current window aperture
-    // The first row is used by all of the results for the lowerOffset num results, plus 1 for the "current row"
-    int stopIndex = Math.min(lowerOffset + 1, windowSize);
-
-    int startIndex = 0;
-    int rowId = rowIdProvider.get();
-    while (rowId < windowSize) {
-      for (Aggregator[] aggregator : aggregators) {
-        for (int j = startIndex; j < stopIndex; ++j) {
-          aggregator[j].aggregate();
-        }
-      }
-
-      if (rowId >= upperOffset) {
-        for (int i = 0; i < aggregators.length; ++i) {
-          results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggregators[i][startIndex].get());
-          aggregators[i][startIndex].close();
-          aggregators[i][startIndex] = null;
-        }
-
-        ++resultStorageIndex;
-        ++startIndex;
-      }
-
-      if (stopIndex < windowSize) {
-        ++stopIndex;
-      }
-      rowId = rowIdProvider.incrementAndGet();
-    }
-
-
-    for (; startIndex < windowSize; ++startIndex) {
-      for (int i = 0; i < aggregators.length; ++i) {
-        results[i][resultStorageIndex] = aggFactories[i].finalizeComputation(aggregators[i][startIndex].get());
-        aggregators[i][startIndex].close();
-        aggregators[i][startIndex] = null;
-      }
-      ++resultStorageIndex;
-    }
-
-    return makeReturnRAC(aggFactories, results);
-  }
-
-  private AppendableRowsAndColumns makeReturnRAC(AggregatorFactory[] aggFactories, Object[][] results)
-  {
-    for (int i = 0; i < aggFactories.length; ++i) {
-      rac.addColumn(
-          aggFactories[i].getName(), new ObjectArrayColumn(results[i], aggFactories[i].getIntermediateType())
-      );
-    }
-    return rac;
-  }
-
   private static class CumulativeColumnSelectorFactory implements ColumnSelectorFactory
   {
     private final ColumnCapabilitiesImpl columnCapabilities;
@@ -831,16 +405,6 @@ public class DefaultFramedOnHeapAggregatable implements FramedOnHeapAggregatable
           .setType(factory.getIntermediateType());
     }
 
-    public void increment()
-    {
-      ++index;
-    }
-
-    public void decrement()
-    {
-      --index;
-    }
-
     @Override
     @Nonnull
     public DimensionSelector makeDimensionSelector(@Nonnull DimensionSpec dimensionSpec)
diff --git a/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/FramedOnHeapAggregatableTest.java b/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/FramedOnHeapAggregatableTest.java
index d00e12b3b66..d5b11f7a612 100644
--- a/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/FramedOnHeapAggregatableTest.java
+++ b/processing/src/test/java/org/apache/druid/query/rowsandcols/semantic/FramedOnHeapAggregatableTest.java
@@ -91,7 +91,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 1, false, 2, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -1, false, 2, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -143,7 +143,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 2, false, 0, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -2, false, 0, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -169,7 +169,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 5, false, 7, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -5, false, 7, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -197,7 +197,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 5, false, 1, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -5, false, 1, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -225,7 +225,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 5, false, 0, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -5, false, 0, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -253,7 +253,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 1, false, 7, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -1, false, 7, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -337,7 +337,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     FramedOnHeapAggregatable agger = FramedOnHeapAggregatable.fromRAC(rac);
 
     final RowsAndColumns results = agger.aggregateAll(
-        new WindowFrame(WindowFrame.PeerType.ROWS, false, 5, false, 0, null),
+        new WindowFrame(WindowFrame.PeerType.ROWS, false, -5, false, 0, null),
         new AggregatorFactory[]{
             new LongSumAggregatorFactory("sumFromLong", "intCol"),
             new DoubleMaxAggregatorFactory("maxFromInt", "intCol"),
@@ -479,7 +479,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     WindowFrame frame = new WindowFrame(
         PeerType.RANGE,
         false,
-        1,
+        -1,
         false,
         0,
         Collections.singletonList(ColumnWithDirection.ascending("c1"))
@@ -517,7 +517,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     WindowFrame frame = new WindowFrame(
         PeerType.RANGE,
         false,
-        1,
+        -1,
         false,
         1,
         Collections.singletonList(ColumnWithDirection.ascending("c1"))
@@ -537,7 +537,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     WindowFrame frame = new WindowFrame(
         PeerType.RANGE,
         false,
-        1,
+        -1,
         false,
         1,
         Collections.singletonList(ColumnWithDirection.ascending("c1"))
@@ -556,7 +556,7 @@ public class FramedOnHeapAggregatableTest extends SemanticTestBase
     WindowFrame frame = new WindowFrame(
         PeerType.RANGE,
         false,
-        1,
+        -1,
         false,
         2,
         Collections.singletonList(ColumnWithDirection.ascending("c1"))
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java
index a6319c40f66..d399786676c 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java
@@ -135,16 +135,6 @@ public class DruidSqlValidator extends BaseDruidSqlValidator
       );
     }
 
-    if (isPrecedingOrFollowing(lowerBound) &&
-        isPrecedingOrFollowing(upperBound) &&
-        lowerBound.getKind() == upperBound.getKind()) {
-      // this limitation can be lifted when https://github.com/apache/druid/issues/15739 is addressed
-      throw buildCalciteContextException(
-          "Query bounds with both lower and upper bounds as PRECEDING or FOLLOWING is not supported.",
-          windowOrId
-      );
-    }
-
     boolean hasBounds = lowerBound != null || upperBound != null;
     if (call.getKind() == SqlKind.NTILE && hasBounds) {
       throw buildCalciteContextException(
@@ -162,7 +152,6 @@ public class DruidSqlValidator extends BaseDruidSqlValidator
       }
     }
 
-
     if (plannerContext.queryContext().isWindowingStrictValidation()) {
       if (!targetWindow.isRows() &&
           (!isUnboundedOrCurrent(lowerBound) || !isUnboundedOrCurrent(upperBound))) {
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java
index 4f0f0eda21b..afd775ef4ee 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Windowing.java
@@ -466,7 +466,9 @@ public class Windowing
       if (bound.isUnbounded() || bound.isCurrentRow()) {
         return 0;
       }
-      return getConstant(((RexInputRef) bound.getOffset()).getIndex());
+
+      final int value = getConstant(((RexInputRef) bound.getOffset()).getIndex());
+      return bound.isPreceding() ? -value : value;
     }
 
     private int getConstant(int refIndex)
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index 9a0a0318210..ec40fb3f871 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -15561,26 +15561,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
     assertThat(e, invalidSqlIs("Window frames with expression based lower/upper bounds are not supported. (line [1], column [31])"));
   }
 
-
-  @Test
-  public void testUnSupportedWindowBoundTypes()
-  {
-    assumeFeatureAvailable(EngineFeature.WINDOW_FUNCTIONS);
-
-    DruidException e;
-    e = assertThrows(DruidException.class, () -> testBuilder()
-        .queryContext(ImmutableMap.of(PlannerContext.CTX_ENABLE_WINDOW_FNS, true))
-        .sql("SELECT dim1,ROW_NUMBER() OVER (ORDER BY dim1 ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) from druid.foo")
-        .run());
-    assertThat(e, invalidSqlIs("Query bounds with both lower and upper bounds as PRECEDING or FOLLOWING is not supported. (line [1], column [31])"));
-
-    e = assertThrows(DruidException.class, () -> testBuilder()
-        .queryContext(ImmutableMap.of(PlannerContext.CTX_ENABLE_WINDOW_FNS, true))
-        .sql("SELECT dim1,ROW_NUMBER() OVER (ORDER BY dim1 ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) from druid.foo")
-        .run());
-    assertThat(e, invalidSqlIs("Query bounds with both lower and upper bounds as PRECEDING or FOLLOWING is not supported. (line [1], column [31])"));
-  }
-
   @Test
   public void testNtileNotSupportedWithFrame()
   {
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java
index f5e161b3a6a..cdc4bc9cbf9 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteWindowQueryTest.java
@@ -278,6 +278,52 @@ public class CalciteWindowQueryTest extends BaseCalciteQueryTest
         .run();
   }
 
+  @Test
+  public void testWindowAllBoundsCombination()
+  {
+    testBuilder()
+        .sql("select\n"
+             + "cityName,\n"
+             + "count(*) over (partition by cityName order by countryName rows between unbounded preceding and 1 preceding) c1,\n"
+             + "count(*) over (partition by cityName order by countryName rows between unbounded preceding and current row) c2,\n"
+             + "count(*) over (partition by cityName order by countryName rows between unbounded preceding and 1 following) c3,\n"
+             + "count(*) over (partition by cityName order by countryName rows between unbounded preceding and unbounded following) c4,\n"
+             + "count(*) over (partition by cityName order by countryName rows between 3 preceding and 1 preceding) c5,\n"
+             + "count(*) over (partition by cityName order by countryName rows between 1 preceding and current row) c6,\n"
+             + "count(*) over (partition by cityName order by countryName rows between 1 preceding and 1 FOLLOWING) c7,\n"
+             + "count(*) over (partition by cityName order by countryName rows between 1 preceding and unbounded FOLLOWING) c8,\n"
+             + "count(*) over (partition by cityName order by countryName rows between 1 FOLLOWING and unbounded FOLLOWING) c9,\n"
+             + "count(*) over (partition by cityName order by countryName rows between 1 FOLLOWING and 3 FOLLOWING) c10,\n"
+             + "count(*) over (partition by cityName order by countryName rows between current row and 1 following) c11,\n"
+             + "count(*) over (partition by cityName order by countryName rows between current row and unbounded following) c12\n"
+             + "from wikipedia\n"
+             + "where cityName in ('Vienna', 'Seoul')\n"
+             + "group by countryName, cityName, added")
+        .queryContext(ImmutableMap.of(
+            PlannerContext.CTX_ENABLE_WINDOW_FNS, true,
+            QueryContexts.ENABLE_DEBUG, true
+        ))
+        .expectedResults(ImmutableList.of(
+            new Object[]{"Seoul", 0L, 1L, 2L, 13L, 0L, 1L, 2L, 13L, 12L, 3L, 2L, 13L},
+            new Object[]{"Seoul", 1L, 2L, 3L, 13L, 1L, 2L, 3L, 13L, 11L, 3L, 2L, 12L},
+            new Object[]{"Seoul", 2L, 3L, 4L, 13L, 2L, 2L, 3L, 12L, 10L, 3L, 2L, 11L},
+            new Object[]{"Seoul", 3L, 4L, 5L, 13L, 3L, 2L, 3L, 11L, 9L, 3L, 2L, 10L},
+            new Object[]{"Seoul", 4L, 5L, 6L, 13L, 3L, 2L, 3L, 10L, 8L, 3L, 2L, 9L},
+            new Object[]{"Seoul", 5L, 6L, 7L, 13L, 3L, 2L, 3L, 9L, 7L, 3L, 2L, 8L},
+            new Object[]{"Seoul", 6L, 7L, 8L, 13L, 3L, 2L, 3L, 8L, 6L, 3L, 2L, 7L},
+            new Object[]{"Seoul", 7L, 8L, 9L, 13L, 3L, 2L, 3L, 7L, 5L, 3L, 2L, 6L},
+            new Object[]{"Seoul", 8L, 9L, 10L, 13L, 3L, 2L, 3L, 6L, 4L, 3L, 2L, 5L},
+            new Object[]{"Seoul", 9L, 10L, 11L, 13L, 3L, 2L, 3L, 5L, 3L, 3L, 2L, 4L},
+            new Object[]{"Seoul", 10L, 11L, 12L, 13L, 3L, 2L, 3L, 4L, 2L, 2L, 2L, 3L},
+            new Object[]{"Seoul", 11L, 12L, 13L, 13L, 3L, 2L, 3L, 3L, 1L, 1L, 2L, 2L},
+            new Object[]{"Seoul", 12L, 13L, 13L, 13L, 3L, 2L, 2L, 2L, 0L, 0L, 1L, 1L},
+            new Object[]{"Vienna", 0L, 1L, 2L, 3L, 0L, 1L, 2L, 3L, 2L, 2L, 2L, 3L},
+            new Object[]{"Vienna", 1L, 2L, 3L, 3L, 1L, 2L, 3L, 3L, 1L, 1L, 2L, 2L},
+            new Object[]{"Vienna", 2L, 3L, 3L, 3L, 2L, 2L, 2L, 2L, 0L, 0L, 1L, 1L}
+        ))
+        .run();
+  }
+
   private WindowOperatorQuery getWindowOperatorQuery(List<Query<?>> queries)
   {
     assertEquals(1, queries.size());
diff --git a/sql/src/test/resources/calcite/tests/window/wikipediaAggregationsMultipleOrdering.sqlTest b/sql/src/test/resources/calcite/tests/window/wikipediaAggregationsMultipleOrdering.sqlTest
index 221f7a52bcd..b8da0dbd658 100644
--- a/sql/src/test/resources/calcite/tests/window/wikipediaAggregationsMultipleOrdering.sqlTest
+++ b/sql/src/test/resources/calcite/tests/window/wikipediaAggregationsMultipleOrdering.sqlTest
@@ -18,7 +18,7 @@ expectedOperators:
       frame:
         peerType: "RANGE"
         lowUnbounded: false
-        lowOffset: 3
+        lowOffset: -3
         uppUnbounded: false
         uppOffset: 2
         orderBy:
diff --git a/sql/src/test/resources/calcite/tests/window/wikipediaFramedAggregations.sqlTest b/sql/src/test/resources/calcite/tests/window/wikipediaFramedAggregations.sqlTest
index 12e5736951b..07cfadfb680 100644
--- a/sql/src/test/resources/calcite/tests/window/wikipediaFramedAggregations.sqlTest
+++ b/sql/src/test/resources/calcite/tests/window/wikipediaFramedAggregations.sqlTest
@@ -17,7 +17,7 @@ expectedOperators:
       frame:
           peerType: "RANGE"
           lowUnbounded: false
-          lowOffset: 3
+          lowOffset: -3
           uppUnbounded: false
           uppOffset: 2
           orderBy: [ {column: "d1", direction: ASC} ]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org