You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/10/05 09:48:18 UTC

[arrow] 39/45: ARROW-6736: [Rust] [DataFusion] Evaluate the input to the aggregate expression just once per batch

This is an automated email from the ASF dual-hosted git repository.

kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git

commit c3619b0fe0a278ea5df5cf27c9d1aced1797db31
Author: Andy Grove <an...@gmail.com>
AuthorDate: Thu Oct 3 18:10:19 2019 -0600

    ARROW-6736: [Rust] [DataFusion] Evaluate the input to the aggregate expression just once per batch
    
    The current implementation of aggregate expressions in the new physical plan had a flaw where the input to the aggregate expression was repeatedly being evaluated (once per row instead of once per batch). This PR fixes this.
    
    Closes #5542 from andygrove/ARROW-6736 and squashes the following commits:
    
    f0fadafb1 <Andy Grove> Evaluate the input to the aggregate expression just once per batch
    
    Authored-by: Andy Grove <an...@gmail.com>
    Signed-off-by: Andy Grove <an...@gmail.com>
---
 .../src/execution/physical_plan/expressions.rs     | 15 ++++++++---
 .../src/execution/physical_plan/hash_aggregate.rs  | 29 +++++++++++++++++++---
 rust/datafusion/src/execution/physical_plan/mod.rs |  9 ++++++-
 3 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/rust/datafusion/src/execution/physical_plan/expressions.rs b/rust/datafusion/src/execution/physical_plan/expressions.rs
index 9e0741c..f63b40c 100644
--- a/rust/datafusion/src/execution/physical_plan/expressions.rs
+++ b/rust/datafusion/src/execution/physical_plan/expressions.rs
@@ -106,6 +106,10 @@ impl AggregateExpr for Sum {
         }
     }
 
+    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef> {
+        self.expr.evaluate(batch)
+    }
+
     fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>> {
         Rc::new(RefCell::new(SumAccumulator {
             expr: self.expr.clone(),
@@ -149,8 +153,12 @@ struct SumAccumulator {
 }
 
 impl Accumulator for SumAccumulator {
-    fn accumulate(&mut self, batch: &RecordBatch, row_index: usize) -> Result<()> {
-        let array = self.expr.evaluate(batch)?;
+    fn accumulate(
+        &mut self,
+        batch: &RecordBatch,
+        array: &ArrayRef,
+        row_index: usize,
+    ) -> Result<()> {
         match self.expr.data_type(batch.schema())? {
             DataType::Int8 => {
                 sum_accumulate!(self, array, row_index, Int8Array, Int64, i64)
@@ -697,9 +705,10 @@ mod tests {
     fn do_sum(batch: &RecordBatch) -> Result<Option<ScalarValue>> {
         let sum = sum(col(0));
         let accum = sum.create_accumulator();
+        let input = sum.evaluate_input(batch)?;
         let mut accum = accum.borrow_mut();
         for i in 0..batch.num_rows() {
-            accum.accumulate(&batch, i)?;
+            accum.accumulate(&batch, &input, i)?;
         }
         accum.get_value()
     }
diff --git a/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs b/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs
index 491a81a..0f59361 100644
--- a/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs
+++ b/rust/datafusion/src/execution/physical_plan/hash_aggregate.rs
@@ -281,6 +281,13 @@ impl BatchIterator for GroupedHashAggregateIterator {
                 .map(|expr| expr.evaluate(&batch))
                 .collect::<Result<Vec<_>>>()?;
 
+            // evaluate the inputs to the aggregate expressions for this batch
+            let aggr_input_values = self
+                .aggr_expr
+                .iter()
+                .map(|expr| expr.evaluate_input(&batch))
+                .collect::<Result<Vec<_>>>()?;
+
             // iterate over each row in the batch
             for row in 0..batch.num_rows() {
                 // create grouping key for this row
@@ -290,7 +297,10 @@ impl BatchIterator for GroupedHashAggregateIterator {
                     Some(accumulators) => {
                         let _ = accumulators
                             .iter()
-                            .map(|accum| accum.borrow_mut().accumulate(&batch, row))
+                            .zip(aggr_input_values.iter())
+                            .map(|(accum, input)| {
+                                accum.borrow_mut().accumulate(&batch, input, row)
+                            })
                             .collect::<Result<Vec<_>>>()?;
                         Ok(true)
                     }
@@ -306,7 +316,10 @@ impl BatchIterator for GroupedHashAggregateIterator {
 
                     let _ = accumulators
                         .iter()
-                        .map(|accum| accum.borrow_mut().accumulate(&batch, row))
+                        .zip(aggr_input_values.iter())
+                        .map(|(accum, input)| {
+                            accum.borrow_mut().accumulate(&batch, input, row)
+                        })
                         .collect::<Result<Vec<_>>>()?;
 
                     map.insert(key.clone(), accumulators);
@@ -511,11 +524,21 @@ impl BatchIterator for HashAggregateIterator {
 
         // iterate over input and perform aggregation
         while let Some(batch) = input.next()? {
+            // evaluate the inputs to the aggregate expressions for this batch
+            let aggr_input_values = self
+                .aggr_expr
+                .iter()
+                .map(|expr| expr.evaluate_input(&batch))
+                .collect::<Result<Vec<_>>>()?;
+
             // iterate over each row in the batch
             for row in 0..batch.num_rows() {
                 let _ = accumulators
                     .iter()
-                    .map(|accum| accum.borrow_mut().accumulate(&batch, row))
+                    .zip(aggr_input_values.iter())
+                    .map(|(accum, input)| {
+                        accum.borrow_mut().accumulate(&batch, input, row)
+                    })
                     .collect::<Result<Vec<_>>>()?;
             }
         }
diff --git a/rust/datafusion/src/execution/physical_plan/mod.rs b/rust/datafusion/src/execution/physical_plan/mod.rs
index b820af8..08a2737 100644
--- a/rust/datafusion/src/execution/physical_plan/mod.rs
+++ b/rust/datafusion/src/execution/physical_plan/mod.rs
@@ -65,6 +65,8 @@ pub trait AggregateExpr: Send + Sync {
     fn name(&self) -> String;
     /// Get the data type of this expression, given the schema of the input
     fn data_type(&self, input_schema: &Schema) -> Result<DataType>;
+    /// Evaluate the expressioon being aggregated
+    fn evaluate_input(&self, batch: &RecordBatch) -> Result<ArrayRef>;
     /// Create an accumulator for this aggregate expression
     fn create_accumulator(&self) -> Rc<RefCell<dyn Accumulator>>;
     /// Create an aggregate expression for combining the results of accumulators from partitions.
@@ -76,7 +78,12 @@ pub trait AggregateExpr: Send + Sync {
 /// Aggregate accumulator
 pub trait Accumulator {
     /// Update the accumulator based on a row in a batch
-    fn accumulate(&mut self, batch: &RecordBatch, row_index: usize) -> Result<()>;
+    fn accumulate(
+        &mut self,
+        batch: &RecordBatch,
+        input: &ArrayRef,
+        row_index: usize,
+    ) -> Result<()>;
     /// Get the final value for the accumulator
     fn get_value(&self) -> Result<Option<ScalarValue>>;
 }