[arrow-datafusion] branch master updated: Optimize filter executor in pull-based executor (#4421)

[xu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xudong963 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/master by this push:
     new 522a2a4ad Optimize filter executor in pull-based executor (#4421)
522a2a4ad is described below

commit 522a2a4adea48777f330cac074db94a4da3ea66b
Author: xudong.w <wx...@gmail.com>
AuthorDate: Wed Nov 30 19:38:33 2022 +0800

    Optimize filter executor in pull-based executor (#4421)
    
    * optimzie filter executor
    
    * Update datafusion/core/src/physical_plan/filter.rs
    
    Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
    
    Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
 datafusion/core/src/physical_plan/filter.rs | 33 ++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/datafusion/core/src/physical_plan/filter.rs b/datafusion/core/src/physical_plan/filter.rs
index 9207c96dd..ed48d7b7d 100644
--- a/datafusion/core/src/physical_plan/filter.rs
+++ b/datafusion/core/src/physical_plan/filter.rs
@@ -238,15 +238,32 @@ impl Stream for FilterExecStream {
         mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
     ) -> Poll<Option<Self::Item>> {
-        let poll = self.input.poll_next_unpin(cx).map(|x| match x {
-            Some(Ok(batch)) => {
-                let timer = self.baseline_metrics.elapsed_compute().timer();
-                let filtered_batch = batch_filter(&batch, &self.predicate);
-                timer.done();
-                Some(filtered_batch)
+        let poll;
+        loop {
+            match self.input.poll_next_unpin(cx) {
+                Poll::Ready(value) => match value {
+                    Some(Ok(batch)) => {
+                        let timer = self.baseline_metrics.elapsed_compute().timer();
+                        let filtered_batch = batch_filter(&batch, &self.predicate)?;
+                        // skip entirely filtered batches
+                        if filtered_batch.num_rows() == 0 {
+                            continue;
+                        }
+                        timer.done();
+                        poll = Poll::Ready(Some(Ok(filtered_batch)));
+                        break;
+                    }
+                    _ => {
+                        poll = Poll::Ready(value);
+                        break;
+                    }
+                },
+                Poll::Pending => {
+                    poll = Poll::Pending;
+                    break;
+                }
             }
-            other => other,
-        });
+        }
         self.baseline_metrics.record_poll(poll)
     }