You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2025/05/12 17:49:35 UTC

(datafusion) branch asf-site updated: Publish built docs triggered by 547c4a75f12090e3a4d446daf4400e2586203049

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/asf-site by this push:
     new 05ad45c82e Publish built docs triggered by 547c4a75f12090e3a4d446daf4400e2586203049
05ad45c82e is described below

commit 05ad45c82e08bc127fa96658afff18e1d6388b24
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Mon May 12 17:49:31 2025 +0000

    Publish built docs triggered by 547c4a75f12090e3a4d446daf4400e2586203049
---
 .../library-user-guide/extending-operators.md.txt  | 41 +++++++++++++++-
 library-user-guide/extending-operators.html        | 57 +++++++++++++++++++++-
 searchindex.js                                     |  2 +-
 3 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/_sources/library-user-guide/extending-operators.md.txt b/_sources/library-user-guide/extending-operators.md.txt
index 631bdc6797..7a5f888848 100644
--- a/_sources/library-user-guide/extending-operators.md.txt
+++ b/_sources/library-user-guide/extending-operators.md.txt
@@ -19,4 +19,43 @@
 
 # Extending DataFusion's operators: custom LogicalPlan and Execution Plans
 
-Coming soon
+DataFusion supports extension of operators by transforming logical plan and execution plan through customized [optimizer rules](https://docs.rs/datafusion/latest/datafusion/optimizer/trait.OptimizerRule.html). This section will use the µWheel project to illustrate such capabilities.
+
+## About DataFusion µWheel
+
+[DataFusion µWheel](https://github.com/uwheel/datafusion-uwheel/tree/main) is a native DataFusion optimizer which improves query performance for time-based analytics through fast temporal aggregation and pruning using custom indices. The integration of µWheel into DataFusion is a joint effort with the DataFusion community.
+
+### Optimizing Logical Plan
+
+The `rewrite` function transforms logical plans by identifying temporal patterns and aggregation functions that match the stored wheel indices. When match is found, it queries the corresponding index to retrieve pre-computed aggregate values, stores these results in a [MemTable](https://docs.rs/datafusion/latest/datafusion/datasource/memory/struct.MemTable.html), and returns as a new `LogicalPlan::TableScan`. If no match is found, the original plan proceeds unchanged through DataFusion's [...]
+
+```rust
+,ignore
+fn rewrite(
+  &self,
+  plan: LogicalPlan,
+  _config: &dyn OptimizerConfig,
+) -> Result<Transformed<LogicalPlan>> {
+    // Attemps to rewrite a logical plan to a uwheel-based plan that either provides
+    // plan-time aggregates or skips execution based on min/max pruning.
+    if let Some(rewritten) = self.try_rewrite(&plan) {
+        Ok(Transformed::yes(rewritten))
+    } else {
+        Ok(Transformed::no(plan))
+    }
+}
+```
+
+```rust
+,ignore
+// Converts a uwheel aggregate result to a TableScan with a MemTable as source
+fn agg_to_table_scan(result: f64, schema: SchemaRef) -> Result<LogicalPlan> {
+  let data = Float64Array::from(vec![result]);
+  let record_batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(data)])?;
+  let df_schema = Arc::new(DFSchema::try_from(schema.clone())?);
+  let mem_table = MemTable::try_new(schema, vec![vec![record_batch]])?;
+  mem_table_as_table_scan(mem_table, df_schema)
+}
+```
+
+To get a deeper dive into the usage of the µWheel project, visit the [blog post](https://uwheel.rs/post/datafusion_uwheel/) by Max Meldrum.
diff --git a/library-user-guide/extending-operators.html b/library-user-guide/extending-operators.html
index 0e548599c8..71f5e9d525 100644
--- a/library-user-guide/extending-operators.html
+++ b/library-user-guide/extending-operators.html
@@ -521,9 +521,26 @@
               
               <div class="toc-item">
                 
+<div class="tocsection onthispage pt-5 pb-3">
+    <i class="fas fa-list"></i> On this page
+</div>
 
 <nav id="bd-toc-nav">
-    
+    <ul class="visible nav section-nav flex-column">
+ <li class="toc-h2 nav-item toc-entry">
+  <a class="reference internal nav-link" href="#about-datafusion-wheel">
+   About DataFusion µWheel
+  </a>
+  <ul class="nav section-nav flex-column">
+   <li class="toc-h3 nav-item toc-entry">
+    <a class="reference internal nav-link" href="#optimizing-logical-plan">
+     Optimizing Logical Plan
+    </a>
+   </li>
+  </ul>
+ </li>
+</ul>
+
 </nav>
               </div>
               
@@ -570,7 +587,43 @@
 -->
 <section id="extending-datafusion-s-operators-custom-logicalplan-and-execution-plans">
 <h1>Extending DataFusion’s operators: custom LogicalPlan and Execution Plans<a class="headerlink" href="#extending-datafusion-s-operators-custom-logicalplan-and-execution-plans" title="Link to this heading">¶</a></h1>
-<p>Coming soon</p>
+<p>DataFusion supports extension of operators by transforming logical plan and execution plan through customized <a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/optimizer/trait.OptimizerRule.html">optimizer rules</a>. This section will use the µWheel project to illustrate such capabilities.</p>
+<section id="about-datafusion-wheel">
+<h2>About DataFusion µWheel<a class="headerlink" href="#about-datafusion-wheel" title="Link to this heading">¶</a></h2>
+<p><a class="reference external" href="https://github.com/uwheel/datafusion-uwheel/tree/main">DataFusion µWheel</a> is a native DataFusion optimizer which improves query performance for time-based analytics through fast temporal aggregation and pruning using custom indices. The integration of µWheel into DataFusion is a joint effort with the DataFusion community.</p>
+<section id="optimizing-logical-plan">
+<h3>Optimizing Logical Plan<a class="headerlink" href="#optimizing-logical-plan" title="Link to this heading">¶</a></h3>
+<p>The <code class="docutils literal notranslate"><span class="pre">rewrite</span></code> function transforms logical plans by identifying temporal patterns and aggregation functions that match the stored wheel indices. When match is found, it queries the corresponding index to retrieve pre-computed aggregate values, stores these results in a <a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/datasource/memory/struct.MemTable.html">MemTable</a>, and returns  [...]
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="p">,</span><span class="n">ignore</span>
+<span class="k">fn</span><span class="w"> </span><span class="nf">rewrite</span><span class="p">(</span>
+<span class="w">  </span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span>
+<span class="w">  </span><span class="n">plan</span><span class="p">:</span><span class="w"> </span><span class="nc">LogicalPlan</span><span class="p">,</span>
+<span class="w">  </span><span class="n">_config</span><span class="p">:</span><span class="w"> </span><span class="kp">&amp;</span><span class="nc">dyn</span><span class="w"> </span><span class="n">OptimizerConfig</span><span class="p">,</span>
+<span class="p">)</span><span class="w"> </span><span class="p">-&gt;</span><span class="w"> </span><span class="nb">Result</span><span class="o">&lt;</span><span class="n">Transformed</span><span class="o">&lt;</span><span class="n">LogicalPlan</span><span class="o">&gt;&gt;</span><span class="w"> </span><span class="p">{</span>
+<span class="w">    </span><span class="c1">// Attemps to rewrite a logical plan to a uwheel-based plan that either provides</span>
+<span class="w">    </span><span class="c1">// plan-time aggregates or skips execution based on min/max pruning.</span>
+<span class="w">    </span><span class="k">if</span><span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="nb">Some</span><span class="p">(</span><span class="n">rewritten</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">try_rewrite</span><span class="p">(</span><span class="o">&amp;</span><span class="n">plan</span><span class="p">)</s [...]
+<span class="w">        </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Transformed</span><span class="p">::</span><span class="n">yes</span><span class="p">(</span><span class="n">rewritten</span><span class="p">))</span>
+<span class="w">    </span><span class="p">}</span><span class="w"> </span><span class="k">else</span><span class="w"> </span><span class="p">{</span>
+<span class="w">        </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Transformed</span><span class="p">::</span><span class="n">no</span><span class="p">(</span><span class="n">plan</span><span class="p">))</span>
+<span class="w">    </span><span class="p">}</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="p">,</span><span class="n">ignore</span>
+<span class="c1">// Converts a uwheel aggregate result to a TableScan with a MemTable as source</span>
+<span class="k">fn</span><span class="w"> </span><span class="nf">agg_to_table_scan</span><span class="p">(</span><span class="n">result</span><span class="p">:</span><span class="w"> </span><span class="kt">f64</span><span class="p">,</span><span class="w"> </span><span class="n">schema</span><span class="p">:</span><span class="w"> </span><span class="nc">SchemaRef</span><span class="p">)</span><span class="w"> </span><span class="p">-&gt;</span><span class="w"> </span><span class="nb" [...]
+<span class="w">  </span><span class="kd">let</span><span class="w"> </span><span class="n">data</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Float64Array</span><span class="p">::</span><span class="n">from</span><span class="p">(</span><span class="fm">vec!</span><span class="p">[</span><span class="n">result</span><span class="p">]);</span>
+<span class="w">  </span><span class="kd">let</span><span class="w"> </span><span class="n">record_batch</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">RecordBatch</span><span class="p">::</span><span class="n">try_new</span><span class="p">(</span><span class="n">schema</span><span class="p">.</span><span class="n">clone</span><span class="p">(),</span><span class="w"> </span><span class="fm">vec!</span><span class="p">[</span><span class=" [...]
+<span class="w">  </span><span class="kd">let</span><span class="w"> </span><span class="n">df_schema</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">Arc</span><span class="p">::</span><span class="n">new</span><span class="p">(</span><span class="n">DFSchema</span><span class="p">::</span><span class="n">try_from</span><span class="p">(</span><span class="n">schema</span><span class="p">.</span><span class="n">clone</span><span class="p">()) [...]
+<span class="w">  </span><span class="kd">let</span><span class="w"> </span><span class="n">mem_table</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">MemTable</span><span class="p">::</span><span class="n">try_new</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span><span class="w"> </span><span class="fm">vec!</span><span class="p">[</span><span class="fm">vec!</span><span class="p">[</span><span class="n">recor [...]
+<span class="w">  </span><span class="n">mem_table_as_table_scan</span><span class="p">(</span><span class="n">mem_table</span><span class="p">,</span><span class="w"> </span><span class="n">df_schema</span><span class="p">)</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+<p>To get a deeper dive into the usage of the µWheel project, visit the <a class="reference external" href="https://uwheel.rs/post/datafusion_uwheel/">blog post</a> by Max Meldrum.</p>
+</section>
+</section>
 </section>
 
 
diff --git a/searchindex.js b/searchindex.js
index 9618c44175..91898291ec 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles":{"!=":[[54,"op-neq"]],"!~":[[54,"op-re-not-match"]],"!~*":[[54,"op-re-not-match-i"]],"!~~":[[54,"id19"]],"!~~*":[[54,"id20"]],"#":[[54,"op-bit-xor"]],"%":[[54,"op-modulo"]],"&":[[54,"op-bit-and"]],"(relation, name) tuples in logical fields and logical columns are unique":[[12,"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]],"*":[[54,"op-multiply"]],"+":[[54,"op-plus"]],"-":[[54,"op-minus"]],"/":[[54,"op-divide"]],"2022 Q2":[[10,"q2"]] [...]
\ No newline at end of file
+Search.setIndex({"alltitles":{"!=":[[54,"op-neq"]],"!~":[[54,"op-re-not-match"]],"!~*":[[54,"op-re-not-match-i"]],"!~~":[[54,"id19"]],"!~~*":[[54,"id20"]],"#":[[54,"op-bit-xor"]],"%":[[54,"op-modulo"]],"&":[[54,"op-bit-and"]],"(relation, name) tuples in logical fields and logical columns are unique":[[12,"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]],"*":[[54,"op-multiply"]],"+":[[54,"op-plus"]],"-":[[54,"op-minus"]],"/":[[54,"op-divide"]],"2022 Q2":[[10,"q2"]] [...]
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org