You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/07/15 16:09:11 UTC
[arrow-site] branch asf-site updated: DataFusion 10.0.0 (#222)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/arrow-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new abdbffe4b5 DataFusion 10.0.0 (#222)
abdbffe4b5 is described below
commit abdbffe4b5dcb718ab95da95bdca1dd30bf0ff6e
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri Jul 15 10:09:07 2022 -0600
DataFusion 10.0.0 (#222)
---
datafusion/.buildinfo | 2 +-
datafusion/_modules/datafusion.html | 147 ++-----
datafusion/_modules/index.html | 138 ++-----
datafusion/_sources/cli/index.rst.txt | 49 +--
datafusion/_sources/community/communication.md.txt | 9 +-
datafusion/_sources/index.rst.txt | 11 +-
.../_sources/python/api/execution_context.rst.txt | 4 +-
datafusion/_sources/python/index.rst.txt | 64 ++-
.../_sources/specification/invariants.md.txt | 2 +-
.../output-field-name-semantic.md.txt | 12 +-
.../specification/quarterly_roadmap.md.txt | 84 ++--
.../_sources/specification/rfcs/template.md.txt | 2 +-
datafusion/_sources/specification/roadmap.md.txt | 6 +-
datafusion/_sources/user-guide/cli.md.txt | 14 +-
datafusion/_sources/user-guide/configs.md.txt | 43 ++
.../_sources/user-guide/example-usage.md.txt | 6 +-
datafusion/_sources/user-guide/faq.md.txt | 2 +-
datafusion/_sources/user-guide/library.md.txt | 58 ++-
.../user-guide/sql/aggregate_functions.md.txt | 62 +++
.../user-guide/sql/datafusion-functions.md.txt | 26 +-
datafusion/_sources/user-guide/sql/ddl.md.txt | 15 +
datafusion/_sources/user-guide/sql/index.rst.txt | 1 +
.../_sources/user-guide/sql/sql_status.md.txt | 15 +-
datafusion/_static/pygments.css | 2 +-
datafusion/cli/index.html | 258 ++++--------
datafusion/community/communication.html | 174 ++------
datafusion/genindex.html | 176 ++------
datafusion/index.html | 174 ++------
datafusion/objects.inv | Bin 1785 -> 3589 bytes
datafusion/py-modindex.html | 138 ++-----
datafusion/python/api.html | 124 +-----
datafusion/python/api/dataframe.html | 119 +-----
datafusion/python/api/execution_context.html | 131 ++----
datafusion/python/api/expression.html | 125 ++----
datafusion/python/api/functions.html | 119 +-----
.../python/generated/datafusion.DataFrame.html | 123 ++----
.../python/generated/datafusion.Expression.html | 119 +-----
.../python/generated/datafusion.functions.html | 119 +-----
datafusion/python/index.html | 233 +++++------
datafusion/search.html | 138 ++-----
datafusion/searchindex.js | 2 +-
datafusion/specification/invariants.html | 303 +++++---------
.../specification/output-field-name-semantic.html | 185 +++------
datafusion/specification/quarterly_roadmap.html | 358 +++++++---------
datafusion/specification/rfcs/template.html | 168 ++------
datafusion/specification/roadmap.html | 199 +++------
datafusion/user-guide/cli.html | 175 ++------
datafusion/user-guide/{faq.html => configs.html} | 228 ++++-------
datafusion/user-guide/example-usage.html | 169 ++------
datafusion/user-guide/faq.html | 157 ++-----
datafusion/user-guide/introduction.html | 153 ++-----
datafusion/user-guide/library.html | 233 +++++------
.../sql/{index.html => aggregate_functions.html} | 307 +++++++-------
.../user-guide/sql/datafusion-functions.html | 225 +++++------
datafusion/user-guide/sql/ddl.html | 225 ++++-------
datafusion/user-guide/sql/index.html | 155 ++-----
datafusion/user-guide/sql/select.html | 247 ++++-------
datafusion/user-guide/sql/sql_status.html | 450 +++++++++------------
58 files changed, 2293 insertions(+), 4690 deletions(-)
diff --git a/datafusion/.buildinfo b/datafusion/.buildinfo
index c17f8f8cd2..a7390a9531 100644
--- a/datafusion/.buildinfo
+++ b/datafusion/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 3a436d2b4c271af4299b5f7631ff709a
+config: 802457ee12bddef1a0ad59103600adc7
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/datafusion/_modules/datafusion.html b/datafusion/_modules/datafusion.html
index a52de5209e..eb34fc50d2 100644
--- a/datafusion/_modules/datafusion.html
+++ b/datafusion/_modules/datafusion.html
@@ -4,7 +4,7 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
- <title>datafusion — Arrow Datafusion documentation</title>
+ <title>datafusion — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -67,7 +67,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -115,29 +115,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -153,8 +141,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -176,7 +164,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -206,8 +194,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -227,6 +215,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -234,86 +227,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -321,7 +238,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -339,7 +256,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -348,7 +265,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -356,16 +273,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -433,12 +345,6 @@
<span class="kn">from</span> <span class="nn">abc</span> <span class="kn">import</span> <span class="n">ABCMeta</span><span class="p">,</span> <span class="n">abstractmethod</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span>
-<span class="k">try</span><span class="p">:</span>
- <span class="kn">import</span> <span class="nn">importlib.metadata</span> <span class="k">as</span> <span class="nn">importlib_metadata</span>
-<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
- <span class="kn">import</span> <span class="nn">importlib_metadata</span>
-
-
<span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span>
<span class="kn">from</span> <span class="nn">._internal</span> <span class="kn">import</span> <span class="p">(</span>
@@ -450,9 +356,6 @@
<span class="p">)</span>
-<span class="n">__version__</span> <span class="o">=</span> <span class="n">importlib_metadata</span><span class="o">.</span><span class="n">version</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
-
-
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
<span class="s2">"DataFrame"</span><span class="p">,</span>
<span class="s2">"ExecutionContext"</span><span class="p">,</span>
@@ -554,7 +457,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/_modules/index.html b/datafusion/_modules/index.html
index 2e1974c643..ab6afa50f7 100644
--- a/datafusion/_modules/index.html
+++ b/datafusion/_modules/index.html
@@ -4,7 +4,7 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
- <title>Overview: module code — Arrow Datafusion documentation</title>
+ <title>Overview: module code — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -67,7 +67,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -115,29 +115,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -153,8 +141,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -176,7 +164,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -206,8 +194,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -227,6 +215,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -234,86 +227,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -321,7 +238,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -339,7 +256,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -348,7 +265,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -356,16 +273,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -435,7 +347,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/_sources/cli/index.rst.txt b/datafusion/_sources/cli/index.rst.txt
index 89f7f5da76..c10db36dfd 100644
--- a/datafusion/_sources/cli/index.rst.txt
+++ b/datafusion/_sources/cli/index.rst.txt
@@ -23,10 +23,15 @@ The Arrow DataFusion CLI is a command-line interactive SQL utility that allows
queries to be executed against CSV and Parquet files. It is a convenient way to
try DataFusion out with your own data sources.
+Install and run using Cargo
+===========================
+
+The easiest way to install DataFusion CLI a spin is via `cargo install datafusion-cli`.
+
Install and run using Homebrew (on MacOS)
=========================================
-The easiest way to give DataFusion CLI a spin is via Homebrew (on MacOS). Install it as any other pre-built software like this:
+DataFusion CLI can also be installed via Homebrew (on MacOS). Install it as any other pre-built software like this:
.. code-block:: bash
@@ -41,26 +46,19 @@ The easiest way to give DataFusion CLI a spin is via Homebrew (on MacOS). Instal
datafusion-cli
-Run using Cargo
-===============
-
-Use the following commands to clone this repository and run the CLI. This will require the Rust toolchain to be installed. Rust can be installed from `https://rustup.rs <https://rustup.rs/>`_.
-
-.. code-block:: bash
-
- git clone https://github.com/apache/arrow-datafusion
- cd arrow-datafusion/datafusion-cli
- cargo run --release
-
Run using Docker
================
+There is no officially published Docker image for the DataFusion CLI, so it is necessary to build from source
+instead.
+
Use the following commands to clone this repository and build a Docker image containing the CLI tool. Note that there is :code:`.dockerignore` file in the root of the repository that may need to be deleted in order for this to work.
.. code-block:: bash
git clone https://github.com/apache/arrow-datafusion
+ git checkout 8.0.0
cd arrow-datafusion
docker build -f datafusion-cli/Dockerfile . --tag datafusion-cli
docker run -it -v $(your_data_location):/data datafusion-cli
@@ -71,25 +69,22 @@ Usage
.. code-block:: bash
- DataFusion 5.1.0-SNAPSHOT
- DataFusion is an in-memory query engine that uses Apache Arrow as the memory model. It supports executing SQL queries
- against CSV and Parquet files as well as querying directly against in-memory data.
+ Apache Arrow <de...@arrow.apache.org>
+ Command Line Client for DataFusion query engine and Ballista distributed computation engine.
USAGE:
- datafusion-cli [FLAGS] [OPTIONS]
-
- FLAGS:
- -h, --help Prints help information
- -q, --quiet Reduce printing other than the results and work quietly
- -V, --version Prints version information
+ datafusion-cli [OPTIONS]
OPTIONS:
- -c, --batch-size <batch-size> The batch size of each query, or use DataFusion default
- -p, --data-path <data-path> Path to your data, default to current directory
- -f, --file <file>... Execute commands from file(s), then exit
- --format <format> Output format [default: table] [possible values: csv, tsv, table, json, ndjson]
- --host <host> Ballista scheduler host
- --port <port> Ballista scheduler port
+ -c, --batch-size <BATCH_SIZE> The batch size of each query, or use DataFusion default
+ -f, --file <FILE>... Execute commands from file(s), then exit
+ --format <FORMAT> [default: table] [possible values: csv, tsv, table, json,
+ nd-json]
+ -h, --help Print help information
+ -p, --data-path <DATA_PATH> Path to your data, default to current directory
+ -q, --quiet Reduce printing other than the results and work quietly
+ -r, --rc <RC>... Run the provided files on startup instead of ~/.datafusionrc
+ -V, --version Print version information
Type `exit` or `quit` to exit the CLI.
diff --git a/datafusion/_sources/community/communication.md.txt b/datafusion/_sources/community/communication.md.txt
index b34b913c6f..d55bfb6234 100644
--- a/datafusion/_sources/community/communication.md.txt
+++ b/datafusion/_sources/community/communication.md.txt
@@ -31,7 +31,7 @@ conduct](https://www.apache.org/foundation/policies/conduct.html).
### Mailing list
We use arrow.apache.org's `dev@` mailing list for project management, release
-coorindation and design discussions
+coordination and design discussions
([subscribe](mailto:dev-subscribe@arrow.apache.org),
[unsubscribe](mailto:dev-unsubscribe@arrow.apache.org),
[archives](https://lists.apache.org/list.html?dev@arrow.apache.org)).
@@ -73,9 +73,10 @@ We will send a summary of all sync ups to the dev@arrow.apache.org mailing list.
## Contributing
Our source code is hosted on
-[GitHub](https://github.com/apache/arrow-datafusion). For developers new to
-the project, we have curated a
-[good-first-issue](https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)
+[GitHub](https://github.com/apache/arrow-datafusion). More information on contributing is in
+the [Contribution Guide](https://github.com/apache/arrow-datafusion/blob/master/CONTRIBUTING.md)
+, and we have curated a [good-first-issue]
+(https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)
list to help you get started. You can find datafusion's major designs in docs/source/specification.
We use GitHub issues for maintaining a queue of development work and as the
diff --git a/datafusion/_sources/index.rst.txt b/datafusion/_sources/index.rst.txt
index 5109e60338..8bced2d067 100644
--- a/datafusion/_sources/index.rst.txt
+++ b/datafusion/_sources/index.rst.txt
@@ -16,11 +16,11 @@
.. under the License.
=======================
-Apache Arrow Datafusion
+Apache Arrow DataFusion
=======================
-Table of content
-================
+Table of Contents
+=================
.. _toc.usage:
@@ -43,7 +43,7 @@ Table of content
user-guide/library
user-guide/cli
user-guide/sql/index
- user-guide/distributed/index
+ user-guide/configs
user-guide/faq
.. _toc.specs:
@@ -63,8 +63,7 @@ Table of content
:maxdepth: 1
:caption: README
- Datafusion <https://github.com/apache/arrow-datafusion/blob/master/README.md>
- Ballista <https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md>
+ DataFusion <https://github.com/apache/arrow-datafusion/blob/master/README.md>
.. _toc.community:
diff --git a/datafusion/_sources/python/api/execution_context.rst.txt b/datafusion/_sources/python/api/execution_context.rst.txt
index 7f8c840ca0..5b7e0f82f9 100644
--- a/datafusion/_sources/python/api/execution_context.rst.txt
+++ b/datafusion/_sources/python/api/execution_context.rst.txt
@@ -18,10 +18,10 @@
.. _api.execution_context:
.. currentmodule:: datafusion
-ExecutionContext
+SessionContext
================
.. autosummary::
:toctree: ../generated/
- ExecutionContext
+ SessionContext
diff --git a/datafusion/_sources/python/index.rst.txt b/datafusion/_sources/python/index.rst.txt
index 57ab8d16bf..167e66b9fe 100644
--- a/datafusion/_sources/python/index.rst.txt
+++ b/datafusion/_sources/python/index.rst.txt
@@ -27,7 +27,7 @@ It also allows you to use UDFs and UDAFs for complex operations.
The major advantage of this library over other execution engines is that this library achieves zero-copy between Python and its execution engine: there is no cost in using UDFs, UDAFs, and collecting the results to Python apart from having to lock the GIL when running those operations.
-Its query engine, DataFusion, is written in `Rust <https://www.rust-lang.org>`_), which makes strong assumptions about thread safety and lack of memory leaks.
+Its query engine, DataFusion, is written in `Rust <https://www.rust-lang.org>`_, which makes strong assumptions about thread safety and lack of memory leaks.
Technically, zero-copy is achieved via the `c data interface <https://arrow.apache.org/docs/format/CDataInterface.html>`_.
@@ -44,7 +44,7 @@ Simple usage:
import pyarrow
# create a context
- ctx = datafusion.ExecutionContext()
+ ctx = datafusion.SessionContext()
# create a RecordBatch and a new DataFrame from it
batch = pyarrow.RecordBatch.from_arrays(
@@ -66,6 +66,66 @@ Simple usage:
assert result.column(1) == pyarrow.array([-3, -3, -3])
+We can also execute a query against data stored in CSV
+
+.. code-block:: bash
+
+ echo "a,b\n1,4\n2,5\n3,6" > example.csv
+
+
+.. code-block:: python
+
+ import datafusion
+ from datafusion import functions as f
+ from datafusion import col
+ import pyarrow
+
+ # create a context
+ ctx = datafusion.SessionContext()
+
+ # register a CSV
+ ctx.register_csv('example', 'example.csv')
+
+ # create a new statement
+ df = ctx.table('example').select(
+ col("a") + col("b"),
+ col("a") - col("b"),
+ )
+
+ # execute and collect the first (and only) batch
+ result = df.collect()[0]
+
+ assert result.column(0) == pyarrow.array([5, 7, 9])
+ assert result.column(1) == pyarrow.array([-3, -3, -3])
+
+
+And how to execute a query against a CSV using SQL:
+
+
+.. code-block:: python
+
+ import datafusion
+ from datafusion import functions as f
+ from datafusion import col
+ import pyarrow
+
+ # create a context
+ ctx = datafusion.SessionContext()
+
+ # register a CSV
+ ctx.register_csv('example', 'example.csv')
+
+ # create a new statement via SQL
+ df = ctx.sql("SELECT a+b, a-b FROM example")
+
+ # execute and collect the first (and only) batch
+ result = df.collect()[0]
+
+ assert result.column(0) == pyarrow.array([5, 7, 9])
+ assert result.column(1) == pyarrow.array([-3, -3, -3])
+
+
+
UDFs
----
diff --git a/datafusion/_sources/specification/invariants.md.txt b/datafusion/_sources/specification/invariants.md.txt
index 17b7c1dbba..4309763060 100644
--- a/datafusion/_sources/specification/invariants.md.txt
+++ b/datafusion/_sources/specification/invariants.md.txt
@@ -315,7 +315,7 @@ optimize(physical_plan).schema === physical_plan.schema
```
This is used to ensure that plans can be optimized without jeopardizing future
-referencs of logical columns (name and index) or assumptions about their
+references of logical columns (name and index) or assumptions about their
schemas.
#### Responsibility
diff --git a/datafusion/_sources/specification/output-field-name-semantic.md.txt b/datafusion/_sources/specification/output-field-name-semantic.md.txt
index bc0813abd0..c866573447 100644
--- a/datafusion/_sources/specification/output-field-name-semantic.md.txt
+++ b/datafusion/_sources/specification/output-field-name-semantic.md.txt
@@ -17,11 +17,11 @@
under the License.
-->
-# Datafusion output field name semantic
+# DataFusion output field name semantic
This specification documents how field names in output record batches should be
generated based on given user queries. The filed name rules apply to
-Datafusion queries planned from both SQL queries and Dataframe APIs.
+DataFusion queries planned from both SQL queries and Dataframe APIs.
## Field name rules
@@ -66,7 +66,7 @@ FROM t1
JOIN t2 ON t1.id = t2.id
```
-Datafusion Arrow record batches output:
+DataFusion Arrow record batches output:
| id | a | id | b |
| --- | --- | --- | ----- |
@@ -95,7 +95,7 @@ Query:
SELECT ABS(t1.id), abs(-id) FROM t1;
```
-Datafusion Arrow record batches output:
+DataFusion Arrow record batches output:
| abs(t1.id) | abs((- t1.id)) |
| ---------- | -------------- |
@@ -138,7 +138,7 @@ Query:
SELECT t1.id + ABS(id), ABS(id * t1.id) FROM t1;
```
-Datafusion Arrow record batches output:
+DataFusion Arrow record batches output:
| t1.id + abs(t1.id) | abs(t1.id \* t1.id) |
| ------------------ | ------------------- |
@@ -181,7 +181,7 @@ Query:
SELECT 1, 2+5, 'foo_bar';
```
-Datafusion Arrow record batches output:
+DataFusion Arrow record batches output:
| 1 | (2 + 5) | foo_bar |
| --- | ------- | ------- |
diff --git a/datafusion/_sources/specification/quarterly_roadmap.md.txt b/datafusion/_sources/specification/quarterly_roadmap.md.txt
index 5bb805d7e7..94c7dd9e2c 100644
--- a/datafusion/_sources/specification/quarterly_roadmap.md.txt
+++ b/datafusion/_sources/specification/quarterly_roadmap.md.txt
@@ -21,52 +21,70 @@
A quarterly roadmap will be published to give the DataFusion community visibility into the priorities of the projects contributors. This roadmap is not binding.
-## 2022 Q1
+## 2022 Q2
### DataFusion Core
-- Publish official Arrow2 branch
-- Implementation of memory manager (i.e. to enable spilling to disk as needed)
+- IO Improvements
+ - Reading, registering, and writing more file formats from both DataFrame API and SQL
+ - Additional options for IO including partitioning and metadata support
+- Work Scheduling
+ - Improve predictability, observability and performance of IO and CPU-bound work
+ - Develop a more explicit story for managing parallelism during plan execution
+- Memory Management
+ - Add more operators for memory limited execution
+- Performance
+ - Incorporate row-format into operators such as aggregate
+ - Add row-format benchmarks
+ - Explore JIT-compiling complex expressions
+ - Explore LLVM for JIT, with inline Rust functions as the primary goal
+ - Improve performance of Sort and Merge using Row Format / JIT expressions
+- Documentation
+ - General improvements to DataFusion website
+ - Publish design documents
+- Streaming
+ - Create `StreamProvider` trait
-### Benchmarking
+### Ballista
-- Inclusion in Db-Benchmark with all quries covered
-- All TPCH queries covered
+- Make production ready
+ - Shuffle file cleanup
+ - Fill functional gaps between DataFusion and Ballista
+ - Improve task scheduling and data exchange efficiency
+ - Better error handling
+ - Task failure
+ - Executor lost
+ - Schedule restart
+ - Improve monitoring and logging
+ - Auto scaling support
+- Support for multi-scheduler deployments. Initially for resiliency and fault tolerance but ultimately to support sharding for scalability and more efficient caching.
+- Executor deployment grouping based on resource allocation
-### Performance Improvements
+### Extensions ([datafusion-contrib](https://github.com/datafusion-contrib]))
-- Predicate evaluation
-- Improve multi-column comparisons (that can't be vectorized at the moment)
-- Null constant support
+#### [DataFusion-Python](https://github.com/datafusion-contrib/datafusion-python)
-### New Features
+- Add missing functionality to DataFrame and SessionContext
+- Improve documentation
-- Read JSON as table
-- Simplify DDL with Datafusion-Cli
-- Add Decimal128 data type and the attendant features such as Arrow Kernel and UDF support
-- Add new experimental e-graph based optimizer
+#### [DataFusion-S3](https://github.com/datafusion-contrib/datafusion-objectstore-s3)
-### Ballista
+- Create Python bindings to use with datafusion-python
-- Begin work on design documents and plan / priorities for development
-
-### Extensions ([datafusion-contrib](https://github.com/datafusion-contrib]))
+#### [DataFusion-Tui](https://github.com/datafusion-contrib/datafusion-tui)
-- Stable S3 support
-- Begin design discussions and prototyping of a stream provider
+- Create multiple SQL editors
+- Expose more Context and query metadata
+- Support new data sources
+ - BigTable, HDFS, HTTP APIs
-## Beyond 2022 Q1
+#### [DataFusion-BigTable](https://github.com/datafusion-contrib/datafusion-bigtable)
-There is no clear timeline for the below, but community members have expressed interest in working on these topics.
+- Python binding to use with datafusion-python
+- Timestamp range predicate pushdown
+- Multi-threaded partition aware execution
+- Production ready Rust SDK
-### DataFusion Core
-
-- Custom SQL support
-- Split DataFusion into multiple crates
-- Push based query execution and code generation
-
-### Ballista
+#### [DataFusion-Streams](https://github.com/datafusion-contrib/datafusion-streams)
-- Evolve architecture so that it can be deployed in a multi-tenant cloud native environment
-- Ensure Ballista is scalable, elastic, and stable for production usage
-- Develop distributed ML capabilities
+- Create experimental implementation of `StreamProvider` trait
diff --git a/datafusion/_sources/specification/rfcs/template.md.txt b/datafusion/_sources/specification/rfcs/template.md.txt
index 98704fd46f..a6f79fe939 100644
--- a/datafusion/_sources/specification/rfcs/template.md.txt
+++ b/datafusion/_sources/specification/rfcs/template.md.txt
@@ -27,7 +27,7 @@ Authors:
RFC PR: #
-Datafusion Issue: #
+DataFusion Issue: #
---
diff --git a/datafusion/_sources/specification/roadmap.md.txt b/datafusion/_sources/specification/roadmap.md.txt
index 76b2896aa7..e1d8ae9c0d 100644
--- a/datafusion/_sources/specification/roadmap.md.txt
+++ b/datafusion/_sources/specification/roadmap.md.txt
@@ -30,7 +30,7 @@ Arrow](https://arrow.apache.org/) project and governed by the Apache
Software Foundation governance model. These projects are entirely
driven by volunteers, and we welcome contributions for items not on
this roadmap. However, before submitting a large PR, we strongly
-suggest you start a coversation using a github issue or the
+suggest you start a conversation using a github issue or the
dev@arrow.apache.org mailing list to make review efficient and avoid
surprises.
@@ -44,8 +44,8 @@ to provide:
1. Best-in-class single node query performance
2. A Declarative SQL query interface compatible with PostgreSQL
3. A Dataframe API, similar to those offered by Pandas and Spark
-4. A Procedural API for programatically creating and running execution plans
-5. High performance, data race free, erogonomic extensibility points at at every layer
+4. A Procedural API for programmatically creating and running execution plans
+5. High performance, data race free, ergonomic extensibility points at at every layer
## Additional SQL Language Features
diff --git a/datafusion/_sources/user-guide/cli.md.txt b/datafusion/_sources/user-guide/cli.md.txt
index 1d2d6b79d4..93c6ecc415 100644
--- a/datafusion/_sources/user-guide/cli.md.txt
+++ b/datafusion/_sources/user-guide/cli.md.txt
@@ -19,8 +19,7 @@
# DataFusion Command-line Interface
-The DataFusion CLI allows SQL queries to be executed by an in-process DataFusion context, or by a distributed
-Ballista context.
+The DataFusion CLI allows SQL queries to be executed by an in-process DataFusion context.
```
USAGE:
@@ -36,8 +35,6 @@ OPTIONS:
-p, --data-path <data-path> Path to your data, default to current directory
-f, --file <file>... Execute commands from file(s), then exit
--format <format> Output format [default: table] [possible values: csv, tsv, table, json, ndjson]
- --host <host> Ballista scheduler host
- --port <port> Ballista scheduler port
```
## Example
@@ -51,7 +48,7 @@ $ echo "1,2" > data.csv
```bash
$ datafusion-cli
-DataFusion CLI v5.1.0-SNAPSHOT
+DataFusion CLI v8.0.0
> CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv';
0 rows in set. Query took 0.001 seconds.
@@ -65,12 +62,13 @@ DataFusion CLI v5.1.0-SNAPSHOT
1 row in set. Query took 0.017 seconds.
```
-## Ballista
+## DataFusion-Cli
-The DataFusion CLI can also connect to a Ballista scheduler for query execution.
+Build the `datafusion-cli`:
```bash
-datafusion-cli --host localhost --port 50050
+cd arrow-datafusion/datafusion-cli
+cargo build
```
## Cli commands
diff --git a/datafusion/_sources/user-guide/configs.md.txt b/datafusion/_sources/user-guide/configs.md.txt
new file mode 100644
index 0000000000..c899bafdb5
--- /dev/null
+++ b/datafusion/_sources/user-guide/configs.md.txt
@@ -0,0 +1,43 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!---
+This file was generated by the dev/update_config_docs.sh script.
+Do not edit it manually as changes will be overwritten.
+Instead, edit dev/update_config_docs.sh or the docstrings in datafusion/core/src/config.rs.
+-->
+
+# Configuration Settings
+
+The following configuration options can be passed to `SessionConfig` to control various aspects of query execution.
+
+For applications which do not expose `SessionConfig`, like `datafusion-cli`, these options may also be set via environment variables.
+To construct a session with options from the environment, use `SessionConfig::from_env`.
+The name of the environment variable is the option's key, transformed to uppercase and with periods replaced with underscores.
+For example, to configure `datafusion.execution.batch_size` you would set the `DATAFUSION_EXECUTION_BATCH_SIZE` environment variable.
+Values are parsed according to the [same rules used in casts from Utf8](https://docs.rs/arrow/latest/arrow/compute/kernels/cast/fn.cast.html).
+If the value in the environment variable cannot be cast to the type of the configuration option, the default value will be used instead and a warning emitted.
+Environment variables are read during `SessionConfig` initialisation so they must be set beforehand and will not affect running sessions.
+
+| key | type | default | description |
+| ----------------------------------------------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| datafusion.execution.batch_size | UInt64 | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would results in too much metadata memory consumption. |
+| datafusion.execution.coalesce_batches | Boolean | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting 'datafusion.execution.coalesce_target_batch_size'. |
+| datafusion.execution.coalesce_target_batch_size | UInt64 | 4096 | Target batch size when coalescing batches. Uses in conjunction with the configuration setting 'datafusion.execution.coalesce_batches'. |
+| datafusion.optimizer.filter_null_join_keys | Boolean | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. |
diff --git a/datafusion/_sources/user-guide/example-usage.md.txt b/datafusion/_sources/user-guide/example-usage.md.txt
index 77930260e0..de8cae2955 100644
--- a/datafusion/_sources/user-guide/example-usage.md.txt
+++ b/datafusion/_sources/user-guide/example-usage.md.txt
@@ -24,7 +24,7 @@
Add the following to your `Cargo.toml` file:
```toml
-datafusion = "7.0.0"
+datafusion = "8.0.0"
tokio = "1.0"
```
@@ -36,7 +36,7 @@ use datafusion::prelude::*;
#[tokio::main]
async fn main() -> datafusion::error::Result<()> {
// register the table
- let mut ctx = ExecutionContext::new();
+ let ctx = SessionContext::new();
ctx.register_csv("example", "tests/example.csv", CsvReadOptions::new()).await?;
// create a plan to run a SQL query
@@ -56,7 +56,7 @@ use datafusion::prelude::*;
#[tokio::main]
async fn main() -> datafusion::error::Result<()> {
// create the dataframe
- let mut ctx = ExecutionContext::new();
+ let ctx = SessionContext::new();
let df = ctx.read_csv("tests/example.csv", CsvReadOptions::new()).await?;
let df = df.filter(col("a").lt_eq(col("b")))?
diff --git a/datafusion/_sources/user-guide/faq.md.txt b/datafusion/_sources/user-guide/faq.md.txt
index f96306c1ce..16a8873fff 100644
--- a/datafusion/_sources/user-guide/faq.md.txt
+++ b/datafusion/_sources/user-guide/faq.md.txt
@@ -28,4 +28,4 @@ DataFusion is a library for executing queries in-process using the Apache Arrow
model and computational kernels. It is designed to run within a single process, using threads
for parallel query execution.
-Ballista is a distributed compute platform built on DataFusion.
+[Ballista](https://github.com/apache/arrow-ballista) is a distributed compute platform built on DataFusion.
diff --git a/datafusion/_sources/user-guide/library.md.txt b/datafusion/_sources/user-guide/library.md.txt
index f4c5083cfb..422c9d6d1e 100644
--- a/datafusion/_sources/user-guide/library.md.txt
+++ b/datafusion/_sources/user-guide/library.md.txt
@@ -19,6 +19,23 @@
# Using DataFusion as a library
+## Create a new project
+
+```shell
+cargo new hello_datafusion
+```
+
+```shell
+$ cd hello_datafusion
+$ tree .
+.
+├── Cargo.toml
+└── src
+ └── main.rs
+
+1 directory, 2 files
+```
+
## Default Configuration
DataFusion is [published on crates.io](https://crates.io/crates/datafusion), and is [well documented on docs.rs](https://docs.rs/datafusion/).
@@ -27,7 +44,29 @@ To get started, add the following to your `Cargo.toml` file:
```toml
[dependencies]
-datafusion = "5.1.0"
+datafusion = "8.0.0"
+```
+
+## Create a main function
+
+Update the main.rs file with your first datafusion application based on [Example usage](https://arrow.apache.org/datafusion/user-guide/example-usage.html)
+
+```rust
+use datafusion::prelude::*;
+
+#[tokio::main]
+async fn main() -> datafusion::error::Result<()> {
+ // register the table
+ let ctx = SessionContext::new();
+ ctx.register_csv("test", "<PATH_TO_YOUR_CSV_FILE>", CsvReadOptions::new()).await?;
+
+ // create a plan to run a SQL query
+ let df = ctx.sql("SELECT * FROM test").await?;
+
+ // execute and print results
+ df.show().await?;
+ Ok(())
+}
```
## Optimized Configuration
@@ -37,7 +76,7 @@ worth noting that using the settings in the `[profile.release]` section will sig
```toml
[dependencies]
-datafusion = { version = "5.0" , features = ["simd"]}
+datafusion = { version = "7.0" , features = ["simd"]}
tokio = { version = "^1.0", features = ["rt-multi-thread"] }
snmalloc-rs = "0.2"
@@ -49,12 +88,23 @@ codegen-units = 1
Then, in `main.rs.` update the memory allocator with the below after your imports:
```rust
+use datafusion::prelude::*;
+
#[global_allocator]
static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;
+
+async fn main() -> datafusion::error::Result<()> {
+ ...
+}
+```
+
+Finally, in order to build with the `simd` optimization `cargo nightly` is required.
+
+```shell
+rustup toolchain install nightly
```
-Finally, in order to build with the `simd` optimization `cargo nightly` is required. Based on the instruction
-set architecture you are building on you will want to configure the `target-cpu` as well, ideally
+Based on the instruction set architecture you are building on you will want to configure the `target-cpu` as well, ideally
with `native` or at least `avx2`.
```
diff --git a/datafusion/_sources/user-guide/sql/aggregate_functions.md.txt b/datafusion/_sources/user-guide/sql/aggregate_functions.md.txt
new file mode 100644
index 0000000000..d3472a7f53
--- /dev/null
+++ b/datafusion/_sources/user-guide/sql/aggregate_functions.md.txt
@@ -0,0 +1,62 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Aggregate Functions
+
+Aggregate functions operate on a set of values to compute a single result. Please refer to [PostgreSQL](https://www.postgresql.org/docs/current/functions-aggregate.html) for usage of standard SQL functions.
+
+## General
+
+- min
+- max
+- count
+- avg
+- sum
+- array_agg
+
+## Statistical
+
+- var / var_samp / var_pop
+- stddev / stddev_samp / stddev_pop
+- covar / covar_samp / covar_pop
+- corr
+
+## Approximate
+
+### approx_distinct
+
+`approx_distinct(x) -> uint64` returns the approximate number (HyperLogLog) of distinct input values
+
+### approx_median
+
+`approx_median(x) -> x` returns the approximate median of input values. it is an alias of `approx_percentile_cont(x, 0.5)`.
+
+### approx_percentile_cont
+
+`approx_percentile_cont(x, p) -> x` return the approximate percentile (TDigest) of input values, where `p` is a float64 between 0 and 1 (inclusive).
+
+It supports raw data as input and build Tdigest sketches during query time, and is approximately equal to `approx_percentile_cont_with_weight(x, 1, p)`.
+
+### approx_percentile_cont_with_weight
+
+`approx_percentile_cont_with_weight(x, w, p) -> x` returns the approximate percentile (TDigest) of input values with weight, where `w` is weight column expression and `p` is a float64 between 0 and 1 (inclusive).
+
+It supports raw data as input or pre-aggregated TDigest sketches, then builds or merges Tdigest sketches during query time. TDigest sketches are a list of centroid `(x, w)`, where `x` stands for mean and `w` stands for weight.
+
+It is suitable for low latency OLAP system where a streaming compute engine (e.g. Spark Streaming/Flink) pre-aggregates data to a data store, then queries using Datafusion.
diff --git a/datafusion/_sources/user-guide/sql/datafusion-functions.md.txt b/datafusion/_sources/user-guide/sql/datafusion-functions.md.txt
index 8431baf2a3..e37ba11e84 100644
--- a/datafusion/_sources/user-guide/sql/datafusion-functions.md.txt
+++ b/datafusion/_sources/user-guide/sql/datafusion-functions.md.txt
@@ -17,7 +17,7 @@
under the License.
-->
-# Datafusion-Specific Functions
+# DataFusion-Specific Functions
These SQL functions are specific to DataFusion, or they are well known and have functionality which is specific to DataFusion. Specifically, the `to_timestamp_xx()` functions exist due to Arrow's support for multiple timestamp resolutions.
@@ -84,3 +84,27 @@ Note that `CAST(.. AS Timestamp)` converts to Timestamps with Nanosecond resolut
- Other Timestamp() columns or values
Note that `CAST(.. AS Timestamp)` converts to Timestamps with Nanosecond resolution; this function is the only way to convert/cast to seconds resolution.
+
+## `extract`
+
+`extract(field FROM source)`
+
+- The `extract` function retrieves subfields such as year or hour from date/time values.
+ `source` must be a value expression of type timestamp, Data32, or Data64. `field` is an identifier that selects what field to extract from the source value.
+ The `extract` function returns values of type u32.
+ - `year` :`extract(year FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 2020`
+ - `month`:`extract(month FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 9`
+ - `week` :`extract(week FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 37`
+ - `day`: `extract(day FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 8`
+ - `hour`: `extract(hour FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 12`
+ - `minute`: `extract(minute FROM to_timestamp('2020-09-08T12:01:00+00:00')) -> 1`
+ - `second`: `extract(second FROM to_timestamp('2020-09-08T12:00:03+00:00')) -> 3`
+
+## `date_part`
+
+`date_part('field', source)`
+
+- The `date_part` function is modeled on the postgres equivalent to the SQL-standard function `extract`.
+ Note that here the field parameter needs to be a string value, not a name.
+ The valid field names for `date_part` are the same as for `extract`.
+ - `date_part('second', to_timestamp('2020-09-08T12:00:12+00:00')) -> 12`
diff --git a/datafusion/_sources/user-guide/sql/ddl.md.txt b/datafusion/_sources/user-guide/sql/ddl.md.txt
index eb10f40de5..75ec0f6cb0 100644
--- a/datafusion/_sources/user-guide/sql/ddl.md.txt
+++ b/datafusion/_sources/user-guide/sql/ddl.md.txt
@@ -55,6 +55,21 @@ WITH HEADER ROW
LOCATION '/path/to/aggregate_test_100.csv';
```
+If data sources are already partitioned in Hive style, `PARTITIONED BY` can be used for partition pruning.
+
+```
+/mnt/nyctaxi/year=2022/month=01/tripdata.parquet
+/mnt/nyctaxi/year=2021/month=12/tripdata.parquet
+/mnt/nyctaxi/year=2021/month=11/tripdata.parquet
+```
+
+```sql
+CREATE EXTERNAL TABLE taxi
+STORED AS PARQUET
+PARTITIONED BY (year, month)
+LOCATION '/mnt/nyctaxi';
+```
+
## CREATE MEMORY TABLE
Memory table can be created with query.
diff --git a/datafusion/_sources/user-guide/sql/index.rst.txt b/datafusion/_sources/user-guide/sql/index.rst.txt
index fc96acc873..f6d3a0bbed 100644
--- a/datafusion/_sources/user-guide/sql/index.rst.txt
+++ b/datafusion/_sources/user-guide/sql/index.rst.txt
@@ -24,4 +24,5 @@ SQL Reference
sql_status
select
ddl
+ aggregate_functions
DataFusion Functions <datafusion-functions>
diff --git a/datafusion/_sources/user-guide/sql/sql_status.md.txt b/datafusion/_sources/user-guide/sql/sql_status.md.txt
index 0df14e58a8..13a7e5e748 100644
--- a/datafusion/_sources/user-guide/sql/sql_status.md.txt
+++ b/datafusion/_sources/user-guide/sql/sql_status.md.txt
@@ -76,14 +76,19 @@
- [x] nullif
- Approximation functions
- [x] approx_distinct
+ - [x] approx_median
+ - [x] approx_percentile_cont
+ - [x] approx_percentile_cont_with_weight
- Common date/time functions
- [ ] Basic date functions
- [ ] Basic time functions
- [x] Basic timestamp functions
- - [x] [to_timestamp](docs/user-guide/book/sql/datafusion-functions.html#to_timestamp)
- - [x] [to_timestamp_millis](docs/user-guide/book/sql/datafusion-functions.html#to_timestamp_millis)
- - [x] [to_timestamp_micros](docs/user-guide/book/sql/datafusion-functions.html#to_timestamp_micros)
- - [x] [to_timestamp_seconds](docs/user-guide/book/sql/datafusion-functions.html#to_timestamp_seconds)
+ - [x] [to_timestamp](./datafusion-functions.md#to_timestamp)
+ - [x] [to_timestamp_millis](./datafusion-functions.md#to_timestamp_millis)
+ - [x] [to_timestamp_micros](./datafusion-functions.md#to_timestamp_micros)
+ - [x] [to_timestamp_seconds](./datafusion-functions.md#to_timestamp_seconds)
+ - [x] [extract](./datafusion-functions.md#extract)
+ - [x] [date_part](./datafusion-functions.md#date_part)
- nested functions
- [x] Array of columns
- [x] Schema Queries
@@ -135,7 +140,7 @@ DataFusion is designed to be extensible at all points. To that end, you can prov
- [x] User Defined `LogicalPlan` nodes
- [x] User Defined `ExecutionPlan` nodes
-## Rust Version Compatbility
+## Rust Version Compatibility
This crate is tested with the latest stable version of Rust. We do not currently test against other, older versions of the Rust compiler.
diff --git a/datafusion/_static/pygments.css b/datafusion/_static/pygments.css
index e76de2a67b..c5d3a660ac 100644
--- a/datafusion/_static/pygments.css
+++ b/datafusion/_static/pygments.css
@@ -54,7 +54,7 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
.highlight .nt { color: #204a87; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #000000 } /* Name.Variable */
.highlight .ow { color: #204a87; font-weight: bold } /* Operator.Word */
-.highlight .w { color: #f8f8f8; text-decoration: underline } /* Text.Whitespace */
+.highlight .w { color: #f8f8f8 } /* Text.Whitespace */
.highlight .mb { color: #0000cf; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000cf; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000cf; font-weight: bold } /* Literal.Number.Hex */
diff --git a/datafusion/cli/index.html b/datafusion/cli/index.html
index a4678d6eba..3770c5905b 100644
--- a/datafusion/cli/index.html
+++ b/datafusion/cli/index.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>DataFusion Command-line — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>DataFusion Command-line — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -414,13 +325,13 @@
<nav id="bd-toc-nav">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#install-and-run-using-homebrew-on-macos">
- Install and run using Homebrew (on MacOS)
+ <a class="reference internal nav-link" href="#install-and-run-using-cargo">
+ Install and run using Cargo
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#run-using-cargo">
- Run using Cargo
+ <a class="reference internal nav-link" href="#install-and-run-using-homebrew-on-macos">
+ Install and run using Homebrew (on MacOS)
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
@@ -471,14 +382,18 @@
<div>
- <section id="datafusion-command-line">
+ <div class="section" id="datafusion-command-line">
<h1>DataFusion Command-line<a class="headerlink" href="#datafusion-command-line" title="Permalink to this headline">¶</a></h1>
<p>The Arrow DataFusion CLI is a command-line interactive SQL utility that allows
queries to be executed against CSV and Parquet files. It is a convenient way to
try DataFusion out with your own data sources.</p>
-<section id="install-and-run-using-homebrew-on-macos">
+<div class="section" id="install-and-run-using-cargo">
+<h2>Install and run using Cargo<a class="headerlink" href="#install-and-run-using-cargo" title="Permalink to this headline">¶</a></h2>
+<p>The easiest way to install DataFusion CLI a spin is via <cite>cargo install datafusion-cli</cite>.</p>
+</div>
+<div class="section" id="install-and-run-using-homebrew-on-macos">
<h2>Install and run using Homebrew (on MacOS)<a class="headerlink" href="#install-and-run-using-homebrew-on-macos" title="Permalink to this headline">¶</a></h2>
-<p>The easiest way to give DataFusion CLI a spin is via Homebrew (on MacOS). Install it as any other pre-built software like this:</p>
+<p>DataFusion CLI can also be installed via Homebrew (on MacOS). Install it as any other pre-built software like this:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>brew install datafusion
<span class="c1"># ==> Downloading https://ghcr.io/v2/homebrew/core/datafusion/manifests/5.0.0</span>
<span class="c1"># ######################################################################## 100.0%</span>
@@ -491,85 +406,76 @@ try DataFusion out with your own data sources.</p>
datafusion-cli
</pre></div>
</div>
-</section>
-<section id="run-using-cargo">
-<h2>Run using Cargo<a class="headerlink" href="#run-using-cargo" title="Permalink to this headline">¶</a></h2>
-<p>Use the following commands to clone this repository and run the CLI. This will require the Rust toolchain to be installed. Rust can be installed from <a class="reference external" href="https://rustup.rs/">https://rustup.rs</a>.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>git clone https://github.com/apache/arrow-datafusion
-<span class="nb">cd</span> arrow-datafusion/datafusion-cli
-cargo run --release
-</pre></div>
</div>
-</section>
-<section id="run-using-docker">
+<div class="section" id="run-using-docker">
<h2>Run using Docker<a class="headerlink" href="#run-using-docker" title="Permalink to this headline">¶</a></h2>
+<p>There is no officially published Docker image for the DataFusion CLI, so it is necessary to build from source
+instead.</p>
<p>Use the following commands to clone this repository and build a Docker image containing the CLI tool. Note that there is <code class="code docutils literal notranslate"><span class="pre">.dockerignore</span></code> file in the root of the repository that may need to be deleted in order for this to work.</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>git clone https://github.com/apache/arrow-datafusion
+git checkout <span class="m">8</span>.0.0
<span class="nb">cd</span> arrow-datafusion
docker build -f datafusion-cli/Dockerfile . --tag datafusion-cli
docker run -it -v <span class="k">$(</span>your_data_location<span class="k">)</span>:/data datafusion-cli
</pre></div>
</div>
-</section>
-<section id="usage">
+</div>
+<div class="section" id="usage">
<h2>Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>DataFusion <span class="m">5</span>.1.0-SNAPSHOT
-DataFusion is an <span class="k">in</span>-memory query engine that uses Apache Arrow as the memory model. It supports executing SQL queries
-against CSV and Parquet files as well as querying directly against <span class="k">in</span>-memory data.
+<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>Apache Arrow <dev@arrow.apache.org>
+Command Line Client <span class="k">for</span> DataFusion query engine and Ballista distributed computation engine.
USAGE:
- datafusion-cli <span class="o">[</span>FLAGS<span class="o">]</span> <span class="o">[</span>OPTIONS<span class="o">]</span>
-
-FLAGS:
- -h, --help Prints <span class="nb">help</span> information
- -q, --quiet Reduce printing other than the results and work quietly
- -V, --version Prints version information
+ datafusion-cli <span class="o">[</span>OPTIONS<span class="o">]</span>
OPTIONS:
- -c, --batch-size <batch-size> The batch size of each query, or use DataFusion default
- -p, --data-path <data-path> Path to your data, default to current directory
- -f, --file <file>... Execute commands from file<span class="o">(</span>s<span class="o">)</span>, <span class="k">then</span> <span class="nb">exit</span>
- --format <format> Output format <span class="o">[</span>default: table<span class="o">]</span> <span class="o">[</span>possible values: csv, tsv, table, json, ndjson<span class="o">]</span>
- --host <host> Ballista scheduler host
- --port <port> Ballista scheduler port
+ -c, --batch-size <BATCH_SIZE> The batch size of each query, or use DataFusion default
+ -f, --file <FILE>... Execute commands from file<span class="o">(</span>s<span class="o">)</span>, <span class="k">then</span> <span class="nb">exit</span>
+ --format <FORMAT> <span class="o">[</span>default: table<span class="o">]</span> <span class="o">[</span>possible values: csv, tsv, table, json,
+ nd-json<span class="o">]</span>
+ -h, --help Print <span class="nb">help</span> information
+ -p, --data-path <DATA_PATH> Path to your data, default to current directory
+ -q, --quiet Reduce printing other than the results and work quietly
+ -r, --rc <RC>... Run the provided files on startup instead of ~/.datafusionrc
+ -V, --version Print version information
</pre></div>
</div>
<p>Type <cite>exit</cite> or <cite>quit</cite> to exit the CLI.</p>
-</section>
-<section id="registering-parquet-data-sources">
+</div>
+<div class="section" id="registering-parquet-data-sources">
<h2>Registering Parquet Data Sources<a class="headerlink" href="#registering-parquet-data-sources" title="Permalink to this headline">¶</a></h2>
<p>Parquet data sources can be registered by executing a <code class="code docutils literal notranslate"><span class="pre">CREATE</span> <span class="pre">EXTERNAL</span> <span class="pre">TABLE</span></code> SQL statement. It is not necessary to provide schema information for Parquet files.</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">taxi</span>
-<span class="n">STORED</span> <span class="k">AS</span> <span class="n">PARQUET</span>
-<span class="k">LOCATION</span> <span class="s1">'/mnt/nyctaxi/tripdata.parquet'</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">EXTERNAL</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">taxi</span><span class="w"></span>
+<span class="n">STORED</span><span class="w"> </span><span class="k">AS</span><span class="w"> </span><span class="n">PARQUET</span><span class="w"></span>
+<span class="k">LOCATION</span><span class="w"> </span><span class="s1">'/mnt/nyctaxi/tripdata.parquet'</span><span class="p">;</span><span class="w"></span>
</pre></div>
</div>
-</section>
-<section id="registering-csv-data-sources">
+</div>
+<div class="section" id="registering-csv-data-sources">
<h2>Registering CSV Data Sources<a class="headerlink" href="#registering-csv-data-sources" title="Permalink to this headline">¶</a></h2>
<p>CSV data sources can be registered by executing a <code class="code docutils literal notranslate"><span class="pre">CREATE</span> <span class="pre">EXTERNAL</span> <span class="pre">TABLE</span></code> SQL statement. It is necessary to provide schema information for CSV files since DataFusion does not automatically infer the schema when using SQL to query CSV files.</p>
-<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">test</span> <span class="p">(</span>
- <span class="n">c1</span> <span class="nb">VARCHAR</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c2</span> <span class="nb">INT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c3</span> <span class="nb">SMALLINT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c4</span> <span class="nb">SMALLINT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c5</span> <span class="nb">INT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c6</span> <span class="nb">BIGINT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c7</span> <span class="nb">SMALLINT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c8</span> <span class="nb">INT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c9</span> <span class="nb">BIGINT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c10</span> <span class="nb">VARCHAR</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c11</span> <span class="nb">FLOAT</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c12</span> <span class="n">DOUBLE</span> <span class="k">NOT</span> <span class="k">NULL</span><span class="p">,</span>
- <span class="n">c13</span> <span class="nb">VARCHAR</span> <span class="k">NOT</span> <span class="k">NULL</span>
-<span class="p">)</span>
-<span class="n">STORED</span> <span class="k">AS</span> <span class="n">CSV</span>
-<span class="k">WITH</span> <span class="n">HEADER</span> <span class="k">ROW</span>
-<span class="k">LOCATION</span> <span class="s1">'/path/to/aggregate_test_100.csv'</span><span class="p">;</span>
+<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">EXTERNAL</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">test</span><span class="w"> </span><span class="p">(</span><span class="w"></span>
+<span class="w"> </span><span class="n">c1</span><span class="w"> </span><span class="nb">VARCHAR</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c2</span><span class="w"> </span><span class="nb">INT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c3</span><span class="w"> </span><span class="nb">SMALLINT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c4</span><span class="w"> </span><span class="nb">SMALLINT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c5</span><span class="w"> </span><span class="nb">INT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c6</span><span class="w"> </span><span class="nb">BIGINT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c7</span><span class="w"> </span><span class="nb">SMALLINT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c8</span><span class="w"> </span><span class="nb">INT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c9</span><span class="w"> </span><span class="nb">BIGINT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c10</span><span class="w"> </span><span class="nb">VARCHAR</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c11</span><span class="w"> </span><span class="nb">FLOAT</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c12</span><span class="w"> </span><span class="n">DOUBLE</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="p">,</span><span class="w"></span>
+<span class="w"> </span><span class="n">c13</span><span class="w"> </span><span class="nb">VARCHAR</span><span class="w"> </span><span class="k">NOT</span><span class="w"> </span><span class="k">NULL</span><span class="w"></span>
+<span class="p">)</span><span class="w"></span>
+<span class="n">STORED</span><span class="w"> </span><span class="k">AS</span><span class="w"> </span><span class="n">CSV</span><span class="w"></span>
+<span class="k">WITH</span><span class="w"> </span><span class="n">HEADER</span><span class="w"> </span><span class="k">ROW</span><span class="w"></span>
+<span class="k">LOCATION</span><span class="w"> </span><span class="s1">'/path/to/aggregate_test_100.csv'</span><span class="p">;</span><span class="w"></span>
</pre></div>
</div>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -605,7 +511,7 @@ OPTIONS:
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/community/communication.html b/datafusion/community/communication.html
index b7fef4e4d9..9ffab590cf 100644
--- a/datafusion/community/communication.html
+++ b/datafusion/community/communication.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Communication — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Communication — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -69,7 +68,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -117,29 +116,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -155,8 +142,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -178,7 +165,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -208,8 +195,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -229,6 +216,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -236,86 +228,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -323,7 +239,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -341,7 +257,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -350,7 +266,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -358,16 +274,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -485,27 +396,27 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="communication">
+<div class="section" id="communication">
<h1>Communication<a class="headerlink" href="#communication" title="Permalink to this headline">¶</a></h1>
<p>We welcome participation from everyone and encourage you to join us, ask
questions, and get involved.</p>
<p>All participation in the Apache Arrow DataFusion project is governed by the
Apache Software Foundation’s <a class="reference external" href="https://www.apache.org/foundation/policies/conduct.html">code of
conduct</a>.</p>
-<section id="questions">
+<div class="section" id="questions">
<h2>Questions?<a class="headerlink" href="#questions" title="Permalink to this headline">¶</a></h2>
-<section id="mailing-list">
+<div class="section" id="mailing-list">
<h3>Mailing list<a class="headerlink" href="#mailing-list" title="Permalink to this headline">¶</a></h3>
<p>We use arrow.apache.org’s <code class="docutils literal notranslate"><span class="pre">dev@</span></code> mailing list for project management, release
-coorindation and design discussions
+coordination and design discussions
(<a class="reference external" href="mailto:dev-subscribe%40arrow.apache.org">subscribe</a>,
<a class="reference external" href="mailto:dev-unsubscribe%40arrow.apache.org">unsubscribe</a>,
<a class="reference external" href="https://lists.apache.org/list.html?dev@arrow.apache.org">archives</a>).</p>
<p>When emailing the dev list, please make sure to prefix the subject line with a
<code class="docutils literal notranslate"><span class="pre">[DataFusion]</span></code> tag, e.g. <code class="docutils literal notranslate"><span class="pre">"[DataFusion]</span> <span class="pre">New</span> <span class="pre">API</span> <span class="pre">for</span> <span class="pre">remote</span> <span class="pre">data</span> <span class="pre">sources"</span></code>, so
that the appropriate people in the Apache Arrow community notice the message.</p>
-</section>
-<section id="slack-and-discord">
+</div>
+<div class="section" id="slack-and-discord">
<h3>Slack and Discord<a class="headerlink" href="#slack-and-discord" title="Permalink to this headline">¶</a></h3>
<p>We use the official <a class="reference external" href="https://s.apache.org/slack-invite">ASF</a> Slack workspace
for informal discussions and coordination. This is a great place to meet other
@@ -515,8 +426,8 @@ contributors and get guidance on where to contribute. Join us in the
server (<a class="reference external" href="https://discord.gg/Qw5gKqHxUM">invite link</a>) in case you are not able
to join the Slack workspace. If you need an invite to the Slack workspace, you
can also ask for one in our Discord server.</p>
-</section>
-<section id="sync-up-video-calls">
+</div>
+<div class="section" id="sync-up-video-calls">
<h3>Sync up video calls<a class="headerlink" href="#sync-up-video-calls" title="Permalink to this headline">¶</a></h3>
<p>We have biweekly sync calls every other Thursdays at both 04:00 UTC
and 16:00 UTC (starting September 30, 2021) depending on if there are
@@ -530,20 +441,21 @@ for the video call link, add topics and to see what others plan to discuss.</p>
</ol>
<p>No decisions are made on the call and anything of substance will be discussed on this mailing list or in github issues / google docs.</p>
<p>We will send a summary of all sync ups to the dev@arrow.apache.org mailing list.</p>
-</section>
-</section>
-<section id="contributing">
+</div>
+</div>
+<div class="section" id="contributing">
<h2>Contributing<a class="headerlink" href="#contributing" title="Permalink to this headline">¶</a></h2>
<p>Our source code is hosted on
-<a class="reference external" href="https://github.com/apache/arrow-datafusion">GitHub</a>. For developers new to
-the project, we have curated a
-<a class="reference external" href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">good-first-issue</a>
+<a class="reference external" href="https://github.com/apache/arrow-datafusion">GitHub</a>. More information on contributing is in
+the <a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/CONTRIBUTING.md">Contribution Guide</a>
+, and we have curated a [good-first-issue]
+(https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)
list to help you get started. You can find datafusion’s major designs in docs/source/specification.</p>
<p>We use GitHub issues for maintaining a queue of development work and as the
public record. We often use Google docs, Github issues and pull requests for
quick and small design discussions. For major design change proposals, we encourage you to write a rfc.</p>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -572,7 +484,7 @@ quick and small design discussions. For major design change proposals, we encour
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/genindex.html b/datafusion/genindex.html
index a42dfba5ec..4aae4a07f6 100644
--- a/datafusion/genindex.html
+++ b/datafusion/genindex.html
@@ -5,7 +5,7 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
- <title>Index — Arrow Datafusion documentation</title>
+ <title>Index — Arrow DataFusion documentation</title>
<link href="_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -68,7 +68,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -116,29 +116,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -154,8 +142,8 @@
<a class="reference internal" href="python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -177,7 +165,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -207,8 +195,8 @@
<a class="reference internal" href="user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -228,6 +216,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -235,86 +228,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="user-guide/faq.html">
@@ -322,7 +239,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -340,7 +257,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -349,7 +266,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -357,16 +274,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -443,8 +355,6 @@
<li><a href="python/generated/datafusion.DataFrame.html#datafusion.DataFrame.__init__">__init__() (datafusion.DataFrame method)</a>
<ul>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.__init__">(datafusion.ExecutionContext method)</a>
-</li>
<li><a href="python/generated/datafusion.Expression.html#datafusion.Expression.__init__">(datafusion.Expression method)</a>
</li>
</ul></li>
@@ -499,8 +409,6 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="python/generated/datafusion.Expression.html#datafusion.Expression.cast">cast() (datafusion.Expression method)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.catalog">catalog() (datafusion.ExecutionContext method)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.ceil">ceil() (in module datafusion.functions)</a>
</li>
@@ -521,8 +429,6 @@
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.cos">cos() (in module datafusion.functions)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.count">count() (in module datafusion.functions)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.create_dataframe">create_dataframe() (datafusion.ExecutionContext method)</a>
</li>
</ul></td>
</tr></table>
@@ -544,14 +450,10 @@
<h2 id="E">E</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.empty_table">empty_table() (datafusion.ExecutionContext method)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext">ExecutionContext (class in datafusion)</a>
+ <li><a href="python/generated/datafusion.functions.html#datafusion.functions.exp">exp() (in module datafusion.functions)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="python/generated/datafusion.functions.html#datafusion.functions.exp">exp() (in module datafusion.functions)</a>
-</li>
<li><a href="python/generated/datafusion.Expression.html#datafusion.Expression">Expression (class in datafusion)</a>
</li>
</ul></td>
@@ -660,20 +562,12 @@
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.regexp_replace">regexp_replace() (in module datafusion.functions)</a>
</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.register_csv">register_csv() (datafusion.ExecutionContext method)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.register_parquet">register_parquet() (datafusion.ExecutionContext method)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.register_record_batches">register_record_batches() (datafusion.ExecutionContext method)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.register_udf">register_udf() (datafusion.ExecutionContext method)</a>
-</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.repeat">repeat() (in module datafusion.functions)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.replace">replace() (in module datafusion.functions)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.reverse">reverse() (in module datafusion.functions)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.right">right() (in module datafusion.functions)</a>
@@ -705,11 +599,11 @@
<li><a href="python/generated/datafusion.DataFrame.html#datafusion.DataFrame.show">show() (datafusion.DataFrame method)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.signum">signum() (in module datafusion.functions)</a>
-</li>
- <li><a href="python/generated/datafusion.functions.html#datafusion.functions.sin">sin() (in module datafusion.functions)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="python/generated/datafusion.functions.html#datafusion.functions.sin">sin() (in module datafusion.functions)</a>
+</li>
<li><a href="python/generated/datafusion.DataFrame.html#datafusion.DataFrame.sort">sort() (datafusion.DataFrame method)</a>
<ul>
@@ -717,8 +611,6 @@
</li>
</ul></li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.split_part">split_part() (in module datafusion.functions)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.sql">sql() (datafusion.ExecutionContext method)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.sqrt">sqrt() (in module datafusion.functions)</a>
</li>
@@ -736,18 +628,14 @@
<h2 id="T">T</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.table">table() (datafusion.ExecutionContext method)</a>
-</li>
- <li><a href="python/generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext.tables">tables() (datafusion.ExecutionContext method)</a>
-</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.tan">tan() (in module datafusion.functions)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.to_hex">to_hex() (in module datafusion.functions)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.to_timestamp">to_timestamp() (in module datafusion.functions)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.translate">translate() (in module datafusion.functions)</a>
</li>
<li><a href="python/generated/datafusion.functions.html#datafusion.functions.trim">trim() (in module datafusion.functions)</a>
@@ -794,7 +682,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/index.html b/datafusion/index.html
index 4f61c9318c..197e62ef92 100644
--- a/datafusion/index.html
+++ b/datafusion/index.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Apache Arrow Datafusion — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Apache Arrow DataFusion — Arrow DataFusion documentation</title>
<link href="_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -69,7 +68,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -117,29 +116,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -155,8 +142,8 @@
<a class="reference internal" href="python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -178,7 +165,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -208,8 +195,8 @@
<a class="reference internal" href="user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -229,6 +216,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -236,86 +228,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="user-guide/faq.html">
@@ -323,7 +239,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -341,7 +257,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -350,7 +266,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -358,16 +274,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -413,8 +324,8 @@
<nav id="bd-toc-nav">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#table-of-content">
- Table of content
+ <a class="reference internal nav-link" href="#table-of-contents">
+ Table of Contents
</a>
<ul class="nav section-nav flex-column">
</ul>
@@ -447,12 +358,12 @@
<div>
- <section id="apache-arrow-datafusion">
-<h1>Apache Arrow Datafusion<a class="headerlink" href="#apache-arrow-datafusion" title="Permalink to this headline">¶</a></h1>
-<section id="table-of-content">
-<h2>Table of content<a class="headerlink" href="#table-of-content" title="Permalink to this headline">¶</a></h2>
+ <div class="section" id="apache-arrow-datafusion">
+<h1>Apache Arrow DataFusion<a class="headerlink" href="#apache-arrow-datafusion" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="table-of-contents">
+<h2>Table of Contents<a class="headerlink" href="#table-of-contents" title="Permalink to this headline">¶</a></h2>
<div class="toctree-wrapper compound" id="toc-usage">
-<p><span class="caption-text">Supported Environments</span><a class="headerlink" href="#toc-usage" title="Permalink to this toctree">¶</a></p>
+<p class="caption"><span class="caption-text">Supported Environments</span><a class="headerlink" href="#toc-usage" title="Permalink to this toctree">¶</a></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://docs.rs/crate/datafusion/">Rust</a></li>
<li class="toctree-l1"><a class="reference internal" href="python/index.html">Python</a></li>
@@ -460,45 +371,44 @@
</ul>
</div>
<div class="toctree-wrapper compound" id="toc-guide">
-<p><span class="caption-text">User Guide</span><a class="headerlink" href="#toc-guide" title="Permalink to this toctree">¶</a></p>
+<p class="caption"><span class="caption-text">User Guide</span><a class="headerlink" href="#toc-guide" title="Permalink to this toctree">¶</a></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="user-guide/introduction.html">Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-guide/example-usage.html">Example Usage</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-guide/library.html">Using DataFusion as a library</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-guide/cli.html">DataFusion Command-line Interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-guide/sql/index.html">SQL Reference</a></li>
-<li class="toctree-l1"><a class="reference internal" href="user-guide/distributed/index.html">Ballista Distributed Compute</a></li>
+<li class="toctree-l1"><a class="reference internal" href="user-guide/configs.html">Configuration Settings</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-guide/faq.html">Frequently Asked Questions</a></li>
</ul>
</div>
<div class="toctree-wrapper compound" id="toc-specs">
-<p><span class="caption-text">Specification</span><a class="headerlink" href="#toc-specs" title="Permalink to this toctree">¶</a></p>
+<p class="caption"><span class="caption-text">Specification</span><a class="headerlink" href="#toc-specs" title="Permalink to this toctree">¶</a></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="specification/roadmap.html">Roadmap</a></li>
<li class="toctree-l1"><a class="reference internal" href="specification/roadmap.html#datafusion">DataFusion</a></li>
<li class="toctree-l1"><a class="reference internal" href="specification/roadmap.html#ballista">Ballista</a></li>
<li class="toctree-l1"><a class="reference internal" href="specification/invariants.html">DataFusion’s Invariants</a></li>
-<li class="toctree-l1"><a class="reference internal" href="specification/output-field-name-semantic.html">Datafusion output field name semantic</a></li>
+<li class="toctree-l1"><a class="reference internal" href="specification/output-field-name-semantic.html">DataFusion output field name semantic</a></li>
<li class="toctree-l1"><a class="reference internal" href="specification/quarterly_roadmap.html">Roadmap</a></li>
</ul>
</div>
<div class="toctree-wrapper compound" id="toc-readme">
-<p><span class="caption-text">README</span><a class="headerlink" href="#toc-readme" title="Permalink to this toctree">¶</a></p>
+<p class="caption"><span class="caption-text">README</span><a class="headerlink" href="#toc-readme" title="Permalink to this toctree">¶</a></p>
<ul>
-<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">Datafusion</a></li>
-<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">Ballista</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">DataFusion</a></li>
</ul>
</div>
<div class="toctree-wrapper compound" id="toc-community">
-<p><span class="caption-text">Community</span><a class="headerlink" href="#toc-community" title="Permalink to this toctree">¶</a></p>
+<p class="caption"><span class="caption-text">Community</span><a class="headerlink" href="#toc-community" title="Permalink to this toctree">¶</a></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="community/communication.html">Communication</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow-datafusion/issues">Issue tracker</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/CODE_OF_CONDUCT.md">Code of conduct</a></li>
</ul>
</div>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -527,7 +437,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/objects.inv b/datafusion/objects.inv
index 34c23e7938..c2765e8daa 100644
Binary files a/datafusion/objects.inv and b/datafusion/objects.inv differ
diff --git a/datafusion/py-modindex.html b/datafusion/py-modindex.html
index c004a03d2a..8cbd063fe2 100644
--- a/datafusion/py-modindex.html
+++ b/datafusion/py-modindex.html
@@ -4,7 +4,7 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
- <title>Python Module Index — Arrow Datafusion documentation</title>
+ <title>Python Module Index — Arrow DataFusion documentation</title>
<link href="_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +70,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +118,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +144,8 @@
<a class="reference internal" href="python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +167,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +197,8 @@
<a class="reference internal" href="user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +218,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +230,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="user-guide/faq.html">
@@ -324,7 +241,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +259,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +268,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +276,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -459,7 +371,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/api.html b/datafusion/python/api.html
index f94a09181b..c482348628 100644
--- a/datafusion/python/api.html
+++ b/datafusion/python/api.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>API Reference — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>API Reference — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="api-reference">
+ <div class="section" id="api-reference">
<span id="api"></span><h1>API Reference<a class="headerlink" href="#api-reference" title="Permalink to this headline">¶</a></h1>
<div class="toctree-wrapper compound">
<ul>
@@ -396,10 +319,7 @@
<li class="toctree-l2"><a class="reference internal" href="generated/datafusion.DataFrame.html">datafusion.DataFrame</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="api/execution_context.html">ExecutionContext</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="generated/datafusion.ExecutionContext.html">datafusion.ExecutionContext</a></li>
-</ul>
-</li>
+<li class="toctree-l1"><a class="reference internal" href="api/execution_context.html">SessionContext</a></li>
<li class="toctree-l1"><a class="reference internal" href="api/expression.html">Expression</a><ul>
<li class="toctree-l2"><a class="reference internal" href="generated/datafusion.Expression.html">datafusion.Expression</a></li>
</ul>
@@ -410,7 +330,7 @@
</li>
</ul>
</div>
-</section>
+</div>
</div>
@@ -446,7 +366,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/api/dataframe.html b/datafusion/python/api/dataframe.html
index dd4d49838c..30101872b7 100644
--- a/datafusion/python/api/dataframe.html
+++ b/datafusion/python/api/dataframe.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>DataFrame — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>DataFrame — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="dataframe">
+ <div class="section" id="dataframe">
<span id="api-dataframe"></span><h1>DataFrame<a class="headerlink" href="#dataframe" title="Permalink to this headline">¶</a></h1>
<table class="longtable table autosummary">
<colgroup>
@@ -401,7 +324,7 @@
</tr>
</tbody>
</table>
-</section>
+</div>
</div>
@@ -437,7 +360,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/api/execution_context.html b/datafusion/python/api/execution_context.html
index 835a04c431..ee71f54b83 100644
--- a/datafusion/python/api/execution_context.html
+++ b/datafusion/python/api/execution_context.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>ExecutionContext — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>SessionContext — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -35,7 +34,7 @@
<script src="../../_static/language_data.js"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="datafusion.ExecutionContext" href="../generated/datafusion.ExecutionContext.html" />
+ <link rel="next" title="Expression" href="expression.html" />
<link rel="prev" title="datafusion.DataFrame" href="../generated/datafusion.DataFrame.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3 current active">
<a class="current reference internal" href="#">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,20 +311,20 @@
<div>
- <section id="executioncontext">
-<span id="api-execution-context"></span><h1>ExecutionContext<a class="headerlink" href="#executioncontext" title="Permalink to this headline">¶</a></h1>
+ <div class="section" id="sessioncontext">
+<span id="api-execution-context"></span><h1>SessionContext<a class="headerlink" href="#sessioncontext" title="Permalink to this headline">¶</a></h1>
<table class="longtable table autosummary">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
-<tr class="row-odd"><td><p><a class="reference internal" href="../generated/datafusion.ExecutionContext.html#datafusion.ExecutionContext" title="datafusion.ExecutionContext"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ExecutionContext</span></code></a></p></td>
-<td><p><cite>PyExecutionContext</cite> is able to plan and execute DataFusion plans.</p></td>
+<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionContext</span></code></p></td>
+<td><p></p></td>
</tr>
</tbody>
</table>
-</section>
+</div>
</div>
@@ -416,10 +339,10 @@
<p class="prev-next-title">datafusion.DataFrame</p>
</div>
</a>
- <a class='right-next' id="next-link" href="../generated/datafusion.ExecutionContext.html" title="next page">
+ <a class='right-next' id="next-link" href="expression.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">datafusion.ExecutionContext</p>
+ <p class="prev-next-title">Expression</p>
</div>
<i class="fas fa-angle-right"></i>
</a>
@@ -437,7 +360,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/api/expression.html b/datafusion/python/api/expression.html
index dabb3b334c..2f3cd4f8cd 100644
--- a/datafusion/python/api/expression.html
+++ b/datafusion/python/api/expression.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Expression — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Expression — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -36,7 +35,7 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="datafusion.Expression" href="../generated/datafusion.Expression.html" />
- <link rel="prev" title="datafusion.ExecutionContext" href="../generated/datafusion.ExecutionContext.html" />
+ <link rel="prev" title="SessionContext" href="execution_context.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3 current active">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="expression">
+ <div class="section" id="expression">
<span id="api-expression"></span><h1>Expression<a class="headerlink" href="#expression" title="Permalink to this headline">¶</a></h1>
<table class="longtable table autosummary">
<colgroup>
@@ -401,7 +324,7 @@
</tr>
</tbody>
</table>
-</section>
+</div>
</div>
@@ -409,11 +332,11 @@
<!-- Previous / next buttons -->
<div class='prev-next-area'>
- <a class='left-prev' id="prev-link" href="../generated/datafusion.ExecutionContext.html" title="previous page">
+ <a class='left-prev' id="prev-link" href="execution_context.html" title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">datafusion.ExecutionContext</p>
+ <p class="prev-next-title">SessionContext</p>
</div>
</a>
<a class='right-next' id="next-link" href="../generated/datafusion.Expression.html" title="next page">
@@ -437,7 +360,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/api/functions.html b/datafusion/python/api/functions.html
index a27d174443..df70a83e88 100644
--- a/datafusion/python/api/functions.html
+++ b/datafusion/python/api/functions.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Functions — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Functions — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="functions">
+ <div class="section" id="functions">
<span id="api-functions"></span><h1>Functions<a class="headerlink" href="#functions" title="Permalink to this headline">¶</a></h1>
<table class="longtable table autosummary">
<colgroup>
@@ -401,7 +324,7 @@
</tr>
</tbody>
</table>
-</section>
+</div>
</div>
@@ -437,7 +360,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/generated/datafusion.DataFrame.html b/datafusion/python/generated/datafusion.DataFrame.html
index 58239fd97b..3c8ce662b2 100644
--- a/datafusion/python/generated/datafusion.DataFrame.html
+++ b/datafusion/python/generated/datafusion.DataFrame.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>datafusion.DataFrame — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>datafusion.DataFrame — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -35,7 +34,7 @@
<script src="../../_static/language_data.js"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="ExecutionContext" href="../api/execution_context.html" />
+ <link rel="next" title="SessionContext" href="../api/execution_context.html" />
<link rel="prev" title="DataFrame" href="../api/dataframe.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="../api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="datafusion-dataframe">
+ <div class="section" id="datafusion-dataframe">
<h1>datafusion.DataFrame<a class="headerlink" href="#datafusion-dataframe" title="Permalink to this headline">¶</a></h1>
<dl class="class">
<dt id="datafusion.DataFrame">
@@ -494,7 +417,7 @@ guarantee of the order of the result.</p>
</dd></dl>
-</section>
+</div>
</div>
@@ -512,7 +435,7 @@ guarantee of the order of the result.</p>
<a class='right-next' id="next-link" href="../api/execution_context.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">ExecutionContext</p>
+ <p class="prev-next-title">SessionContext</p>
</div>
<i class="fas fa-angle-right"></i>
</a>
@@ -530,7 +453,7 @@ guarantee of the order of the result.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/generated/datafusion.Expression.html b/datafusion/python/generated/datafusion.Expression.html
index 5b20356367..83806ac5a4 100644
--- a/datafusion/python/generated/datafusion.Expression.html
+++ b/datafusion/python/generated/datafusion.Expression.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>datafusion.Expression — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>datafusion.Expression — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="../api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3 current active">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="datafusion-expression">
+ <div class="section" id="datafusion-expression">
<h1>datafusion.Expression<a class="headerlink" href="#datafusion-expression" title="Permalink to this headline">¶</a></h1>
<dl class="class">
<dt id="datafusion.Expression">
@@ -465,7 +388,7 @@
</dd></dl>
-</section>
+</div>
</div>
@@ -501,7 +424,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/generated/datafusion.functions.html b/datafusion/python/generated/datafusion.functions.html
index f77ad34895..c6368f5b99 100644
--- a/datafusion/python/generated/datafusion.functions.html
+++ b/datafusion/python/generated/datafusion.functions.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>datafusion.functions — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>datafusion.functions — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -108,7 +107,7 @@
</li>
<li class="toctree-l3">
<a class="reference internal" href="../api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
</li>
<li class="toctree-l3">
@@ -131,7 +130,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -182,6 +181,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -189,86 +193,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../user-guide/faq.html">
@@ -276,7 +204,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -294,7 +222,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -303,7 +231,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -311,16 +239,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -388,7 +311,7 @@
<div>
- <section id="module-datafusion.functions">
+ <div class="section" id="module-datafusion.functions">
<span id="datafusion-functions"></span><h1>datafusion.functions<a class="headerlink" href="#module-datafusion.functions" title="Permalink to this headline">¶</a></h1>
<dl class="function">
<dt id="datafusion.functions.abs">
@@ -758,7 +681,7 @@ Standard algorithms are md5, sha224, sha256, sha384, sha512, blake2s, blake2b, a
<dd><p>Creates a new Window function expression</p>
</dd></dl>
-</section>
+</div>
</div>
@@ -794,7 +717,7 @@ Standard algorithms are md5, sha224, sha256, sha384, sha512, blake2s, blake2b, a
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/python/index.html b/datafusion/python/index.html
index 57784b93f6..500ecbdf83 100644
--- a/datafusion/python/index.html
+++ b/datafusion/python/index.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>DataFusion in Python — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>DataFusion in Python — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -36,7 +35,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="API Reference" href="api.html" />
- <link rel="prev" title="Apache Arrow Datafusion" href="../index.html" />
+ <link rel="prev" title="Apache Arrow DataFusion" href="../index.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -478,15 +389,15 @@
<div>
- <section id="datafusion-in-python">
+ <div class="section" id="datafusion-in-python">
<h1>DataFusion in Python<a class="headerlink" href="#datafusion-in-python" title="Permalink to this headline">¶</a></h1>
<p>This is a Python library that binds to <a class="reference external" href="https://arrow.apache.org/">Apache Arrow</a> in-memory query engine <a class="reference external" href="https://github.com/apache/arrow/tree/master/rust/datafusion">DataFusion</a>.</p>
<p>Like pyspark, it allows you to build a plan through SQL or a DataFrame API against in-memory data, parquet or CSV files, run it in a multi-threaded environment, and obtain the result back in Python.</p>
<p>It also allows you to use UDFs and UDAFs for complex operations.</p>
<p>The major advantage of this library over other execution engines is that this library achieves zero-copy between Python and its execution engine: there is no cost in using UDFs, UDAFs, and collecting the results to Python apart from having to lock the GIL when running those operations.</p>
-<p>Its query engine, DataFusion, is written in <a class="reference external" href="https://www.rust-lang.org">Rust</a>), which makes strong assumptions about thread safety and lack of memory leaks.</p>
+<p>Its query engine, DataFusion, is written in <a class="reference external" href="https://www.rust-lang.org">Rust</a>, which makes strong assumptions about thread safety and lack of memory leaks.</p>
<p>Technically, zero-copy is achieved via the <a class="reference external" href="https://arrow.apache.org/docs/format/CDataInterface.html">c data interface</a>.</p>
-<section id="how-to-use-it">
+<div class="section" id="how-to-use-it">
<h2>How to use it<a class="headerlink" href="#how-to-use-it" title="Permalink to this headline">¶</a></h2>
<p>Simple usage:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">datafusion</span>
@@ -495,7 +406,7 @@
<span class="kn">import</span> <span class="nn">pyarrow</span>
<span class="c1"># create a context</span>
-<span class="n">ctx</span> <span class="o">=</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">ExecutionContext</span><span class="p">()</span>
+<span class="n">ctx</span> <span class="o">=</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
<span class="c1"># create a RecordBatch and a new DataFrame from it</span>
<span class="n">batch</span> <span class="o">=</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span>
@@ -517,7 +428,57 @@
<span class="k">assert</span> <span class="n">result</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">==</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span>< [...]
</pre></div>
</div>
-<section id="udfs">
+<p>We can also execute a query against data stored in CSV</p>
+<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">echo</span> <span class="s2">"a,b\n1,4\n2,5\n3,6"</span> > example.csv
+</pre></div>
+</div>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">datafusion</span>
+<span class="kn">from</span> <span class="nn">datafusion</span> <span class="kn">import</span> <span class="n">functions</span> <span class="k">as</span> <span class="n">f</span>
+<span class="kn">from</span> <span class="nn">datafusion</span> <span class="kn">import</span> <span class="n">col</span>
+<span class="kn">import</span> <span class="nn">pyarrow</span>
+
+<span class="c1"># create a context</span>
+<span class="n">ctx</span> <span class="o">=</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+
+<span class="c1"># register a CSV</span>
+<span class="n">ctx</span><span class="o">.</span><span class="n">register_csv</span><span class="p">(</span><span class="s1">'example'</span><span class="p">,</span> <span class="s1">'example.csv'</span><span class="p">)</span>
+
+<span class="c1"># create a new statement</span>
+<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">table</span><span class="p">(</span><span class="s1">'example'</span><span class="p">)</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+ <span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">+</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">),</span>
+ <span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">-</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">),</span>
+<span class="p">)</span>
+
+<span class="c1"># execute and collect the first (and only) batch</span>
+<span class="n">result</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
+
+<span class="k">assert</span> <span class="n">result</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">==</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">9</span><span class="p">])</span>
+<span class="k">assert</span> <span class="n">result</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">==</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span>< [...]
+</pre></div>
+</div>
+<p>And how to execute a query against a CSV using SQL:</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">datafusion</span>
+<span class="kn">from</span> <span class="nn">datafusion</span> <span class="kn">import</span> <span class="n">functions</span> <span class="k">as</span> <span class="n">f</span>
+<span class="kn">from</span> <span class="nn">datafusion</span> <span class="kn">import</span> <span class="n">col</span>
+<span class="kn">import</span> <span class="nn">pyarrow</span>
+
+<span class="c1"># create a context</span>
+<span class="n">ctx</span> <span class="o">=</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+
+<span class="c1"># register a CSV</span>
+<span class="n">ctx</span><span class="o">.</span><span class="n">register_csv</span><span class="p">(</span><span class="s1">'example'</span><span class="p">,</span> <span class="s1">'example.csv'</span><span class="p">)</span>
+
+<span class="c1"># create a new statement via SQL</span>
+<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT a+b, a-b FROM example"</span><span class="p">)</span>
+
+<span class="c1"># execute and collect the first (and only) batch</span>
+<span class="n">result</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
+
+<span class="k">assert</span> <span class="n">result</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="o">==</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">9</span><span class="p">])</span>
+<span class="k">assert</span> <span class="n">result</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">==</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span>< [...]
+</pre></div>
+</div>
+<div class="section" id="udfs">
<h3>UDFs<a class="headerlink" href="#udfs" title="Permalink to this headline">¶</a></h3>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">is_null</span><span class="p">(</span><span class="n">array</span><span class="p">:</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">Array</span><span class="p">)</span> <span class="o">-></span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">Array</span><span class="p">:</span>
<span class="k">return</span> <span class="n">array</span><span class="o">.</span><span class="n">is_null</span><span class="p">()</span>
@@ -527,8 +488,8 @@
<span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">udf</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)))</span>
</pre></div>
</div>
-</section>
-<section id="udaf">
+</div>
+<div class="section" id="udaf">
<h3>UDAF<a class="headerlink" href="#udaf" title="Permalink to this headline">¶</a></h3>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pyarrow</span>
<span class="kn">import</span> <span class="nn">pyarrow.compute</span>
@@ -566,15 +527,15 @@
<span class="p">)</span>
</pre></div>
</div>
-</section>
-</section>
-<section id="how-to-install-from-pip">
+</div>
+</div>
+<div class="section" id="how-to-install-from-pip">
<h2>How to install (from pip)<a class="headerlink" href="#how-to-install-from-pip" title="Permalink to this headline">¶</a></h2>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>pip install datafusion
</pre></div>
</div>
-</section>
-<section id="how-to-develop">
+</div>
+<div class="section" id="how-to-develop">
<h2>How to develop<a class="headerlink" href="#how-to-develop" title="Permalink to this headline">¶</a></h2>
<p>This assumes that you have rust and cargo installed. We use the workflow recommended by <a class="reference external" href="https://github.com/PyO3/pyo3">pyo3</a> and <a class="reference external" href="https://github.com/PyO3/maturin">maturin</a>.</p>
<p>Bootstrap:</p>
@@ -596,8 +557,8 @@ maturin develop
python -m pytest
</pre></div>
</div>
-</section>
-<section id="how-to-update-dependencies">
+</div>
+<div class="section" id="how-to-update-dependencies">
<h2>How to update dependencies<a class="headerlink" href="#how-to-update-dependencies" title="Permalink to this headline">¶</a></h2>
<p>To change test dependencies, change the <cite>requirements.in</cite> and run</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="c1"># install pip-tools (this can be done only once), also consider running in venv</span>
@@ -612,22 +573,22 @@ pip-compile --generate-hashes
</pre></div>
</div>
<p>More details about pip-tools <a class="reference external" href="https://github.com/jazzband/pip-tools">here</a></p>
-</section>
-<section id="api-reference">
+</div>
+<div class="section" id="api-reference">
<h2>API reference<a class="headerlink" href="#api-reference" title="Permalink to this headline">¶</a></h2>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="api.html">API Reference</a><ul>
<li class="toctree-l2"><a class="reference internal" href="api/dataframe.html">DataFrame</a></li>
-<li class="toctree-l2"><a class="reference internal" href="api/execution_context.html">ExecutionContext</a></li>
+<li class="toctree-l2"><a class="reference internal" href="api/execution_context.html">SessionContext</a></li>
<li class="toctree-l2"><a class="reference internal" href="api/expression.html">Expression</a></li>
<li class="toctree-l2"><a class="reference internal" href="api/functions.html">Functions</a></li>
</ul>
</li>
</ul>
</div>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -639,7 +600,7 @@ pip-compile --generate-hashes
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Apache Arrow Datafusion</p>
+ <p class="prev-next-title">Apache Arrow DataFusion</p>
</div>
</a>
<a class='right-next' id="next-link" href="api.html" title="next page">
@@ -663,7 +624,7 @@ pip-compile --generate-hashes
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/search.html b/datafusion/search.html
index 2bc415c43d..76aaced16d 100644
--- a/datafusion/search.html
+++ b/datafusion/search.html
@@ -4,7 +4,7 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
- <title>Search — Arrow Datafusion documentation</title>
+ <title>Search — Arrow DataFusion documentation</title>
<link href="_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -72,7 +72,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -120,29 +120,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -158,8 +146,8 @@
<a class="reference internal" href="python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -181,7 +169,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -211,8 +199,8 @@
<a class="reference internal" href="user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -232,6 +220,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -239,86 +232,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="user-guide/faq.html">
@@ -326,7 +243,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -344,7 +261,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -353,7 +270,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -361,16 +278,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -460,7 +372,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/searchindex.js b/datafusion/searchindex.js
index 174fc99d76..5a225eef79 100644
--- a/datafusion/searchindex.js
+++ b/datafusion/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["cli/index","community/communication","index","python/api","python/api/dataframe","python/api/execution_context","python/api/expression","python/api/functions","python/generated/datafusion.DataFrame","python/generated/datafusion.ExecutionContext","python/generated/datafusion.Expression","python/generated/datafusion.functions","python/index","specification/invariants","specification/output-field-name-semantic","specification/quarterly_roadmap","specification/rfc [...]
\ No newline at end of file
+Search.setIndex({docnames:["cli/index","community/communication","index","python/api","python/api/dataframe","python/api/execution_context","python/api/expression","python/api/functions","python/generated/datafusion.DataFrame","python/generated/datafusion.Expression","python/generated/datafusion.functions","python/index","specification/invariants","specification/output-field-name-semantic","specification/quarterly_roadmap","specification/roadmap","user-guide/cli","user-guide/configs","us [...]
\ No newline at end of file
diff --git a/datafusion/specification/invariants.html b/datafusion/specification/invariants.html
index 7ec5e8d95c..56bd859270 100644
--- a/datafusion/specification/invariants.html
+++ b/datafusion/specification/invariants.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>DataFusion’s Invariants — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>DataFusion’s Invariants — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -35,7 +34,7 @@
<script src="../_static/language_data.js"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Datafusion output field name semantic" href="output-field-name-semantic.html" />
+ <link rel="next" title="DataFusion output field name semantic" href="output-field-name-semantic.html" />
<link rel="prev" title="Roadmap" href="roadmap.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -671,13 +582,13 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="datafusion-s-invariants">
+<div class="section" id="datafusion-s-invariants">
<h1>DataFusion’s Invariants<a class="headerlink" href="#datafusion-s-invariants" title="Permalink to this headline">¶</a></h1>
<p>This document enumerates invariants of DataFusion’s logical and physical planes
(functions, and nodes). Some of these invariants are currently not enforced.
This document assumes that the reader is familiar with some of the codebase,
including rust arrow’s RecordBatch and Array.</p>
-<section id="rational">
+<div class="section" id="rational">
<h2>Rational<a class="headerlink" href="#rational" title="Permalink to this headline">¶</a></h2>
<p>DataFusion’s computational model is built on top of a dynamically typed arrow
object, Array, that offers the interface <code class="docutils literal notranslate"><span class="pre">Array::as_any</span></code> to downcast itself to
@@ -691,8 +602,8 @@ built on top of this principle.</p>
invariances. This document declares some of these invariants, so that users
know what they can expect from a query in DataFusion, and DataFusion developers
know what they need to enforce at the coding level.</p>
-</section>
-<section id="notation">
+</div>
+<div class="section" id="notation">
<h2>Notation<a class="headerlink" href="#notation" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Field or physical field: the tuple name, <code class="docutils literal notranslate"><span class="pre">arrow::DataType</span></code> and nullability flag (a bool whether values can be null), represented in this document by <code class="docutils literal notranslate"><span class="pre">PF(name,</span> <span class="pre">type,</span> <span class="pre">nullable)</span></code></p></li>
@@ -701,9 +612,9 @@ know what they need to enforce at the coding level.</p>
<li><p>Logical schema: a vector of logical fields, used by logical plan.</p></li>
<li><p>Physical schema: a vector of physical fields, used by both physical plan and Arrow record batch.</p></li>
</ul>
-<section id="logical">
+<div class="section" id="logical">
<h3>Logical<a class="headerlink" href="#logical" title="Permalink to this headline">¶</a></h3>
-<section id="function">
+<div class="section" id="function">
<h4>Function<a class="headerlink" href="#function" title="Permalink to this headline">¶</a></h4>
<p>An object that knows its valid incoming logical fields and how to derive its
output logical field from its arguments’ logical fields. A functions’ output
@@ -716,22 +627,22 @@ field is itself a function of its input fields:</p>
<li><p><code class="docutils literal notranslate"><span class="pre">plus(a,b)</span> <span class="pre">-></span> <span class="pre">LF(None,</span> <span class="pre">"{a}</span> <span class="pre">Plus</span> <span class="pre">{b}",</span> <span class="pre">d(a.type,b.type),</span> <span class="pre">a.nullable</span> <span class="pre">|</span> <span class="pre">b.nullable)</span></code> where d is the function mapping input types to output type (<code class="docutils literal n [...]
<li><p><code class="docutils literal notranslate"><span class="pre">length(a)</span> <span class="pre">-></span> <span class="pre">LF(None,</span> <span class="pre">"length({a})",</span> <span class="pre">u32,</span> <span class="pre">a.nullable)</span></code></p></li>
</ul>
-</section>
-<section id="plan">
+</div>
+<div class="section" id="plan">
<h4>Plan<a class="headerlink" href="#plan" title="Permalink to this headline">¶</a></h4>
<p>A tree composed of other plans and functions (e.g. <code class="docutils literal notranslate"><span class="pre">Projection</span> <span class="pre">c1</span> <span class="pre">+</span> <span class="pre">c2,</span> <span class="pre">c1</span> <span class="pre">-</span> <span class="pre">c2</span> <span class="pre">AS</span> <span class="pre">sum12;</span> <span class="pre">Scan</span> <span class="pre">c1</span> <span class="pre">as</span> <span class="pre">u32,</span> <span class="pre [...]
that knows how to derive its schema.</p>
<p>Certain plans have a frozen schema (e.g. Scan), while others derive their
schema from their child nodes.</p>
-</section>
-<section id="column">
+</div>
+<div class="section" id="column">
<h4>Column<a class="headerlink" href="#column" title="Permalink to this headline">¶</a></h4>
<p>An identifier in a logical plan consists of field name and relation name.</p>
-</section>
-</section>
-<section id="physical">
+</div>
+</div>
+<div class="section" id="physical">
<h3>Physical<a class="headerlink" href="#physical" title="Permalink to this headline">¶</a></h3>
-<section id="id1">
+<div class="section" id="id1">
<h4>Function<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h4>
<p>An object that knows how to derive its physical field from its arguments’
physical fields, and also how to actually perform the computation on data. A
@@ -744,44 +655,44 @@ functions’ output physical field is a function of its input physical fields:</
<li><p><code class="docutils literal notranslate"><span class="pre">plus(a,b)</span> <span class="pre">-></span> <span class="pre">PF("{a}</span> <span class="pre">Plus</span> <span class="pre">{b}",</span> <span class="pre">d(a.type,b.type),</span> <span class="pre">a.nullable</span> <span class="pre">|</span> <span class="pre">b.nullable)</span></code> where d is a complex function (<code class="docutils literal notranslate"><span class="pre">get_supertype</span></code> in [...]
<li><p><code class="docutils literal notranslate"><span class="pre">length(&str)</span> <span class="pre">-></span> <span class="pre">PF("length({a})",</span> <span class="pre">u32,</span> <span class="pre">a.nullable)</span></code> whose computation is “count number of bytes in the string”.</p></li>
</ul>
-</section>
-<section id="id2">
+</div>
+<div class="section" id="id2">
<h4>Plan<a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h4>
<p>A tree (e.g. <code class="docutils literal notranslate"><span class="pre">Projection</span> <span class="pre">c1</span> <span class="pre">+</span> <span class="pre">c2,</span> <span class="pre">c1</span> <span class="pre">-</span> <span class="pre">c2</span> <span class="pre">AS</span> <span class="pre">sum12;</span> <span class="pre">Scan</span> <span class="pre">c1</span> <span class="pre">as</span> <span class="pre">u32,</span> <span class="pre">c2</span> <span class="pre">as</span [...]
that knows how to derive its metadata and compute itself.</p>
<p>Note how the physical plane does not know how to derive field names: field
names are solely a property of the logical plane, as they are not needed in the
physical plane.</p>
-</section>
-<section id="id3">
+</div>
+<div class="section" id="id3">
<h4>Column<a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h4>
<p>A type of physical node in a physical plan consists of a field name and unique index.</p>
-</section>
-</section>
-<section id="data-sources-registry">
+</div>
+</div>
+<div class="section" id="data-sources-registry">
<h3>Data Sources’ registry<a class="headerlink" href="#data-sources-registry" title="Permalink to this headline">¶</a></h3>
<p>A map of source name/relation -> Schema plus associated properties necessary to read data from it (e.g. file path).</p>
-</section>
-<section id="functions-registry">
+</div>
+<div class="section" id="functions-registry">
<h3>Functions’ registry<a class="headerlink" href="#functions-registry" title="Permalink to this headline">¶</a></h3>
<p>A map of function name -> logical + physical function.</p>
-</section>
-<section id="physical-planner">
+</div>
+<div class="section" id="physical-planner">
<h3>Physical Planner<a class="headerlink" href="#physical-planner" title="Permalink to this headline">¶</a></h3>
<p>A function that knows how to derive a physical plan from a logical plan:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">plan</span><span class="p">(</span><span class="n">LogicalPlan</span><span class="p">)</span> <span class="o">-></span> <span class="n">PhysicalPlan</span>
</pre></div>
</div>
-</section>
-<section id="logical-optimizer">
+</div>
+<div class="section" id="logical-optimizer">
<h3>Logical Optimizer<a class="headerlink" href="#logical-optimizer" title="Permalink to this headline">¶</a></h3>
<p>A function that accepts a logical plan and returns an (optimized) logical plan
which computes the same results, but in a more efficient manner:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">optimize</span><span class="p">(</span><span class="n">LogicalPlan</span><span class="p">)</span> <span class="o">-></span> <span class="n">LogicalPlan</span>
</pre></div>
</div>
-</section>
-<section id="physical-optimizer">
+</div>
+<div class="section" id="physical-optimizer">
<h3>Physical Optimizer<a class="headerlink" href="#physical-optimizer" title="Permalink to this headline">¶</a></h3>
<p>A function that accepts a physical plan and returns an (optimized) physical
plan which computes the same results, but may differ based on the actual
@@ -789,17 +700,17 @@ hardware or execution environment being run:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">optimize</span><span class="p">(</span><span class="n">PhysicalPlan</span><span class="p">)</span> <span class="o">-></span> <span class="n">PhysicalPlan</span>
</pre></div>
</div>
-</section>
-<section id="builder">
+</div>
+<div class="section" id="builder">
<h3>Builder<a class="headerlink" href="#builder" title="Permalink to this headline">¶</a></h3>
<p>A function that knows how to build a new logical plan from an existing logical
plan and some extra parameters.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">build</span><span class="p">(</span><span class="n">logical_plan</span><span class="p">,</span> <span class="n">params</span><span class="o">...</span><span class="p">)</span> <span class="o">-></span> <span class="n">logical_plan</span>
</pre></div>
</div>
-</section>
-</section>
-<section id="invariants">
+</div>
+</div>
+<div class="section" id="invariants">
<h2>Invariants<a class="headerlink" href="#invariants" title="Permalink to this headline">¶</a></h2>
<p>The following subsections describe invariants. Since functions’ output schema
depends on its arguments’ schema (e.g. min, plus), the resulting schema can
@@ -808,44 +719,44 @@ Likewise, schemas of functions depend on the specific registry of functions
registered (e.g. does <code class="docutils literal notranslate"><span class="pre">my_op</span></code> return u32 or u64?). Thus, in this section, the
wording “same schema” is understood to mean “same schema under a given registry
of data sources and functions”.</p>
-<section id="relation-name-tuples-in-logical-fields-and-logical-columns-are-unique">
+<div class="section" id="relation-name-tuples-in-logical-fields-and-logical-columns-are-unique">
<h3>(relation, name) tuples in logical fields and logical columns are unique<a class="headerlink" href="#relation-name-tuples-in-logical-fields-and-logical-columns-are-unique" title="Permalink to this headline">¶</a></h3>
<p>Every logical field’s (relation, name) tuple in a logical schema MUST be unique.
Every logical column’s (relation, name) tuple in a logical plan MUST be unique.</p>
<p>This invariant guarantees that <code class="docutils literal notranslate"><span class="pre">SELECT</span> <span class="pre">t1.id,</span> <span class="pre">t2.id</span> <span class="pre">FROM</span> <span class="pre">t1</span> <span class="pre">JOIN</span> <span class="pre">t2...</span></code>
unambiguously selects the field <code class="docutils literal notranslate"><span class="pre">t1.id</span></code> and <code class="docutils literal notranslate"><span class="pre">t2.id</span></code> in a logical schema in the
logical plane.</p>
-<section id="responsibility">
+<div class="section" id="responsibility">
<h4>Responsibility<a class="headerlink" href="#responsibility" title="Permalink to this headline">¶</a></h4>
<p>It is the logical builder and optimizer’s responsibility to guarantee this
invariant.</p>
-</section>
-<section id="validation">
+</div>
+<div class="section" id="validation">
<h4>Validation<a class="headerlink" href="#validation" title="Permalink to this headline">¶</a></h4>
<p>Builder and optimizer MUST error if this invariant is violated on any logical
node that creates a new schema (e.g. scan, projection, aggregation, join, etc.).</p>
-</section>
-</section>
-<section id="physical-schema-is-consistent-with-data">
+</div>
+</div>
+<div class="section" id="physical-schema-is-consistent-with-data">
<h3>Physical schema is consistent with data<a class="headerlink" href="#physical-schema-is-consistent-with-data" title="Permalink to this headline">¶</a></h3>
<p>The contents of every Array in every RecordBatch in every partition returned by
a physical plan MUST be consistent with RecordBatch’s schema, in that every
Array in the RecordBatch must be downcastable to its corresponding type
declared in the RecordBatch.</p>
-<section id="id4">
+<div class="section" id="id4">
<h4>Responsibility<a class="headerlink" href="#id4" title="Permalink to this headline">¶</a></h4>
<p>Physical functions MUST guarantee this invariant. This is particularly
important in aggregate functions, whose aggregating type may be different from
the intermediary types during calculations (e.g. sum(i32) -> i64).</p>
-</section>
-<section id="id5">
+</div>
+<div class="section" id="id5">
<h4>Validation<a class="headerlink" href="#id5" title="Permalink to this headline">¶</a></h4>
<p>Since the validation of this invariant is computationally expensive, execution
contexts CAN validate this invariant. It is acceptable for physical nodes to
<code class="docutils literal notranslate"><span class="pre">panic!</span></code> if their input does not satisfy this invariant.</p>
-</section>
-</section>
-<section id="physical-schema-is-consistent-in-physical-functions">
+</div>
+</div>
+<div class="section" id="physical-schema-is-consistent-in-physical-functions">
<h3>Physical schema is consistent in physical functions<a class="headerlink" href="#physical-schema-is-consistent-in-physical-functions" title="Permalink to this headline">¶</a></h3>
<p>The schema of every Array returned by a physical function MUST match the
DataType reported by the physical function itself.</p>
@@ -853,7 +764,7 @@ DataType reported by the physical function itself.</p>
(e.g. Int32), users can safely downcast its resulting Array to the
corresponding type (e.g. Int32Array), as well as to write data to formats that
have a schema with nullability flag (e.g. parquet).</p>
-<section id="id6">
+<div class="section" id="id6">
<h4>Responsibility<a class="headerlink" href="#id6" title="Permalink to this headline">¶</a></h4>
<p>It is the responsibility of the developer that writes a physical function to
guarantee this invariant.</p>
@@ -862,14 +773,14 @@ guarantee this invariant.</p>
<li><p>The derived DataType matches the code it uses to build the array for every branch of valid input type combinations.</p></li>
<li><p>The nullability flag matches how the values are built.</p></li>
</ul>
-</section>
-<section id="id7">
+</div>
+<div class="section" id="id7">
<h4>Validation<a class="headerlink" href="#id7" title="Permalink to this headline">¶</a></h4>
<p>Since the validation of this invariant is computationally expensive, execution
contexts CAN validate this invariant.</p>
-</section>
-</section>
-<section id="the-physical-schema-is-invariant-under-planning">
+</div>
+</div>
+<div class="section" id="the-physical-schema-is-invariant-under-planning">
<h3>The physical schema is invariant under planning<a class="headerlink" href="#the-physical-schema-is-invariant-under-planning" title="Permalink to this headline">¶</a></h3>
<p>The physical schema derived by a physical plan returned by the planner MUST be
equivalent to the physical schema derived by the logical plan passed to the
@@ -889,19 +800,19 @@ to know the resulting physical schema.</p>
single function, <code class="docutils literal notranslate"><span class="pre">Projection</span> <span class="pre">f(c1,c2)</span></code>, a corollary of this is that the
physical schema of every <code class="docutils literal notranslate"><span class="pre">logical</span> <span class="pre">function</span> <span class="pre">-></span> <span class="pre">physical</span> <span class="pre">function</span></code> must be
invariant under planning.</p>
-<section id="id8">
+<div class="section" id="id8">
<h4>Responsibility<a class="headerlink" href="#id8" title="Permalink to this headline">¶</a></h4>
<p>Developers of physical and logical plans and planners MUST guarantee this
invariant for every triplet (logical plan, physical plan, conversion rule).</p>
-</section>
-<section id="id9">
+</div>
+<div class="section" id="id9">
<h4>Validation<a class="headerlink" href="#id9" title="Permalink to this headline">¶</a></h4>
<p>Planners MUST validate this invariant. In particular they MUST return an error
when, during planning, a physical function’s derived schema does not match the
logical functions’ derived schema.</p>
-</section>
-</section>
-<section id="the-output-schema-equals-the-physical-plan-schema">
+</div>
+</div>
+<div class="section" id="the-output-schema-equals-the-physical-plan-schema">
<h3>The output schema equals the physical plan schema<a class="headerlink" href="#the-output-schema-equals-the-physical-plan-schema" title="Permalink to this headline">¶</a></h3>
<p>The schema of every RecordBatch in every partition outputted by a physical plan
MUST be equal to the schema of the physical plan. Specifically:</p>
@@ -911,16 +822,16 @@ MUST be equal to the schema of the physical plan. Specifically:</p>
<p>Together with other invariants, this ensures that the consumers of record
batches do not need to know the output schema of the physical plan; they can
safely rely on the record batch’s schema to perform downscaling and naming.</p>
-<section id="id10">
+<div class="section" id="id10">
<h4>Responsibility<a class="headerlink" href="#id10" title="Permalink to this headline">¶</a></h4>
<p>Physical nodes MUST guarantee this invariant.</p>
-</section>
-<section id="id11">
+</div>
+<div class="section" id="id11">
<h4>Validation<a class="headerlink" href="#id11" title="Permalink to this headline">¶</a></h4>
<p>Execution Contexts CAN validate this invariant.</p>
-</section>
-</section>
-<section id="logical-schema-is-invariant-under-logical-optimization">
+</div>
+</div>
+<div class="section" id="logical-schema-is-invariant-under-logical-optimization">
<h3>Logical schema is invariant under logical optimization<a class="headerlink" href="#logical-schema-is-invariant-under-logical-optimization" title="Permalink to this headline">¶</a></h3>
<p>The logical schema derived by a projected logical plan returned by the logical
optimizer MUST be equivalent to the logical schema derived by the logical plan
@@ -931,16 +842,16 @@ passed to the planner:</p>
<p>This is used to ensure that plans can be optimized without jeopardizing future
referencing logical columns (name and index) or assumptions about their
schemas.</p>
-<section id="id12">
+<div class="section" id="id12">
<h4>Responsibility<a class="headerlink" href="#id12" title="Permalink to this headline">¶</a></h4>
<p>Logical optimizers MUST guarantee this invariant.</p>
-</section>
-<section id="id13">
+</div>
+<div class="section" id="id13">
<h4>Validation<a class="headerlink" href="#id13" title="Permalink to this headline">¶</a></h4>
<p>Users of logical optimizers SHOULD validate this invariant.</p>
-</section>
-</section>
-<section id="physical-schema-is-invariant-under-physical-optimization">
+</div>
+</div>
+<div class="section" id="physical-schema-is-invariant-under-physical-optimization">
<h3>Physical schema is invariant under physical optimization<a class="headerlink" href="#physical-schema-is-invariant-under-physical-optimization" title="Permalink to this headline">¶</a></h3>
<p>The physical schema derived by a projected physical plan returned by the
physical optimizer MUST match the physical schema derived by the physical plan
@@ -949,19 +860,19 @@ passed to the planner:</p>
</pre></div>
</div>
<p>This is used to ensure that plans can be optimized without jeopardizing future
-referencs of logical columns (name and index) or assumptions about their
+references of logical columns (name and index) or assumptions about their
schemas.</p>
-<section id="id14">
+<div class="section" id="id14">
<h4>Responsibility<a class="headerlink" href="#id14" title="Permalink to this headline">¶</a></h4>
<p>Optimizers MUST guarantee this invariant.</p>
-</section>
-<section id="id15">
+</div>
+<div class="section" id="id15">
<h4>Validation<a class="headerlink" href="#id15" title="Permalink to this headline">¶</a></h4>
<p>Users of optimizers SHOULD validate this invariant.</p>
-</section>
-</section>
-</section>
-</section>
+</div>
+</div>
+</div>
+</div>
</div>
@@ -979,7 +890,7 @@ schemas.</p>
<a class='right-next' id="next-link" href="output-field-name-semantic.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Datafusion output field name semantic</p>
+ <p class="prev-next-title">DataFusion output field name semantic</p>
</div>
<i class="fas fa-angle-right"></i>
</a>
@@ -997,7 +908,7 @@ schemas.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/specification/output-field-name-semantic.html b/datafusion/specification/output-field-name-semantic.html
index 26d652fe01..09690f9a96 100644
--- a/datafusion/specification/output-field-name-semantic.html
+++ b/datafusion/specification/output-field-name-semantic.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Datafusion output field name semantic — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>DataFusion output field name semantic — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1 current active">
<a class="current reference internal" href="#">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -498,12 +409,12 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="datafusion-output-field-name-semantic">
-<h1>Datafusion output field name semantic<a class="headerlink" href="#datafusion-output-field-name-semantic" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="datafusion-output-field-name-semantic">
+<h1>DataFusion output field name semantic<a class="headerlink" href="#datafusion-output-field-name-semantic" title="Permalink to this headline">¶</a></h1>
<p>This specification documents how field names in output record batches should be
generated based on given user queries. The filed name rules apply to
-Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
-<section id="field-name-rules">
+DataFusion queries planned from both SQL queries and Dataframe APIs.</p>
+<div class="section" id="field-name-rules">
<h2>Field name rules<a class="headerlink" href="#field-name-rules" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>All bare column field names MUST not contain relation/table qualifier.</p>
@@ -542,10 +453,10 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
</ul>
</li>
</ul>
-</section>
-<section id="appendices">
+</div>
+<div class="section" id="appendices">
<h2>Appendices<a class="headerlink" href="#appendices" title="Permalink to this headline">¶</a></h2>
-<section id="examples-and-comparison-with-other-systems">
+<div class="section" id="examples-and-comparison-with-other-systems">
<h3>Examples and comparison with other systems<a class="headerlink" href="#examples-and-comparison-with-other-systems" title="Permalink to this headline">¶</a></h3>
<p>Data schema for test sample queries:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">CREATE</span> <span class="n">TABLE</span> <span class="n">t1</span> <span class="p">(</span><span class="nb">id</span> <span class="n">INT</span><span class="p">,</span> <span class="n">a</span> <span class="n">VARCHAR</span><span class="p">(</span><span class="mi">5</span><span class="p">));</span>
@@ -557,7 +468,7 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
<span class="n">INSERT</span> <span class="n">INTO</span> <span class="n">t2</span> <span class="p">(</span><span class="nb">id</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> <span class="n">VALUES</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="s1">'world'</span><span class="p">);</span>
</pre></div>
</div>
-<section id="projected-columns">
+<div class="section" id="projected-columns">
<h4>Projected columns<a class="headerlink" href="#projected-columns" title="Permalink to this headline">¶</a></h4>
<p>Query:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">SELECT</span> <span class="n">t1</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">a</span><span class="p">,</span> <span class="n">t2</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">b</span>
@@ -565,7 +476,7 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
<span class="n">JOIN</span> <span class="n">t2</span> <span class="n">ON</span> <span class="n">t1</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="n">t2</span><span class="o">.</span><span class="n">id</span>
</pre></div>
</div>
-<p>Datafusion Arrow record batches output:</p>
+<p>DataFusion Arrow record batches output:</p>
<table class="colwidths-auto table">
<thead>
<tr class="row-odd"><th class="head"><p>id</p></th>
@@ -628,14 +539,14 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
</tr>
</tbody>
</table>
-</section>
-<section id="function-transformed-columns">
+</div>
+<div class="section" id="function-transformed-columns">
<h4>Function transformed columns<a class="headerlink" href="#function-transformed-columns" title="Permalink to this headline">¶</a></h4>
<p>Query:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">SELECT</span> <span class="n">ABS</span><span class="p">(</span><span class="n">t1</span><span class="o">.</span><span class="n">id</span><span class="p">),</span> <span class="nb">abs</span><span class="p">(</span><span class="o">-</span><span class="nb">id</span><span class="p">)</span> <span class="n">FROM</span> <span class="n">t1</span><span class="p">;</span>
</pre></div>
</div>
-<p>Datafusion Arrow record batches output:</p>
+<p>DataFusion Arrow record batches output:</p>
<table class="colwidths-auto table">
<thead>
<tr class="row-odd"><th class="head"><p>abs(t1.id)</p></th>
@@ -715,14 +626,14 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
</tr>
</tbody>
</table>
-</section>
-<section id="function-with-operators">
+</div>
+<div class="section" id="function-with-operators">
<h4>Function with operators<a class="headerlink" href="#function-with-operators" title="Permalink to this headline">¶</a></h4>
<p>Query:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">SELECT</span> <span class="n">t1</span><span class="o">.</span><span class="n">id</span> <span class="o">+</span> <span class="n">ABS</span><span class="p">(</span><span class="nb">id</span><span class="p">),</span> <span class="n">ABS</span><span class="p">(</span><span class="nb">id</span> <span class="o">*</span> <span class="n">t1</span><span class="o">.</span><span class="n">id</span> [...]
</pre></div>
</div>
-<p>Datafusion Arrow record batches output:</p>
+<p>DataFusion Arrow record batches output:</p>
<table class="colwidths-auto table">
<thead>
<tr class="row-odd"><th class="head"><p>t1.id + abs(t1.id)</p></th>
@@ -802,14 +713,14 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
</tr>
</tbody>
</table>
-</section>
-<section id="project-literals">
+</div>
+<div class="section" id="project-literals">
<h4>Project literals<a class="headerlink" href="#project-literals" title="Permalink to this headline">¶</a></h4>
<p>Query:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">SELECT</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="o">+</span><span class="mi">5</span><span class="p">,</span> <span class="s1">'foo_bar'</span><span class="p">;</span>
</pre></div>
</div>
-<p>Datafusion Arrow record batches output:</p>
+<p>DataFusion Arrow record batches output:</p>
<table class="colwidths-auto table">
<thead>
<tr class="row-odd"><th class="head"><p>1</p></th>
@@ -884,10 +795,10 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
</tr>
</tbody>
</table>
-</section>
-</section>
-</section>
-</section>
+</div>
+</div>
+</div>
+</div>
</div>
@@ -923,7 +834,7 @@ Datafusion queries planned from both SQL queries and Dataframe APIs.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/specification/quarterly_roadmap.html b/datafusion/specification/quarterly_roadmap.html
index 32d80c7671..388b7e09d4 100644
--- a/datafusion/specification/quarterly_roadmap.html
+++ b/datafusion/specification/quarterly_roadmap.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Roadmap — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Roadmap — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -36,7 +35,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Communication" href="../community/communication.html" />
- <link rel="prev" title="Datafusion output field name semantic" href="output-field-name-semantic.html" />
+ <link rel="prev" title="DataFusion output field name semantic" href="output-field-name-semantic.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1 current active">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -414,8 +325,8 @@
<nav id="bd-toc-nav">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#q1">
- 2022 Q1
+ <a class="reference internal nav-link" href="#q2">
+ 2022 Q2
</a>
<ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
@@ -423,21 +334,6 @@
DataFusion Core
</a>
</li>
- <li class="toc-h3 nav-item toc-entry">
- <a class="reference internal nav-link" href="#benchmarking">
- Benchmarking
- </a>
- </li>
- <li class="toc-h3 nav-item toc-entry">
- <a class="reference internal nav-link" href="#performance-improvements">
- Performance Improvements
- </a>
- </li>
- <li class="toc-h3 nav-item toc-entry">
- <a class="reference internal nav-link" href="#new-features">
- New Features
- </a>
- </li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#ballista">
Ballista
@@ -447,23 +343,33 @@
<a class="reference internal nav-link" href="#extensions-datafusion-contrib">
Extensions (datafusion-contrib)
</a>
- </li>
- </ul>
- </li>
- <li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#beyond-2022-q1">
- Beyond 2022 Q1
- </a>
- <ul class="nav section-nav flex-column">
- <li class="toc-h3 nav-item toc-entry">
- <a class="reference internal nav-link" href="#id1">
- DataFusion Core
- </a>
- </li>
- <li class="toc-h3 nav-item toc-entry">
- <a class="reference internal nav-link" href="#id2">
- Ballista
- </a>
+ <ul class="nav section-nav flex-column">
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion-python">
+ DataFusion-Python
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion-s3">
+ DataFusion-S3
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion-tui">
+ DataFusion-Tui
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion-bigtable">
+ DataFusion-BigTable
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion-streams">
+ DataFusion-Streams
+ </a>
+ </li>
+ </ul>
</li>
</ul>
</li>
@@ -513,77 +419,121 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="roadmap">
+<div class="section" id="roadmap">
<h1>Roadmap<a class="headerlink" href="#roadmap" title="Permalink to this headline">¶</a></h1>
<p>A quarterly roadmap will be published to give the DataFusion community visibility into the priorities of the projects contributors. This roadmap is not binding.</p>
-<section id="q1">
-<h2>2022 Q1<a class="headerlink" href="#q1" title="Permalink to this headline">¶</a></h2>
-<section id="datafusion-core">
+<div class="section" id="q2">
+<h2>2022 Q2<a class="headerlink" href="#q2" title="Permalink to this headline">¶</a></h2>
+<div class="section" id="datafusion-core">
<h3>DataFusion Core<a class="headerlink" href="#datafusion-core" title="Permalink to this headline">¶</a></h3>
<ul class="simple">
-<li><p>Publish official Arrow2 branch</p></li>
-<li><p>Implementation of memory manager (i.e. to enable spilling to disk as needed)</p></li>
+<li><p>IO Improvements</p>
+<ul>
+<li><p>Reading, registering, and writing more file formats from both DataFrame API and SQL</p></li>
+<li><p>Additional options for IO including partitioning and metadata support</p></li>
</ul>
-</section>
-<section id="benchmarking">
-<h3>Benchmarking<a class="headerlink" href="#benchmarking" title="Permalink to this headline">¶</a></h3>
-<ul class="simple">
-<li><p>Inclusion in Db-Benchmark with all quries covered</p></li>
-<li><p>All TPCH queries covered</p></li>
+</li>
+<li><p>Work Scheduling</p>
+<ul>
+<li><p>Improve predictability, observability and performance of IO and CPU-bound work</p></li>
+<li><p>Develop a more explicit story for managing parallelism during plan execution</p></li>
</ul>
-</section>
-<section id="performance-improvements">
-<h3>Performance Improvements<a class="headerlink" href="#performance-improvements" title="Permalink to this headline">¶</a></h3>
-<ul class="simple">
-<li><p>Predicate evaluation</p></li>
-<li><p>Improve multi-column comparisons (that can’t be vectorized at the moment)</p></li>
-<li><p>Null constant support</p></li>
+</li>
+<li><p>Memory Management</p>
+<ul>
+<li><p>Add more operators for memory limited execution</p></li>
</ul>
-</section>
-<section id="new-features">
-<h3>New Features<a class="headerlink" href="#new-features" title="Permalink to this headline">¶</a></h3>
-<ul class="simple">
-<li><p>Read JSON as table</p></li>
-<li><p>Simplify DDL with Datafusion-Cli</p></li>
-<li><p>Add Decimal128 data type and the attendant features such as Arrow Kernel and UDF support</p></li>
-<li><p>Add new experimental e-graph based optimizer</p></li>
+</li>
+<li><p>Performance</p>
+<ul>
+<li><p>Incorporate row-format into operators such as aggregate</p></li>
+<li><p>Add row-format benchmarks</p></li>
+<li><p>Explore JIT-compiling complex expressions</p></li>
+<li><p>Explore LLVM for JIT, with inline Rust functions as the primary goal</p></li>
+<li><p>Improve performance of Sort and Merge using Row Format / JIT expressions</p></li>
</ul>
-</section>
-<section id="ballista">
+</li>
+<li><p>Documentation</p>
+<ul>
+<li><p>General improvements to DataFusion website</p></li>
+<li><p>Publish design documents</p></li>
+</ul>
+</li>
+<li><p>Streaming</p>
+<ul>
+<li><p>Create <code class="docutils literal notranslate"><span class="pre">StreamProvider</span></code> trait</p></li>
+</ul>
+</li>
+</ul>
+</div>
+<div class="section" id="ballista">
<h3>Ballista<a class="headerlink" href="#ballista" title="Permalink to this headline">¶</a></h3>
<ul class="simple">
-<li><p>Begin work on design documents and plan / priorities for development</p></li>
+<li><p>Make production ready</p>
+<ul>
+<li><p>Shuffle file cleanup</p></li>
+<li><p>Fill functional gaps between DataFusion and Ballista</p></li>
+<li><p>Improve task scheduling and data exchange efficiency</p></li>
+<li><p>Better error handling</p>
+<ul>
+<li><p>Task failure</p></li>
+<li><p>Executor lost</p></li>
+<li><p>Schedule restart</p></li>
</ul>
-</section>
-<section id="extensions-datafusion-contrib">
+</li>
+<li><p>Improve monitoring and logging</p></li>
+<li><p>Auto scaling support</p></li>
+</ul>
+</li>
+<li><p>Support for multi-scheduler deployments. Initially for resiliency and fault tolerance but ultimately to support sharding for scalability and more efficient caching.</p></li>
+<li><p>Executor deployment grouping based on resource allocation</p></li>
+</ul>
+</div>
+<div class="section" id="extensions-datafusion-contrib">
<h3>Extensions (<a class="reference external" href="https://github.com/datafusion-contrib%5D">datafusion-contrib</a>)<a class="headerlink" href="#extensions-datafusion-contrib" title="Permalink to this headline">¶</a></h3>
+<div class="section" id="datafusion-python">
+<h4><a class="reference external" href="https://github.com/datafusion-contrib/datafusion-python">DataFusion-Python</a><a class="headerlink" href="#datafusion-python" title="Permalink to this headline">¶</a></h4>
+<ul class="simple">
+<li><p>Add missing functionality to DataFrame and SessionContext</p></li>
+<li><p>Improve documentation</p></li>
+</ul>
+</div>
+<div class="section" id="datafusion-s3">
+<h4><a class="reference external" href="https://github.com/datafusion-contrib/datafusion-objectstore-s3">DataFusion-S3</a><a class="headerlink" href="#datafusion-s3" title="Permalink to this headline">¶</a></h4>
+<ul class="simple">
+<li><p>Create Python bindings to use with datafusion-python</p></li>
+</ul>
+</div>
+<div class="section" id="datafusion-tui">
+<h4><a class="reference external" href="https://github.com/datafusion-contrib/datafusion-tui">DataFusion-Tui</a><a class="headerlink" href="#datafusion-tui" title="Permalink to this headline">¶</a></h4>
<ul class="simple">
-<li><p>Stable S3 support</p></li>
-<li><p>Begin design discussions and prototyping of a stream provider</p></li>
+<li><p>Create multiple SQL editors</p></li>
+<li><p>Expose more Context and query metadata</p></li>
+<li><p>Support new data sources</p>
+<ul>
+<li><p>BigTable, HDFS, HTTP APIs</p></li>
+</ul>
+</li>
</ul>
-</section>
-</section>
-<section id="beyond-2022-q1">
-<h2>Beyond 2022 Q1<a class="headerlink" href="#beyond-2022-q1" title="Permalink to this headline">¶</a></h2>
-<p>There is no clear timeline for the below, but community members have expressed interest in working on these topics.</p>
-<section id="id1">
-<h3>DataFusion Core<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3>
+</div>
+<div class="section" id="datafusion-bigtable">
+<h4><a class="reference external" href="https://github.com/datafusion-contrib/datafusion-bigtable">DataFusion-BigTable</a><a class="headerlink" href="#datafusion-bigtable" title="Permalink to this headline">¶</a></h4>
<ul class="simple">
-<li><p>Custom SQL support</p></li>
-<li><p>Split DataFusion into multiple crates</p></li>
-<li><p>Push based query execution and code generation</p></li>
+<li><p>Python binding to use with datafusion-python</p></li>
+<li><p>Timestamp range predicate pushdown</p></li>
+<li><p>Multi-threaded partition aware execution</p></li>
+<li><p>Production ready Rust SDK</p></li>
</ul>
-</section>
-<section id="id2">
-<h3>Ballista<a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h3>
+</div>
+<div class="section" id="datafusion-streams">
+<h4><a class="reference external" href="https://github.com/datafusion-contrib/datafusion-streams">DataFusion-Streams</a><a class="headerlink" href="#datafusion-streams" title="Permalink to this headline">¶</a></h4>
<ul class="simple">
-<li><p>Evolve architecture so that it can be deployed in a multi-tenant cloud native environment</p></li>
-<li><p>Ensure Ballista is scalable, elastic, and stable for production usage</p></li>
-<li><p>Develop distributed ML capabilities</p></li>
+<li><p>Create experimental implementation of <code class="docutils literal notranslate"><span class="pre">StreamProvider</span></code> trait</p></li>
</ul>
-</section>
-</section>
-</section>
+</div>
+</div>
+</div>
+</div>
</div>
@@ -595,7 +545,7 @@
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Datafusion output field name semantic</p>
+ <p class="prev-next-title">DataFusion output field name semantic</p>
</div>
</a>
<a class='right-next' id="next-link" href="../community/communication.html" title="next page">
@@ -619,7 +569,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/specification/rfcs/template.html b/datafusion/specification/rfcs/template.html
index e14a089fd2..62e5dc9fec 100644
--- a/datafusion/specification/rfcs/template.html
+++ b/datafusion/specification/rfcs/template.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Background — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Background — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -68,7 +67,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -116,29 +115,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -154,8 +141,8 @@
<a class="reference internal" href="../../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -177,7 +164,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -207,8 +194,8 @@
<a class="reference internal" href="../../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -229,90 +216,14 @@
</a>
</li>
<li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
- DataFusion Functions
+ <a class="reference internal" href="../../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
</a>
</li>
- </ul>
- </li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../../user-guide/distributed/index.html">
- Ballista Distributed Compute
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
<li class="toctree-l2">
- <a class="reference internal" href="../../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../../user-guide/distributed/clients/index.html">
- Clients
+ <a class="reference internal" href="../../user-guide/sql/datafusion-functions.html">
+ DataFusion Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
</li>
</ul>
</li>
@@ -322,7 +233,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -340,7 +251,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -349,7 +260,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -357,16 +268,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -497,35 +403,35 @@
<p>Start Date: YYYY-MM-DD</p>
<p>Authors:</p>
<p>RFC PR: #</p>
-<p>Datafusion Issue: #</p>
+<p>DataFusion Issue: #</p>
<hr class="docutils" />
-<section id="background">
+<div class="section" id="background">
<h1>Background<a class="headerlink" href="#background" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
<hr class="docutils" />
-<section id="goals">
+<div class="section" id="goals">
<h1>Goals<a class="headerlink" href="#goals" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
<hr class="docutils" />
-<section id="non-goals">
+<div class="section" id="non-goals">
<h1>Non-Goals<a class="headerlink" href="#non-goals" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
<hr class="docutils" />
-<section id="survey">
+<div class="section" id="survey">
<h1>Survey<a class="headerlink" href="#survey" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
<hr class="docutils" />
-<section id="general-design">
+<div class="section" id="general-design">
<h1>General design<a class="headerlink" href="#general-design" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
<hr class="docutils" />
-<section id="detailed-design">
+<div class="section" id="detailed-design">
<h1>Detailed design<a class="headerlink" href="#detailed-design" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
<hr class="docutils" />
-<section id="others">
+<div class="section" id="others">
<h1>Others<a class="headerlink" href="#others" title="Permalink to this headline">¶</a></h1>
-</section>
+</div>
</div>
@@ -547,7 +453,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/specification/roadmap.html b/datafusion/specification/roadmap.html
index 9061c62a69..da036c804f 100644
--- a/datafusion/specification/roadmap.html
+++ b/datafusion/specification/roadmap.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Roadmap — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Roadmap — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="../user-guide/sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="../user-guide/sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../user-guide/distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../user-guide/configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../user-guide/distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../user-guide/distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../user-guide/distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../user-guide/faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -534,7 +445,7 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<section id="roadmap">
+<div class="section" id="roadmap">
<h1>Roadmap<a class="headerlink" href="#roadmap" title="Permalink to this headline">¶</a></h1>
<p>This document describes high level goals of the DataFusion and
Ballista development community. It is not meant to restrict
@@ -546,11 +457,11 @@ Arrow</a> project and governed by the Apache
Software Foundation governance model. These projects are entirely
driven by volunteers, and we welcome contributions for items not on
this roadmap. However, before submitting a large PR, we strongly
-suggest you start a coversation using a github issue or the
+suggest you start a conversation using a github issue or the
dev@arrow.apache.org mailing list to make review efficient and avoid
surprises.</p>
-</section>
-<section id="datafusion">
+</div>
+<div class="section" id="datafusion">
<h1>DataFusion<a class="headerlink" href="#datafusion" title="Permalink to this headline">¶</a></h1>
<p>DataFusion’s goal is to become the embedded query engine of choice
for new analytic applications, by leveraging the unique features of
@@ -560,10 +471,10 @@ to provide:</p>
<li><p>Best-in-class single node query performance</p></li>
<li><p>A Declarative SQL query interface compatible with PostgreSQL</p></li>
<li><p>A Dataframe API, similar to those offered by Pandas and Spark</p></li>
-<li><p>A Procedural API for programatically creating and running execution plans</p></li>
-<li><p>High performance, data race free, erogonomic extensibility points at at every layer</p></li>
+<li><p>A Procedural API for programmatically creating and running execution plans</p></li>
+<li><p>High performance, data race free, ergonomic extensibility points at at every layer</p></li>
</ol>
-<section id="additional-sql-language-features">
+<div class="section" id="additional-sql-language-features">
<h2>Additional SQL Language Features<a class="headerlink" href="#additional-sql-language-features" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Decimal Support <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/122">#122</a></p></li>
@@ -573,41 +484,41 @@ to provide:</p>
<li><p>Support for nested structures (fields, lists, structs) <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/119">#119</a></p></li>
<li><p>Run all queries from the TPCH benchmark (see <a class="reference external" href="https://github.com/apache/arrow-datafusion/milestone/2">milestone</a> for more details)</p></li>
</ul>
-</section>
-<section id="query-optimizer">
+</div>
+<div class="section" id="query-optimizer">
<h2>Query Optimizer<a class="headerlink" href="#query-optimizer" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>More sophisticated cost based optimizer for join ordering</p></li>
<li><p>Implement advanced query optimization framework (Tokomak) #440</p></li>
<li><p>Finer optimizations for group by and aggregate functions</p></li>
</ul>
-</section>
-<section id="datasources">
+</div>
+<div class="section" id="datasources">
<h2>Datasources<a class="headerlink" href="#datasources" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Better support for reading data from remote filesystems (e.g. S3) without caching it locally <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/907">#907</a> <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/1060">#1060</a></p></li>
<li><p>Improve performances of file format datasources (parallelize file listings, async Arrow readers, file chunk prefetching capability…)</p></li>
</ul>
-</section>
-<section id="runtime-infrastructure">
+</div>
+<div class="section" id="runtime-infrastructure">
<h2>Runtime / Infrastructure<a class="headerlink" href="#runtime-infrastructure" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Migrate to some sort of arrow2 based implementation (see <a class="reference external" href="https://github.com/apache/arrow-datafusion/milestone/3">milestone</a> for more details)</p></li>
<li><p>Add DataFusion to h2oai/db-benchmark <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/147">147</a></p></li>
<li><p>Improve build time <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/348">348</a></p></li>
</ul>
-</section>
-<section id="resource-management">
+</div>
+<div class="section" id="resource-management">
<h2>Resource Management<a class="headerlink" href="#resource-management" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p>Finer grain control and limit of runtime memory <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/587">#587</a> and CPU usage <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/64">#54</a></p></li>
</ul>
-</section>
-<section id="python-interface">
+</div>
+<div class="section" id="python-interface">
<h2>Python Interface<a class="headerlink" href="#python-interface" title="Permalink to this headline">¶</a></h2>
<p>TBD</p>
-</section>
-<section id="datafusion-cli-datafusion-cli">
+</div>
+<div class="section" id="datafusion-cli-datafusion-cli">
<h2>DataFusion CLI (<code class="docutils literal notranslate"><span class="pre">datafusion-cli</span></code>)<a class="headerlink" href="#datafusion-cli-datafusion-cli" title="Permalink to this headline">¶</a></h2>
<p>Note: There are some additional thoughts on a datafusion-cli vision on <a class="reference external" href="https://github.com/apache/arrow-datafusion/issues/1096#issuecomment-939418770">#1096</a>.</p>
<ul class="simple">
@@ -617,9 +528,9 @@ to provide:</p>
<li><p>publishing to apt, brew, and possible NuGet registry so that people can use it more easily</p></li>
<li><p>adopt a shorter name, like dfcli?</p></li>
</ul>
-</section>
-</section>
-<section id="ballista">
+</div>
+</div>
+<div class="section" id="ballista">
<h1>Ballista<a class="headerlink" href="#ballista" title="Permalink to this headline">¶</a></h1>
<p>Ballista is a distributed compute platform based on Apache Arrow and DataFusion. It provides a query scheduler that
breaks a physical plan into stages and tasks and then schedules tasks for execution across the available executors
@@ -627,24 +538,24 @@ in the cluster.</p>
<p>Having Ballista as part of the DataFusion codebase helps ensure that DataFusion remains suitable for distributed
compute. For example, it helps ensure that physical query plans can be serialized to protobuf format and that they
remain language-agnostic so that executors can be built in languages other than Rust.</p>
-<section id="ballista-roadmap">
+<div class="section" id="ballista-roadmap">
<h2>Ballista Roadmap<a class="headerlink" href="#ballista-roadmap" title="Permalink to this headline">¶</a></h2>
-</section>
-<section id="move-query-scheduler-into-datafusion">
+</div>
+<div class="section" id="move-query-scheduler-into-datafusion">
<h2>Move query scheduler into DataFusion<a class="headerlink" href="#move-query-scheduler-into-datafusion" title="Permalink to this headline">¶</a></h2>
<p>The Ballista scheduler has some advantages over DataFusion query execution because it doesn’t try to eagerly execute
the entire query at once but breaks it down into a directionally-acyclic graph (DAG) of stages and executes a
configurable number of stages and tasks concurrently. It should be possible to push some of this logic down to
DataFusion so that the same scheduler can be used to scale across cores in-process and across nodes in a cluster.</p>
-</section>
-<section id="implement-execution-time-cost-based-optimizations-based-on-statistics">
+</div>
+<div class="section" id="implement-execution-time-cost-based-optimizations-based-on-statistics">
<h2>Implement execution-time cost-based optimizations based on statistics<a class="headerlink" href="#implement-execution-time-cost-based-optimizations-based-on-statistics" title="Permalink to this headline">¶</a></h2>
<p>After the execution of a query stage, accurate statistics are available for the resulting data. These statistics
could be leveraged by the scheduler to optimize the query during execution. For example, when performing a hash join
it is desirable to load the smaller side of the join into memory and in some cases we cannot predict which side will
be smaller until execution time.</p>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -680,7 +591,7 @@ be smaller until execution time.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/cli.html b/datafusion/user-guide/cli.html
index 119dd99958..8eadc13822 100644
--- a/datafusion/user-guide/cli.html
+++ b/datafusion/user-guide/cli.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>DataFusion Command-line Interface — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>DataFusion Command-line Interface — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -419,8 +330,8 @@
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#ballista">
- Ballista
+ <a class="reference internal nav-link" href="#datafusion-cli">
+ DataFusion-Cli
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
@@ -474,10 +385,9 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="datafusion-command-line-interface">
+<div class="section" id="datafusion-command-line-interface">
<h1>DataFusion Command-line Interface<a class="headerlink" href="#datafusion-command-line-interface" title="Permalink to this headline">¶</a></h1>
-<p>The DataFusion CLI allows SQL queries to be executed by an in-process DataFusion context, or by a distributed
-Ballista context.</p>
+<p>The DataFusion CLI allows SQL queries to be executed by an in-process DataFusion context.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">USAGE</span><span class="p">:</span>
<span class="n">datafusion</span><span class="o">-</span><span class="n">cli</span> <span class="p">[</span><span class="n">FLAGS</span><span class="p">]</span> <span class="p">[</span><span class="n">OPTIONS</span><span class="p">]</span>
@@ -491,11 +401,9 @@ Ballista context.</p>
<span class="o">-</span><span class="n">p</span><span class="p">,</span> <span class="o">--</span><span class="n">data</span><span class="o">-</span><span class="n">path</span> <span class="o"><</span><span class="n">data</span><span class="o">-</span><span class="n">path</span><span class="o">></span> <span class="n">Path</span> <span class="n">to</span> <span class="n">your</span> <span class="n">data</span><span class="p">,</span> <span class="n">default</span> <span cl [...]
<span class="o">-</span><span class="n">f</span><span class="p">,</span> <span class="o">--</span><span class="n">file</span> <span class="o"><</span><span class="n">file</span><span class="o">>...</span> <span class="n">Execute</span> <span class="n">commands</span> <span class="kn">from</span> <span class="nn">file</span><span class="p">(</span><span class="n">s</span><span class="p">),</span> <span class="n">then</span> <span class="n">exit</span>
<span class="o">--</span><span class="nb">format</span> <span class="o"><</span><span class="nb">format</span><span class="o">></span> <span class="n">Output</span> <span class="nb">format</span> <span class="p">[</span><span class="n">default</span><span class="p">:</span> <span class="n">table</span><span class="p">]</span> <span class="p">[</span><span class="n">possible</span> <span class="n">values</span><span class="p">:</span> <span class="n">csv</span><s [...]
- <span class="o">--</span><span class="n">host</span> <span class="o"><</span><span class="n">host</span><span class="o">></span> <span class="n">Ballista</span> <span class="n">scheduler</span> <span class="n">host</span>
- <span class="o">--</span><span class="n">port</span> <span class="o"><</span><span class="n">port</span><span class="o">></span> <span class="n">Ballista</span> <span class="n">scheduler</span> <span class="n">port</span>
</pre></div>
</div>
-<section id="example">
+<div class="section" id="example">
<h2>Example<a class="headerlink" href="#example" title="Permalink to this headline">¶</a></h2>
<p>Create a CSV file to query.</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ <span class="nb">echo</span> <span class="s2">"1,2"</span> > data.csv
@@ -503,7 +411,7 @@ Ballista context.</p>
</div>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ datafusion-cli
-DataFusion CLI v5.1.0-SNAPSHOT
+DataFusion CLI v8.0.0
> CREATE EXTERNAL TABLE foo <span class="o">(</span>a INT, b INT<span class="o">)</span> STORED AS CSV LOCATION <span class="s1">'data.csv'</span><span class="p">;</span>
<span class="m">0</span> rows <span class="k">in</span> set. Query took <span class="m">0</span>.001 seconds.
@@ -517,15 +425,16 @@ DataFusion CLI v5.1.0-SNAPSHOT
<span class="m">1</span> row <span class="k">in</span> set. Query took <span class="m">0</span>.017 seconds.
</pre></div>
</div>
-</section>
-<section id="ballista">
-<h2>Ballista<a class="headerlink" href="#ballista" title="Permalink to this headline">¶</a></h2>
-<p>The DataFusion CLI can also connect to a Ballista scheduler for query execution.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>datafusion-cli --host localhost --port <span class="m">50050</span>
+</div>
+<div class="section" id="datafusion-cli">
+<h2>DataFusion-Cli<a class="headerlink" href="#datafusion-cli" title="Permalink to this headline">¶</a></h2>
+<p>Build the <code class="docutils literal notranslate"><span class="pre">datafusion-cli</span></code>:</p>
+<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span> arrow-datafusion/datafusion-cli
+cargo build
</pre></div>
</div>
-</section>
-<section id="cli-commands">
+</div>
+<div class="section" id="cli-commands">
<h2>Cli commands<a class="headerlink" href="#cli-commands" title="Permalink to this headline">¶</a></h2>
<p>Available commands inside DataFusion CLI are:</p>
<ul class="simple">
@@ -570,8 +479,8 @@ DataFusion CLI v5.1.0-SNAPSHOT
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>> <span class="se">\h</span> function_table
</pre></div>
</div>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -607,7 +516,7 @@ DataFusion CLI v5.1.0-SNAPSHOT
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/faq.html b/datafusion/user-guide/configs.html
similarity index 66%
copy from datafusion/user-guide/faq.html
copy to datafusion/user-guide/configs.html
index 7b1a39c9ea..332fe2488b 100644
--- a/datafusion/user-guide/faq.html
+++ b/datafusion/user-guide/configs.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Frequently Asked Questions — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Configuration Settings — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -35,8 +34,8 @@
<script src="../_static/language_data.js"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Roadmap" href="../specification/roadmap.html" />
- <link rel="prev" title="Python" href="distributed/clients/python.html" />
+ <link rel="next" title="Frequently Asked Questions" href="faq.html" />
+ <link rel="prev" title="DataFusion-Specific Functions" href="sql/datafusion-functions.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -231,100 +218,29 @@
</a>
</li>
<li class="toctree-l2">
- <a class="reference internal" href="sql/datafusion-functions.html">
- DataFusion Functions
+ <a class="reference internal" href="sql/aggregate_functions.html">
+ Aggregate Functions
</a>
</li>
- </ul>
- </li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="distributed/index.html">
- Ballista Distributed Compute
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
<li class="toctree-l2">
- <a class="reference internal" href="distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/clients/index.html">
- Clients
+ <a class="reference internal" href="sql/datafusion-functions.html">
+ DataFusion Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
</li>
</ul>
</li>
<li class="toctree-l1 current active">
<a class="current reference internal" href="#">
+ Configuration Settings
+ </a>
+ </li>
+ <li class="toctree-l1">
+ <a class="reference internal" href="faq.html">
Frequently Asked Questions
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -407,19 +318,9 @@
<div class="toc-item">
-<div class="tocsection onthispage pt-5 pb-3">
- <i class="fas fa-list"></i> On this page
-</div>
<nav id="bd-toc-nav">
- <ul class="visible nav section-nav flex-column">
- <li class="toc-h2 nav-item toc-entry">
- <a class="reference internal nav-link" href="#what-is-the-relationship-between-apache-arrow-datafusion-and-ballista">
- What is the relationship between Apache Arrow, DataFusion, and Ballista?
- </a>
- </li>
-</ul>
-
+
</nav>
</div>
@@ -427,7 +328,7 @@
<div class="tocsection editthispage">
- <a href="https://github.com/apache/arrow-datafusion/edit/master/docs/source/user-guide/faq.md">
+ <a href="https://github.com/apache/arrow-datafusion/edit/master/docs/source/user-guide/configs.md">
<i class="fas fa-pencil-alt"></i> Edit this page
</a>
</div>
@@ -464,18 +365,53 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="frequently-asked-questions">
-<h1>Frequently Asked Questions<a class="headerlink" href="#frequently-asked-questions" title="Permalink to this headline">¶</a></h1>
-<section id="what-is-the-relationship-between-apache-arrow-datafusion-and-ballista">
-<h2>What is the relationship between Apache Arrow, DataFusion, and Ballista?<a class="headerlink" href="#what-is-the-relationship-between-apache-arrow-datafusion-and-ballista" title="Permalink to this headline">¶</a></h2>
-<p>Apache Arrow is a library which provides a standardized memory representation for columnar data. It also provides
-“kernels” for performing common operations on this data.</p>
-<p>DataFusion is a library for executing queries in-process using the Apache Arrow memory
-model and computational kernels. It is designed to run within a single process, using threads
-for parallel query execution.</p>
-<p>Ballista is a distributed compute platform built on DataFusion.</p>
-</section>
-</section>
+<!---
+This file was generated by the dev/update_config_docs.sh script.
+Do not edit it manually as changes will be overwritten.
+Instead, edit dev/update_config_docs.sh or the docstrings in datafusion/core/src/config.rs.
+-->
+<div class="section" id="configuration-settings">
+<h1>Configuration Settings<a class="headerlink" href="#configuration-settings" title="Permalink to this headline">¶</a></h1>
+<p>The following configuration options can be passed to <code class="docutils literal notranslate"><span class="pre">SessionConfig</span></code> to control various aspects of query execution.</p>
+<p>For applications which do not expose <code class="docutils literal notranslate"><span class="pre">SessionConfig</span></code>, like <code class="docutils literal notranslate"><span class="pre">datafusion-cli</span></code>, these options may also be set via environment variables.
+To construct a session with options from the environment, use <code class="docutils literal notranslate"><span class="pre">SessionConfig::from_env</span></code>.
+The name of the environment variable is the option’s key, transformed to uppercase and with periods replaced with underscores.
+For example, to configure <code class="docutils literal notranslate"><span class="pre">datafusion.execution.batch_size</span></code> you would set the <code class="docutils literal notranslate"><span class="pre">DATAFUSION_EXECUTION_BATCH_SIZE</span></code> environment variable.
+Values are parsed according to the <a class="reference external" href="https://docs.rs/arrow/latest/arrow/compute/kernels/cast/fn.cast.html">same rules used in casts from Utf8</a>.
+If the value in the environment variable cannot be cast to the type of the configuration option, the default value will be used instead and a warning emitted.
+Environment variables are read during <code class="docutils literal notranslate"><span class="pre">SessionConfig</span></code> initialisation so they must be set beforehand and will not affect running sessions.</p>
+<table class="colwidths-auto table">
+<thead>
+<tr class="row-odd"><th class="head"><p>key</p></th>
+<th class="head"><p>type</p></th>
+<th class="head"><p>default</p></th>
+<th class="head"><p>description</p></th>
+</tr>
+</thead>
+<tbody>
+<tr class="row-even"><td><p>datafusion.execution.batch_size</p></td>
+<td><p>UInt64</p></td>
+<td><p>8192</p></td>
+<td><p>Default batch size while creating new batches, it’s especially useful for buffer-in-memory batches since creating tiny batches would results in too much metadata memory consumption.</p></td>
+</tr>
+<tr class="row-odd"><td><p>datafusion.execution.coalesce_batches</p></td>
+<td><p>Boolean</p></td>
+<td><p>true</p></td>
+<td><p>When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting ‘datafusion.execution.coalesce_target_batch_size’.</p></td>
+</tr>
+<tr class="row-even"><td><p>datafusion.execution.coalesce_target_batch_size</p></td>
+<td><p>UInt64</p></td>
+<td><p>4096</p></td>
+<td><p>Target batch size when coalescing batches. Uses in conjunction with the configuration setting ‘datafusion.execution.coalesce_batches’.</p></td>
+</tr>
+<tr class="row-odd"><td><p>datafusion.optimizer.filter_null_join_keys</p></td>
+<td><p>Boolean</p></td>
+<td><p>false</p></td>
+<td><p>When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down.</p></td>
+</tr>
+</tbody>
+</table>
+</div>
</div>
@@ -483,17 +419,17 @@ for parallel query execution.</p>
<!-- Previous / next buttons -->
<div class='prev-next-area'>
- <a class='left-prev' id="prev-link" href="distributed/clients/python.html" title="previous page">
+ <a class='left-prev' id="prev-link" href="sql/datafusion-functions.html" title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Python</p>
+ <p class="prev-next-title">DataFusion-Specific Functions</p>
</div>
</a>
- <a class='right-next' id="next-link" href="../specification/roadmap.html" title="next page">
+ <a class='right-next' id="next-link" href="faq.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Roadmap</p>
+ <p class="prev-next-title">Frequently Asked Questions</p>
</div>
<i class="fas fa-angle-right"></i>
</a>
@@ -511,7 +447,7 @@ for parallel query execution.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/example-usage.html b/datafusion/user-guide/example-usage.html
index c7c4843d4b..b4d6a3325b 100644
--- a/datafusion/user-guide/example-usage.html
+++ b/datafusion/user-guide/example-usage.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Example Usage — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Example Usage — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -484,24 +395,24 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="example-usage">
+<div class="section" id="example-usage">
<h1>Example Usage<a class="headerlink" href="#example-usage" title="Permalink to this headline">¶</a></h1>
-<section id="update-cargo-toml">
+<div class="section" id="update-cargo-toml">
<h2>Update <code class="docutils literal notranslate"><span class="pre">Cargo.toml</span></code><a class="headerlink" href="#update-cargo-toml" title="Permalink to this headline">¶</a></h2>
<p>Add the following to your <code class="docutils literal notranslate"><span class="pre">Cargo.toml</span></code> file:</p>
-<div class="highlight-toml notranslate"><div class="highlight"><pre><span></span><span class="n">datafusion</span> <span class="o">=</span> <span class="s">"7.0.0"</span>
-<span class="n">tokio</span> <span class="o">=</span> <span class="s">"1.0"</span>
+<div class="highlight-toml notranslate"><div class="highlight"><pre><span></span><span class="n">datafusion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"8.0.0"</span><span class="w"></span>
+<span class="n">tokio</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"1.0"</span><span class="w"></span>
</pre></div>
</div>
-</section>
-<section id="run-a-sql-query-against-data-stored-in-a-csv">
+</div>
+<div class="section" id="run-a-sql-query-against-data-stored-in-a-csv">
<h2>Run a SQL query against data stored in a CSV:<a class="headerlink" href="#run-a-sql-query-against-data-stored-in-a-csv" title="Permalink to this headline">¶</a></h2>
<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">prelude</span>::<span class="o">*</span><span class="p">;</span><span class="w"></span>
<span class="cp">#[tokio::main]</span><span class="w"></span>
<span class="k">async</span><span class="w"> </span><span class="k">fn</span> <span class="nf">main</span><span class="p">()</span><span class="w"> </span>-> <span class="nc">datafusion</span>::<span class="n">error</span>::<span class="nb">Result</span><span class="o"><</span><span class="p">()</span><span class="o">></span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="w"> </span><span class="c1">// register the table</span>
-<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="k">mut</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ExecutionContext</span>::<span class="n">new</span><span class="p">();</span><span class="w"></span>
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">SessionContext</span>::<span class="n">new</span><span class="p">();</span><span class="w"></span>
<span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">register_csv</span><span class="p">(</span><span class="s">"example"</span><span class="p">,</span><span class="w"> </span><span class="s">"tests/example.csv"</span><span class="p">,</span><span class="w"> </span><span class="n">CsvReadOptions</span>::<span class="n">new</span><span class="p">()).</span><span class="k">await</span><span class="o">?</span><span class="p">;</span> [...]
<span class="w"> </span><span class="c1">// create a plan to run a SQL query</span>
@@ -513,15 +424,15 @@
<span class="p">}</span><span class="w"></span>
</pre></div>
</div>
-</section>
-<section id="use-the-dataframe-api-to-process-data-stored-in-a-csv">
+</div>
+<div class="section" id="use-the-dataframe-api-to-process-data-stored-in-a-csv">
<h2>Use the DataFrame API to process data stored in a CSV:<a class="headerlink" href="#use-the-dataframe-api-to-process-data-stored-in-a-csv" title="Permalink to this headline">¶</a></h2>
<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">prelude</span>::<span class="o">*</span><span class="p">;</span><span class="w"></span>
<span class="cp">#[tokio::main]</span><span class="w"></span>
<span class="k">async</span><span class="w"> </span><span class="k">fn</span> <span class="nf">main</span><span class="p">()</span><span class="w"> </span>-> <span class="nc">datafusion</span>::<span class="n">error</span>::<span class="nb">Result</span><span class="o"><</span><span class="p">()</span><span class="o">></span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="w"> </span><span class="c1">// create the dataframe</span>
-<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="k">mut</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ExecutionContext</span>::<span class="n">new</span><span class="p">();</span><span class="w"></span>
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">SessionContext</span>::<span class="n">new</span><span class="p">();</span><span class="w"></span>
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">df</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">"tests/example.csv"</span><span class="p">,</span><span class="w"> </span><span class="n">CsvReadOptions</span>::<span class="n">new</span><span class="p">()).</span><span class="k">await< [...]
<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">df</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">df</span><span class="p">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s">"a"</span><span class="p">).</span><span class="n">lt_eq</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><spa [...]
@@ -533,8 +444,8 @@
<span class="p">}</span><span class="w"></span>
</pre></div>
</div>
-</section>
-<section id="output-from-both-examples">
+</div>
+<div class="section" id="output-from-both-examples">
<h2>Output from both examples<a class="headerlink" href="#output-from-both-examples" title="Permalink to this headline">¶</a></h2>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>+---+--------+
| a | MIN(b) |
@@ -543,8 +454,8 @@
+---+--------+
</pre></div>
</div>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -580,7 +491,7 @@
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/faq.html b/datafusion/user-guide/faq.html
index 7b1a39c9ea..e4243d3eeb 100644
--- a/datafusion/user-guide/faq.html
+++ b/datafusion/user-guide/faq.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Frequently Asked Questions — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Frequently Asked Questions — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -36,7 +35,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Roadmap" href="../specification/roadmap.html" />
- <link rel="prev" title="Python" href="distributed/clients/python.html" />
+ <link rel="prev" title="Configuration Settings" href="configs.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1 current active">
<a class="current reference internal" href="#">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -464,18 +375,18 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="frequently-asked-questions">
+<div class="section" id="frequently-asked-questions">
<h1>Frequently Asked Questions<a class="headerlink" href="#frequently-asked-questions" title="Permalink to this headline">¶</a></h1>
-<section id="what-is-the-relationship-between-apache-arrow-datafusion-and-ballista">
+<div class="section" id="what-is-the-relationship-between-apache-arrow-datafusion-and-ballista">
<h2>What is the relationship between Apache Arrow, DataFusion, and Ballista?<a class="headerlink" href="#what-is-the-relationship-between-apache-arrow-datafusion-and-ballista" title="Permalink to this headline">¶</a></h2>
<p>Apache Arrow is a library which provides a standardized memory representation for columnar data. It also provides
“kernels” for performing common operations on this data.</p>
<p>DataFusion is a library for executing queries in-process using the Apache Arrow memory
model and computational kernels. It is designed to run within a single process, using threads
for parallel query execution.</p>
-<p>Ballista is a distributed compute platform built on DataFusion.</p>
-</section>
-</section>
+<p><a class="reference external" href="https://github.com/apache/arrow-ballista">Ballista</a> is a distributed compute platform built on DataFusion.</p>
+</div>
+</div>
</div>
@@ -483,11 +394,11 @@ for parallel query execution.</p>
<!-- Previous / next buttons -->
<div class='prev-next-area'>
- <a class='left-prev' id="prev-link" href="distributed/clients/python.html" title="previous page">
+ <a class='left-prev' id="prev-link" href="configs.html" title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Python</p>
+ <p class="prev-next-title">Configuration Settings</p>
</div>
</a>
<a class='right-next' id="next-link" href="../specification/roadmap.html" title="next page">
@@ -511,7 +422,7 @@ for parallel query execution.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/introduction.html b/datafusion/user-guide/introduction.html
index cade389ec1..a1cbaf6450 100644
--- a/datafusion/user-guide/introduction.html
+++ b/datafusion/user-guide/introduction.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Introduction — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Introduction — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -469,7 +380,7 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="introduction">
+<div class="section" id="introduction">
<h1>Introduction<a class="headerlink" href="#introduction" title="Permalink to this headline">¶</a></h1>
<p>DataFusion is an extensible query execution framework, written in
Rust, that uses <a class="reference external" href="https://arrow.apache.org">Apache Arrow</a> as its
@@ -478,14 +389,14 @@ in-memory format.</p>
logical query plans as well as a query optimizer and execution engine
capable of parallel execution against partitioned data sources (CSV
and Parquet) using threads.</p>
-<section id="use-cases">
+<div class="section" id="use-cases">
<h2>Use Cases<a class="headerlink" href="#use-cases" title="Permalink to this headline">¶</a></h2>
<p>DataFusion is used to create modern, fast and efficient data
pipelines, ETL processes, and database systems, which need the
performance of Rust and Apache Arrow and want to provide their users
the convenience of an SQL interface or a DataFrame API.</p>
-</section>
-<section id="why-datafusion">
+</div>
+<div class="section" id="why-datafusion">
<h2>Why DataFusion?<a class="headerlink" href="#why-datafusion" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
<li><p><em>High Performance</em>: Leveraging Rust and Arrow’s memory model, DataFusion achieves very high performance</p></li>
@@ -493,8 +404,8 @@ the convenience of an SQL interface or a DataFrame API.</p>
<li><p><em>Easy to Embed</em>: Allowing extension at almost any point in its design, DataFusion can be tailored for your specific usecase</p></li>
<li><p><em>High Quality</em>: Extensively tested, both by itself and with the rest of the Arrow ecosystem, DataFusion can be used as the foundation for production systems.</p></li>
</ul>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -530,7 +441,7 @@ the convenience of an SQL interface or a DataFrame API.</p>
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/library.html b/datafusion/user-guide/library.html
index 34d878db8e..7898482574 100644
--- a/datafusion/user-guide/library.html
+++ b/datafusion/user-guide/library.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>Using DataFusion as a library — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Using DataFusion as a library — Arrow DataFusion documentation</title>
<link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -209,8 +196,8 @@
<a class="reference internal" href="sql/index.html">
SQL Reference
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="sql/aggregate_functions.html">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="sql/datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/deployment/configuration.html">
- Configuration
- </a>
- </li>
- </ul>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="distributed/clients/index.html">
- Clients
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
- <label for="toctree-checkbox-10">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/rust.html">
- Ballista Rust Client
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="distributed/clients/python.html">
- Python
- </a>
- </li>
- </ul>
- </li>
- </ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="faq.html">
@@ -324,7 +240,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Specification
</span>
@@ -342,7 +258,7 @@
</li>
<li class="toctree-l1">
<a class="reference internal" href="../specification/output-field-name-semantic.html">
- Datafusion output field name semantic
+ DataFusion output field name semantic
</a>
</li>
<li class="toctree-l1">
@@ -351,7 +267,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
README
</span>
@@ -359,16 +275,11 @@
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow-datafusion/blob/master/README.md">
- Datafusion
- </a>
- </li>
- <li class="toctree-l1">
- <a class="reference external" href="https://github.com/apache/arrow-datafusion/tree/master/ballista/README.md">
- Ballista
+ DataFusion
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
Community
</span>
@@ -413,11 +324,21 @@
<nav id="bd-toc-nav">
<ul class="visible nav section-nav flex-column">
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#create-a-new-project">
+ Create a new project
+ </a>
+ </li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#default-configuration">
Default Configuration
</a>
</li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#create-a-main-function">
+ Create a main function
+ </a>
+ </li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#optimized-configuration">
Optimized Configuration
@@ -469,44 +390,90 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="using-datafusion-as-a-library">
+<div class="section" id="using-datafusion-as-a-library">
<h1>Using DataFusion as a library<a class="headerlink" href="#using-datafusion-as-a-library" title="Permalink to this headline">¶</a></h1>
-<section id="default-configuration">
+<div class="section" id="create-a-new-project">
+<h2>Create a new project<a class="headerlink" href="#create-a-new-project" title="Permalink to this headline">¶</a></h2>
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>cargo new hello_datafusion
+</pre></div>
+</div>
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>$ <span class="nb">cd</span> hello_datafusion
+$ tree .
+.
+├── Cargo.toml
+└── src
+ └── main.rs
+
+<span class="m">1</span> directory, <span class="m">2</span> files
+</pre></div>
+</div>
+</div>
+<div class="section" id="default-configuration">
<h2>Default Configuration<a class="headerlink" href="#default-configuration" title="Permalink to this headline">¶</a></h2>
<p>DataFusion is <a class="reference external" href="https://crates.io/crates/datafusion">published on crates.io</a>, and is <a class="reference external" href="https://docs.rs/datafusion/">well documented on docs.rs</a>.</p>
<p>To get started, add the following to your <code class="docutils literal notranslate"><span class="pre">Cargo.toml</span></code> file:</p>
-<div class="highlight-toml notranslate"><div class="highlight"><pre><span></span><span class="k">[dependencies]</span>
-<span class="n">datafusion</span> <span class="o">=</span> <span class="s">"5.1.0"</span>
+<div class="highlight-toml notranslate"><div class="highlight"><pre><span></span><span class="k">[dependencies]</span><span class="w"></span>
+<span class="n">datafusion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"8.0.0"</span><span class="w"></span>
</pre></div>
</div>
-</section>
-<section id="optimized-configuration">
+</div>
+<div class="section" id="create-a-main-function">
+<h2>Create a main function<a class="headerlink" href="#create-a-main-function" title="Permalink to this headline">¶</a></h2>
+<p>Update the main.rs file with your first datafusion application based on <a class="reference external" href="https://arrow.apache.org/datafusion/user-guide/example-usage.html">Example usage</a></p>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">prelude</span>::<span class="o">*</span><span class="p">;</span><span class="w"></span>
+
+<span class="cp">#[tokio::main]</span><span class="w"></span>
+<span class="k">async</span><span class="w"> </span><span class="k">fn</span> <span class="nf">main</span><span class="p">()</span><span class="w"> </span>-> <span class="nc">datafusion</span>::<span class="n">error</span>::<span class="nb">Result</span><span class="o"><</span><span class="p">()</span><span class="o">></span><span class="w"> </span><span class="p">{</span><span class="w"></span>
+<span class="w"> </span><span class="c1">// register the table</span>
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">ctx</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">SessionContext</span>::<span class="n">new</span><span class="p">();</span><span class="w"></span>
+<span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">register_csv</span><span class="p">(</span><span class="s">"test"</span><span class="p">,</span><span class="w"> </span><span class="s">"<PATH_TO_YOUR_CSV_FILE>"</span><span class="p">,</span><span class="w"> </span><span class="n">CsvReadOptions</span>::<span class="n">new</span><span class="p">()).</span><span class="k">await</span><span class="o">?</span><span class="p" [...]
+
+<span class="w"> </span><span class="c1">// create a plan to run a SQL query</span>
+<span class="w"> </span><span class="kd">let</span><span class="w"> </span><span class="n">df</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ctx</span><span class="p">.</span><span class="n">sql</span><span class="p">(</span><span class="s">"SELECT * FROM test"</span><span class="p">).</span><span class="k">await</span><span class="o">?</span><span class="p">;</span><span class="w"></span>
+
+<span class="w"> </span><span class="c1">// execute and print results</span>
+<span class="w"> </span><span class="n">df</span><span class="p">.</span><span class="n">show</span><span class="p">().</span><span class="k">await</span><span class="o">?</span><span class="p">;</span><span class="w"></span>
+<span class="w"> </span><span class="nb">Ok</span><span class="p">(())</span><span class="w"></span>
+<span class="p">}</span><span class="w"></span>
+</pre></div>
+</div>
+</div>
+<div class="section" id="optimized-configuration">
<h2>Optimized Configuration<a class="headerlink" href="#optimized-configuration" title="Permalink to this headline">¶</a></h2>
<p>For an optimized build several steps are required. First, use the below in your <code class="docutils literal notranslate"><span class="pre">Cargo.toml</span></code>. It is
worth noting that using the settings in the <code class="docutils literal notranslate"><span class="pre">[profile.release]</span></code> section will significantly increase the build time.</p>
-<div class="highlight-toml notranslate"><div class="highlight"><pre><span></span><span class="k">[dependencies]</span>
-<span class="n">datafusion</span> <span class="o">=</span> <span class="p">{</span> <span class="n">version</span> <span class="o">=</span> <span class="s">"5.0"</span> <span class="p">,</span> <span class="n">features</span> <span class="o">=</span> <span class="p">[</span><span class="s">"simd"</span><span class="p">]}</span>
-<span class="n">tokio</span> <span class="o">=</span> <span class="p">{</span> <span class="n">version</span> <span class="o">=</span> <span class="s">"^1.0"</span><span class="p">,</span> <span class="n">features</span> <span class="o">=</span> <span class="p">[</span><span class="s">"rt-multi-thread"</span><span class="p">]</span> <span class="p">}</span>
-<span class="n">snmalloc-rs</span> <span class="o">=</span> <span class="s">"0.2"</span>
+<div class="highlight-toml notranslate"><div class="highlight"><pre><span></span><span class="k">[dependencies]</span><span class="w"></span>
+<span class="n">datafusion</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="n">version</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"7.0"</span><span class="w"> </span><span class="p">,</span><span class="w"> </span><span class="n">features</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span> [...]
+<span class="n">tokio</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="n">version</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"^1.0"</span><span class="p">,</span><span class="w"> </span><span class="n">features</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span><span class="s">"rt-mul [...]
+<span class="n">snmalloc-rs</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"0.2"</span><span class="w"></span>
-<span class="k">[profile.release]</span>
-<span class="n">lto</span> <span class="o">=</span> <span class="kc">true</span>
-<span class="n">codegen-units</span> <span class="o">=</span> <span class="mi">1</span>
+<span class="k">[profile.release]</span><span class="w"></span>
+<span class="n">lto</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="kc">true</span><span class="w"></span>
+<span class="n">codegen-units</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="w"></span>
</pre></div>
</div>
<p>Then, in <code class="docutils literal notranslate"><span class="pre">main.rs.</span></code> update the memory allocator with the below after your imports:</p>
-<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="cp">#[global_allocator]</span><span class="w"></span>
+<div class="highlight-rust notranslate"><div class="highlight"><pre><span></span><span class="k">use</span><span class="w"> </span><span class="n">datafusion</span>::<span class="n">prelude</span>::<span class="o">*</span><span class="p">;</span><span class="w"></span>
+
+<span class="cp">#[global_allocator]</span><span class="w"></span>
<span class="k">static</span><span class="w"> </span><span class="n">ALLOC</span>: <span class="nc">snmalloc_rs</span>::<span class="n">SnMalloc</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">snmalloc_rs</span>::<span class="n">SnMalloc</span><span class="p">;</span><span class="w"></span>
+
+<span class="k">async</span><span class="w"> </span><span class="k">fn</span> <span class="nf">main</span><span class="p">()</span><span class="w"> </span>-> <span class="nc">datafusion</span>::<span class="n">error</span>::<span class="nb">Result</span><span class="o"><</span><span class="p">()</span><span class="o">></span><span class="w"> </span><span class="p">{</span><span class="w"></span>
+<span class="w"> </span><span class="o">..</span><span class="p">.</span><span class="w"></span>
+<span class="p">}</span><span class="w"></span>
+</pre></div>
+</div>
+<p>Finally, in order to build with the <code class="docutils literal notranslate"><span class="pre">simd</span></code> optimization <code class="docutils literal notranslate"><span class="pre">cargo</span> <span class="pre">nightly</span></code> is required.</p>
+<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>rustup toolchain install nightly
</pre></div>
</div>
-<p>Finally, in order to build with the <code class="docutils literal notranslate"><span class="pre">simd</span></code> optimization <code class="docutils literal notranslate"><span class="pre">cargo</span> <span class="pre">nightly</span></code> is required. Based on the instruction
-set architecture you are building on you will want to configure the <code class="docutils literal notranslate"><span class="pre">target-cpu</span></code> as well, ideally
+<p>Based on the instruction set architecture you are building on you will want to configure the <code class="docutils literal notranslate"><span class="pre">target-cpu</span></code> as well, ideally
with <code class="docutils literal notranslate"><span class="pre">native</span></code> or at least <code class="docutils literal notranslate"><span class="pre">avx2</span></code>.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">RUSTFLAGS</span><span class="o">=</span><span class="s1">'-C target-cpu=native'</span> <span class="n">cargo</span> <span class="o">+</span><span class="n">nightly</span> <span class="n">run</span> <span class="o">--</span><span class="n">release</span>
</pre></div>
</div>
-</section>
-</section>
+</div>
+</div>
</div>
@@ -542,7 +509,7 @@ with <code class="docutils literal notranslate"><span class="pre">native</span><
<div class="footer-item">
<p class="copyright">
- © Copyright 2021, Apache Software Foundation.<br>
+ © Copyright 2022, Apache Software Foundation.<br>
</p>
</div>
diff --git a/datafusion/user-guide/sql/index.html b/datafusion/user-guide/sql/aggregate_functions.html
similarity index 56%
copy from datafusion/user-guide/sql/index.html
copy to datafusion/user-guide/sql/aggregate_functions.html
index b8e277605a..4e685e1a22 100644
--- a/datafusion/user-guide/sql/index.html
+++ b/datafusion/user-guide/sql/aggregate_functions.html
@@ -3,9 +3,8 @@
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
- <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
-
- <title>SQL Reference — Arrow Datafusion documentation</title>
+ <meta charset="utf-8" />
+ <title>Aggregate Functions — Arrow DataFusion documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
@@ -35,8 +34,8 @@
<script src="../../_static/language_data.js"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Status" href="sql_status.html" />
- <link rel="prev" title="DataFusion Command-line Interface" href="../cli.html" />
+ <link rel="next" title="DataFusion-Specific Functions" href="datafusion-functions.html" />
+ <link rel="prev" title="DDL" href="ddl.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="None">
@@ -70,7 +69,7 @@
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
- <p>
+ <p class="caption">
<span class="caption-text">
Supported Environments
</span>
@@ -118,29 +117,17 @@
</li>
</ul>
</li>
- <li class="toctree-l3 has-children">
+ <li class="toctree-l3">
<a class="reference internal" href="../../python/api/execution_context.html">
- ExecutionContext
+ SessionContext
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
- <label for="toctree-checkbox-4">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l4">
- <a class="reference internal" href="../../python/generated/datafusion.ExecutionContext.html">
- datafusion.ExecutionContext
- </a>
- </li>
- </ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../../python/api/expression.html">
Expression
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
- <label for="toctree-checkbox-5">
+ <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
+ <label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -156,8 +143,8 @@
<a class="reference internal" href="../../python/api/functions.html">
Functions
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
- <label for="toctree-checkbox-6">
+ <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
+ <label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
@@ -179,7 +166,7 @@
</a>
</li>
</ul>
-<p>
+<p class="caption">
<span class="caption-text">
User Guide
</span>
@@ -206,15 +193,15 @@
</a>
</li>
<li class="toctree-l1 current active has-children">
- <a class="current reference internal" href="#">
+ <a class="reference internal" href="index.html">
SQL Reference
</a>
- <input checked="" class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
- <label for="toctree-checkbox-7">
+ <input checked="" class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
+ <label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
- <ul>
+ <ul class="current">
<li class="toctree-l2">
<a class="reference internal" href="sql_status.html">
Status
@@ -230,6 +217,11 @@
DDL
</a>
</li>
+ <li class="toctree-l2 current active">
+ <a class="current reference internal" href="#">
+ Aggregate Functions
+ </a>
+ </li>
<li class="toctree-l2">
<a class="reference internal" href="datafusion-functions.html">
DataFusion Functions
@@ -237,86 +229,10 @@
</li>
</ul>
</li>
- <li class="toctree-l1 has-children">
- <a class="reference internal" href="../distributed/index.html">
- Ballista Distributed Compute
+ <li class="toctree-l1">
+ <a class="reference internal" href="../configs.html">
+ Configuration Settings
</a>
- <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
- <label for="toctree-checkbox-8">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l2">
- <a class="reference internal" href="../distributed/introduction.html">
- Overview
- </a>
- </li>
- <li class="toctree-l2 has-children">
- <a class="reference internal" href="../distributed/deployment/index.html">
- Start a Ballista Cluster
- </a>
- <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
- <label for="toctree-checkbox-9">
- <i class="fas fa-chevron-down">
- </i>
- </label>
- <ul>
- <li class="toctree-l3">
- <a class="reference internal" href="../distributed/deployment/cargo-install.html">
- Deploying a standalone Ballista cluster using cargo install
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../distributed/deployment/docker.html">
- Starting a Ballista cluster using Docker
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../distributed/deployment/docker-compose.html">
- Starting a Ballista cluster using Docker Compose
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../distributed/deployment/kubernetes.html">
- Deploying Ballista with Kubernetes
- </a>
- </li>
- <li class="toctree-l3">
- <a class="reference internal" href="../distributed/deployment/raspberrypi.html">
- Running Ballista on Raspberry Pi
... 2411 lines suppressed ...