You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2024/05/19 14:31:54 UTC
(datafusion-site) 01/01: initial site
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-site.git
commit d5180cdca56a074fb70d73afd550e6d5d1b3f39c
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sun May 19 08:26:15 2024 -0600
initial site
---
404.html | 87 +
README.md | 3 -
about/index.html | 80 +
assets/datafusion_fast_grouping/full.png | Bin 0 -> 50146 bytes
assets/datafusion_fast_grouping/summary.png | Bin 0 -> 52822 bytes
assets/main.css | 649 +++++++
assets/main.css.map | 1 +
assets/minima-social-icons.svg | 33 +
blog/2019/02/03/datafusion-donation/index.html | 178 ++
blog/2021/04/12/ballista-donation/index.html | 129 ++
blog/2021/08/18/ballista-0.5.0/index.html | 144 ++
blog/2021/08/18/datafusion-5.0.0/index.html | 175 ++
blog/2021/11/19/datafusion-6.0.0/index.html | 225 +++
blog/2022/02/28/datafusion-7.0.0/index.html | 261 +++
blog/2022/03/21/datafusion-contrib/index.html | 276 +++
blog/2022/05/16/datafusion-8.0.0/index.html | 266 +++
blog/2022/10/25/datafusion-13.0.0/index.html | 306 ++++
blog/2022/10/28/ballista-0.9.0/index.html | 172 ++
blog/2023/01/19/datafusion-16.0.0/index.html | 326 ++++
blog/2023/06/24/datafusion-25.0.0/index.html | 303 ++++
.../2023/08/05/datafusion_fast_grouping/index.html | 460 +++++
blog/2024/01/19/datafusion-34.0.0/index.html | 358 ++++
blog/2024/03/06/comet-donation/index.html | 136 ++
blog/2024/05/07/datafusion-tlp/index.html | 135 ++
feed.xml | 1902 ++++++++++++++++++++
img/2021-08-18-datafusion500perf.png | Bin 0 -> 21245 bytes
img/2x_bgwhite_original.png | Bin 0 -> 60451 bytes
img/datafusion-34.0.0/compare-new.png | Bin 0 -> 176796 bytes
img/datafusion-34.0.0/compare.png | Bin 0 -> 149835 bytes
img/datafusion-comet/comet-architecture.png | Bin 0 -> 30027 bytes
index.html | 152 ++
31 files changed, 6754 insertions(+), 3 deletions(-)
diff --git a/404.html b/404.html
new file mode 100644
index 0000000..8e6f5b6
--- /dev/null
+++ b/404.html
@@ -0,0 +1,87 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache DataFusion Project News & Blog | Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache DataFusion Project News & Blog" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format." />
+<meta property="og:description" content="Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format." />
+<link rel="canonical" href="https://datafusion.apache.org/blog/404.html" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/404.html" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="website" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache DataFusion Project News & Blog" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"WebPage","description":"Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.","headline":"Apache DataFusion Project News & Blog","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwhite_original.png"}},"url":"https://datafusion.apache.org/blog/404.html"}</script>
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <style type="text/css" media="screen">
+ .container {
+ margin: 10px auto;
+ max-width: 600px;
+ text-align: center;
+ }
+ h1 {
+ margin: 30px 0;
+ font-size: 4em;
+ line-height: 1;
+ letter-spacing: -1px;
+ }
+</style>
+
+<div class="container">
+ <h1>404</h1>
+
+ <p><strong>Page not found :(</strong></p>
+ <p>The requested page could not be found.</p>
+</div>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/README.md b/README.md
deleted file mode 100644
index f453edf..0000000
--- a/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Apache DataFusion Web Site
-
-Coming soon
diff --git a/about/index.html b/about/index.html
new file mode 100644
index 0000000..4df8b1e
--- /dev/null
+++ b/about/index.html
@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>About | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="About" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format." />
+<meta property="og:description" content="Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format." />
+<link rel="canonical" href="https://datafusion.apache.org/blog/about/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/about/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="website" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="About" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"WebSite","description":"Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.","headline":"About","name":"Apache DataFusion Project News & Blog","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwhite_original.png"}},"url":"https://datafusion.apache.org/blog/about/"}</script>
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post">
+
+ <header class="post-header">
+ <h1 class="post-title">About</h1>
+ </header>
+
+ <div class="post-content">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality
+data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+
+ </div>
+
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/assets/datafusion_fast_grouping/full.png b/assets/datafusion_fast_grouping/full.png
new file mode 100644
index 0000000..9f07b04
Binary files /dev/null and b/assets/datafusion_fast_grouping/full.png differ
diff --git a/assets/datafusion_fast_grouping/summary.png b/assets/datafusion_fast_grouping/summary.png
new file mode 100644
index 0000000..f92df82
Binary files /dev/null and b/assets/datafusion_fast_grouping/summary.png differ
diff --git a/assets/main.css b/assets/main.css
new file mode 100644
index 0000000..5ae9716
--- /dev/null
+++ b/assets/main.css
@@ -0,0 +1,649 @@
+/**
+ * Reset some basic elements
+ */
+body, h1, h2, h3, h4, h5, h6,
+p, blockquote, pre, hr,
+dl, dd, ol, ul, figure {
+ margin: 0;
+ padding: 0;
+}
+
+/**
+ * Basic styling
+ */
+body {
+ font: 400 16px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+ color: #111;
+ background-color: #fdfdfd;
+ -webkit-text-size-adjust: 100%;
+ -webkit-font-feature-settings: "kern" 1;
+ -moz-font-feature-settings: "kern" 1;
+ -o-font-feature-settings: "kern" 1;
+ font-feature-settings: "kern" 1;
+ font-kerning: normal;
+ display: flex;
+ min-height: 100vh;
+ flex-direction: column;
+}
+
+/**
+ * Set `margin-bottom` to maintain vertical rhythm
+ */
+h1, h2, h3, h4, h5, h6,
+p, blockquote, pre,
+ul, ol, dl, figure,
+.highlight {
+ margin-bottom: 15px;
+}
+
+/**
+ * `main` element
+ */
+main {
+ display: block; /* Default value of `display` of `main` element is 'inline' in IE 11. */
+}
+
+/**
+ * Images
+ */
+img {
+ max-width: 100%;
+ vertical-align: middle;
+}
+
+/**
+ * Figures
+ */
+figure > img {
+ display: block;
+}
+
+figcaption {
+ font-size: 14px;
+}
+
+/**
+ * Lists
+ */
+ul, ol {
+ margin-left: 30px;
+}
+
+li > ul,
+li > ol {
+ margin-bottom: 0;
+}
+
+/**
+ * Headings
+ */
+h1, h2, h3, h4, h5, h6 {
+ font-weight: 400;
+}
+
+/**
+ * Links
+ */
+a {
+ color: #2a7ae2;
+ text-decoration: none;
+}
+a:visited {
+ color: #1756a9;
+}
+a:hover {
+ color: #111;
+ text-decoration: underline;
+}
+.social-media-list a:hover {
+ text-decoration: none;
+}
+.social-media-list a:hover .username {
+ text-decoration: underline;
+}
+
+/**
+ * Blockquotes
+ */
+blockquote {
+ color: #828282;
+ border-left: 4px solid #e8e8e8;
+ padding-left: 15px;
+ font-size: 18px;
+ letter-spacing: -1px;
+ font-style: italic;
+}
+blockquote > :last-child {
+ margin-bottom: 0;
+}
+
+/**
+ * Code formatting
+ */
+pre,
+code {
+ font-size: 15px;
+ border: 1px solid #e8e8e8;
+ border-radius: 3px;
+ background-color: #eef;
+}
+
+code {
+ padding: 1px 5px;
+}
+
+pre {
+ padding: 8px 12px;
+ overflow-x: auto;
+}
+pre > code {
+ border: 0;
+ padding-right: 0;
+ padding-left: 0;
+}
+
+/**
+ * Wrapper
+ */
+.wrapper {
+ max-width: -webkit-calc(800px - (30px * 2));
+ max-width: calc(800px - 30px * 2);
+ margin-right: auto;
+ margin-left: auto;
+ padding-right: 30px;
+ padding-left: 30px;
+}
+@media screen and (max-width: 800px) {
+ .wrapper {
+ max-width: -webkit-calc(800px - (30px));
+ max-width: calc(800px - (30px));
+ padding-right: 15px;
+ padding-left: 15px;
+ }
+}
+
+/**
+ * Clearfix
+ */
+.footer-col-wrapper:after, .wrapper:after {
+ content: "";
+ display: table;
+ clear: both;
+}
+
+/**
+ * Icons
+ */
+.svg-icon {
+ width: 16px;
+ height: 16px;
+ display: inline-block;
+ fill: #828282;
+ padding-right: 5px;
+ vertical-align: text-top;
+}
+
+.social-media-list li + li {
+ padding-top: 5px;
+}
+
+/**
+ * Tables
+ */
+table {
+ margin-bottom: 30px;
+ width: 100%;
+ text-align: left;
+ color: #3f3f3f;
+ border-collapse: collapse;
+ border: 1px solid #e8e8e8;
+}
+table tr:nth-child(even) {
+ background-color: #f7f7f7;
+}
+table th, table td {
+ padding: 10px 15px;
+}
+table th {
+ background-color: #f0f0f0;
+ border: 1px solid #dedede;
+ border-bottom-color: #c9c9c9;
+}
+table td {
+ border: 1px solid #e8e8e8;
+}
+
+/**
+ * Site header
+ */
+.site-header {
+ border-top: 5px solid #424242;
+ border-bottom: 1px solid #e8e8e8;
+ min-height: 55.95px;
+ position: relative;
+}
+
+.site-title {
+ font-size: 26px;
+ font-weight: 300;
+ line-height: 54px;
+ letter-spacing: -1px;
+ margin-bottom: 0;
+ float: left;
+}
+.site-title, .site-title:visited {
+ color: #424242;
+}
+
+.site-nav {
+ float: right;
+ line-height: 54px;
+}
+.site-nav .nav-trigger {
+ display: none;
+}
+.site-nav .menu-icon {
+ display: none;
+}
+.site-nav .page-link {
+ color: #111;
+ line-height: 1.5;
+}
+.site-nav .page-link:not(:last-child) {
+ margin-right: 20px;
+}
+@media screen and (max-width: 600px) {
+ .site-nav {
+ position: absolute;
+ top: 9px;
+ right: 15px;
+ background-color: #fdfdfd;
+ border: 1px solid #e8e8e8;
+ border-radius: 5px;
+ text-align: right;
+ }
+ .site-nav label[for=nav-trigger] {
+ display: block;
+ float: right;
+ width: 36px;
+ height: 36px;
+ z-index: 2;
+ cursor: pointer;
+ }
+ .site-nav .menu-icon {
+ display: block;
+ float: right;
+ width: 36px;
+ height: 26px;
+ line-height: 0;
+ padding-top: 10px;
+ text-align: center;
+ }
+ .site-nav .menu-icon > svg {
+ fill: #424242;
+ }
+ .site-nav input ~ .trigger {
+ clear: both;
+ display: none;
+ }
+ .site-nav input:checked ~ .trigger {
+ display: block;
+ padding-bottom: 5px;
+ }
+ .site-nav .page-link {
+ display: block;
+ padding: 5px 10px;
+ margin-left: 20px;
+ }
+ .site-nav .page-link:not(:last-child) {
+ margin-right: 0;
+ }
+}
+
+/**
+ * Site footer
+ */
+.site-footer {
+ border-top: 1px solid #e8e8e8;
+ padding: 30px 0;
+}
+
+.footer-heading {
+ font-size: 18px;
+ margin-bottom: 15px;
+}
+
+.contact-list,
+.social-media-list {
+ list-style: none;
+ margin-left: 0;
+}
+
+.footer-col-wrapper {
+ font-size: 15px;
+ color: #828282;
+ margin-left: -15px;
+}
+
+.footer-col {
+ float: left;
+ margin-bottom: 15px;
+ padding-left: 15px;
+}
+
+.footer-col-1 {
+ width: -webkit-calc(35% - (30px / 2));
+ width: calc(35% - 30px / 2);
+}
+
+.footer-col-2 {
+ width: -webkit-calc(20% - (30px / 2));
+ width: calc(20% - 30px / 2);
+}
+
+.footer-col-3 {
+ width: -webkit-calc(45% - (30px / 2));
+ width: calc(45% - 30px / 2);
+}
+
+@media screen and (max-width: 800px) {
+ .footer-col-1,
+ .footer-col-2 {
+ width: -webkit-calc(50% - (30px / 2));
+ width: calc(50% - 30px / 2);
+ }
+ .footer-col-3 {
+ width: -webkit-calc(100% - (30px / 2));
+ width: calc(100% - 30px / 2);
+ }
+}
+@media screen and (max-width: 600px) {
+ .footer-col {
+ float: none;
+ width: -webkit-calc(100% - (30px / 2));
+ width: calc(100% - 30px / 2);
+ }
+}
+/**
+ * Page content
+ */
+.page-content {
+ padding: 30px 0;
+ flex: 1;
+}
+
+.page-heading {
+ font-size: 32px;
+}
+
+.post-list-heading {
+ font-size: 28px;
+}
+
+.post-list {
+ margin-left: 0;
+ list-style: none;
+}
+.post-list > li {
+ margin-bottom: 30px;
+}
+
+.post-meta {
+ font-size: 14px;
+ color: #828282;
+}
+
+.post-link {
+ display: block;
+ font-size: 24px;
+}
+
+/**
+ * Posts
+ */
+.post-header {
+ margin-bottom: 30px;
+}
+
+.post-title {
+ font-size: 42px;
+ letter-spacing: -1px;
+ line-height: 1;
+}
+@media screen and (max-width: 800px) {
+ .post-title {
+ font-size: 36px;
+ }
+}
+
+.post-content {
+ margin-bottom: 30px;
+}
+.post-content h2 {
+ font-size: 32px;
+}
+@media screen and (max-width: 800px) {
+ .post-content h2 {
+ font-size: 28px;
+ }
+}
+.post-content h3 {
+ font-size: 26px;
+}
+@media screen and (max-width: 800px) {
+ .post-content h3 {
+ font-size: 22px;
+ }
+}
+.post-content h4 {
+ font-size: 20px;
+}
+@media screen and (max-width: 800px) {
+ .post-content h4 {
+ font-size: 18px;
+ }
+}
+
+/**
+ * Syntax highlighting styles
+ */
+.highlight {
+ background: #fff;
+}
+.highlighter-rouge .highlight {
+ background: #eef;
+}
+.highlight .c {
+ color: #998;
+ font-style: italic;
+}
+.highlight .err {
+ color: #a61717;
+ background-color: #e3d2d2;
+}
+.highlight .k {
+ font-weight: bold;
+}
+.highlight .o {
+ font-weight: bold;
+}
+.highlight .cm {
+ color: #998;
+ font-style: italic;
+}
+.highlight .cp {
+ color: #999;
+ font-weight: bold;
+}
+.highlight .c1 {
+ color: #998;
+ font-style: italic;
+}
+.highlight .cs {
+ color: #999;
+ font-weight: bold;
+ font-style: italic;
+}
+.highlight .gd {
+ color: #000;
+ background-color: #fdd;
+}
+.highlight .gd .x {
+ color: #000;
+ background-color: #faa;
+}
+.highlight .ge {
+ font-style: italic;
+}
+.highlight .gr {
+ color: #a00;
+}
+.highlight .gh {
+ color: #999;
+}
+.highlight .gi {
+ color: #000;
+ background-color: #dfd;
+}
+.highlight .gi .x {
+ color: #000;
+ background-color: #afa;
+}
+.highlight .go {
+ color: #888;
+}
+.highlight .gp {
+ color: #555;
+}
+.highlight .gs {
+ font-weight: bold;
+}
+.highlight .gu {
+ color: #aaa;
+}
+.highlight .gt {
+ color: #a00;
+}
+.highlight .kc {
+ font-weight: bold;
+}
+.highlight .kd {
+ font-weight: bold;
+}
+.highlight .kp {
+ font-weight: bold;
+}
+.highlight .kr {
+ font-weight: bold;
+}
+.highlight .kt {
+ color: #458;
+ font-weight: bold;
+}
+.highlight .m {
+ color: #099;
+}
+.highlight .s {
+ color: #d14;
+}
+.highlight .na {
+ color: #008080;
+}
+.highlight .nb {
+ color: #0086B3;
+}
+.highlight .nc {
+ color: #458;
+ font-weight: bold;
+}
+.highlight .no {
+ color: #008080;
+}
+.highlight .ni {
+ color: #800080;
+}
+.highlight .ne {
+ color: #900;
+ font-weight: bold;
+}
+.highlight .nf {
+ color: #900;
+ font-weight: bold;
+}
+.highlight .nn {
+ color: #555;
+}
+.highlight .nt {
+ color: #000080;
+}
+.highlight .nv {
+ color: #008080;
+}
+.highlight .ow {
+ font-weight: bold;
+}
+.highlight .w {
+ color: #bbb;
+}
+.highlight .mf {
+ color: #099;
+}
+.highlight .mh {
+ color: #099;
+}
+.highlight .mi {
+ color: #099;
+}
+.highlight .mo {
+ color: #099;
+}
+.highlight .sb {
+ color: #d14;
+}
+.highlight .sc {
+ color: #d14;
+}
+.highlight .sd {
+ color: #d14;
+}
+.highlight .s2 {
+ color: #d14;
+}
+.highlight .se {
+ color: #d14;
+}
+.highlight .sh {
+ color: #d14;
+}
+.highlight .si {
+ color: #d14;
+}
+.highlight .sx {
+ color: #d14;
+}
+.highlight .sr {
+ color: #009926;
+}
+.highlight .s1 {
+ color: #d14;
+}
+.highlight .ss {
+ color: #990073;
+}
+.highlight .bp {
+ color: #999;
+}
+.highlight .vc {
+ color: #008080;
+}
+.highlight .vg {
+ color: #008080;
+}
+.highlight .vi {
+ color: #008080;
+}
+.highlight .il {
+ color: #099;
+}
+
+/*# sourceMappingURL=main.css.map */
\ No newline at end of file
diff --git a/assets/main.css.map b/assets/main.css.map
new file mode 100644
index 0000000..4da063c
--- /dev/null
+++ b/assets/main.css.map
@@ -0,0 +1 @@
+{"version":3,"sourceRoot":"","sources":["../../../../.gem/ruby/3.1.3/gems/minima-2.5.1/_sass/minima/_base.scss","../../../../.gem/ruby/3.1.3/gems/minima-2.5.1/_sass/minima.scss","../../../../.gem/ruby/3.1.3/gems/minima-2.5.1/_sass/minima/_layout.scss","../../../../.gem/ruby/3.1.3/gems/minima-2.5.1/_sass/minima/_syntax-highlighting.scss"],"names":[],"mappings":"AAAA;AAAA;AAAA;AAGA;AAAA;AAAA;EAGE;EACA;;;AAKF;AAAA;AAAA;AAGA;EACE;EACA,OCLiB;EDMjB,kBCLiB;EDMjB;EACA;EACG;EACE;EACG;EACR;EACA;EA [...]
\ No newline at end of file
diff --git a/assets/minima-social-icons.svg b/assets/minima-social-icons.svg
new file mode 100644
index 0000000..fa7399f
--- /dev/null
+++ b/assets/minima-social-icons.svg
@@ -0,0 +1,33 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+
+<symbol id="dribbble" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M8 16c-4.41 0-8-3.59-8-8s3.59-8 8-8 8 3.59 8 8-3.59 8-8 8zm6.747-6.905c-.234-.074-2.115-.635-4.257-.292.894 2.456 1.258 4.456 1.328 4.872 1.533-1.037 2.624-2.68 2.93-4.58zM10.67 14.3c-.102-.6-.5-2.688-1.46-5.18l-.044.014C5.312 10.477 3.93 13.15 3.806 13.4c1.158.905 2.614 1.444 4.194 1.444.947 0 1.85-.194 2.67-.543zm-7.747-1.72c.155-.266 2.03-3.37 5.555-4.51.09-.03.18-. [...]
+
+<symbol id="facebook" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M15.117 0H.883C.395 0 0 .395 0 .883v14.234c0 .488.395.883.883.883h7.663V9.804H6.46V7.39h2.086V5.607c0-2.066 1.262-3.19 3.106-3.19.883 0 1.642.064 1.863.094v2.16h-1.28c-1 0-1.195.48-1.195 1.18v1.54h2.39l-.31 2.42h-2.08V16h4.077c.488 0 .883-.395.883-.883V.883C16 .395 15.605 0 15.117 0" fill-rule="nonzero"/></symbol>
+
+<symbol id="flickr" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M0 8c0 2.05 1.662 3.71 3.71 3.71 2.05 0 3.713-1.66 3.713-3.71S5.76 4.29 3.71 4.29C1.663 4.29 0 5.95 0 8zm8.577 0c0 2.05 1.662 3.71 3.712 3.71C14.33 11.71 16 10.05 16 8s-1.662-3.71-3.71-3.71c-2.05 0-3.713 1.66-3.713 3.71z"/></symbol>
+
+<symbol id="github" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M8 0C3.58 0 0 3.582 0 8c0 3.535 2.292 6.533 5.47 7.59.4.075.547-.172.547-.385 0-.19-.007-.693-.01-1.36-2.226.483-2.695-1.073-2.695-1.073-.364-.924-.89-1.17-.89-1.17-.725-.496.056-.486.056-.486.803.056 1.225.824 1.225.824.714 1.223 1.873.87 2.33.665.072-.517.278-.87.507-1.07-1.777-.2-3.644-.888-3.644-3.953 0-.873.31-1.587.823-2.147-.09-.202-.36-1.015.07-2.117 0 0 .67-.215 [...]
+
+<symbol id="googleplus" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M5.09 7.273v1.745h2.89c-.116.75-.873 2.197-2.887 2.197-1.737 0-3.155-1.44-3.155-3.215S3.353 4.785 5.09 4.785c.99 0 1.652.422 2.03.786l1.382-1.33c-.887-.83-2.037-1.33-3.41-1.33C2.275 2.91 0 5.19 0 8s2.276 5.09 5.09 5.09c2.94 0 4.888-2.065 4.888-4.974 0-.334-.036-.59-.08-.843H5.09zm10.91 0h-1.455V5.818H13.09v1.455h-1.454v1.454h1.455v1.455h1.46V8.727H16"/></symbol>
+
+<symbol id="instagram" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M8 0C5.827 0 5.555.01 4.702.048 3.85.088 3.27.222 2.76.42c-.526.204-.973.478-1.417.923-.445.444-.72.89-.923 1.417-.198.51-.333 1.09-.372 1.942C.008 5.555 0 5.827 0 8s.01 2.445.048 3.298c.04.852.174 1.433.372 1.942.204.526.478.973.923 1.417.444.445.89.72 1.417.923.51.198 1.09.333 1.942.372.853.04 1.125.048 3.298.048s2.445-.01 3.298-.048c.852-.04 1.433-.174 1.942-.372.5 [...]
+
+<symbol id="linkedin" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M13.632 13.635h-2.37V9.922c0-.886-.018-2.025-1.234-2.025-1.235 0-1.424.964-1.424 1.96v3.778h-2.37V6H8.51v1.04h.03c.318-.6 1.092-1.233 2.247-1.233 2.4 0 2.845 1.58 2.845 3.637v4.188zM3.558 4.955c-.762 0-1.376-.617-1.376-1.377 0-.758.614-1.375 1.376-1.375.76 0 1.376.617 1.376 1.375 0 .76-.617 1.377-1.376 1.377zm1.188 8.68H2.37V6h2.376v7.635zM14.816 0H1.18C.528 0 0 .516 0 [...]
+
+<symbol id="pinterest" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M8 0C3.582 0 0 3.582 0 8c0 3.39 2.108 6.285 5.084 7.45-.07-.633-.133-1.604.028-2.295.146-.625.938-3.977.938-3.977s-.24-.48-.24-1.188c0-1.11.646-1.943 1.448-1.943.683 0 1.012.513 1.012 1.127 0 .687-.436 1.713-.662 2.664-.19.797.4 1.445 1.185 1.445 1.42 0 2.514-1.498 2.514-3.662 0-1.91-1.376-3.25-3.342-3.25-2.276 0-3.61 1.71-3.61 3.47 0 .69.263 1.43.593 1.83.066.08.075. [...]
+
+<symbol id="rss" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M12.8 16C12.8 8.978 7.022 3.2 0 3.2V0c8.777 0 16 7.223 16 16h-3.2zM2.194 11.61c1.21 0 2.195.985 2.195 2.196 0 1.21-.99 2.194-2.2 2.194C.98 16 0 15.017 0 13.806c0-1.21.983-2.195 2.194-2.195zM10.606 16h-3.11c0-4.113-3.383-7.497-7.496-7.497v-3.11c5.818 0 10.606 4.79 10.606 10.607z"/></symbol>
+
+<symbol id="stackoverflow" class="svg-icon" xmlns="http://www.w3.org/2000/svg" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M12.658 14.577v-4.27h1.423V16H1.23v-5.693h1.42v4.27h10.006zm-8.583-1.423h7.16V11.73h-7.16v1.424zm.173-3.235l6.987 1.46.3-1.38L4.55 8.54l-.302 1.38zm.906-3.37l6.47 3.02.602-1.3-6.47-3.02-.602 1.29zm1.81-3.19l5.478 4.57.906-1.08L7.87 2.28l-.9 1.078zM10.502 0L9.338.863l4.27 5.735 1.164-.862L10.5 0z"/></symbol>
+
+<symbol id="twitter" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M16 3.038c-.59.26-1.22.437-1.885.517.677-.407 1.198-1.05 1.443-1.816-.634.37-1.337.64-2.085.79-.598-.64-1.45-1.04-2.396-1.04-1.812 0-3.282 1.47-3.282 3.28 0 .26.03.51.085.75-2.728-.13-5.147-1.44-6.766-3.42C.83 2.58.67 3.14.67 3.75c0 1.14.58 2.143 1.46 2.732-.538-.017-1.045-.165-1.487-.41v.04c0 1.59 1.13 2.918 2.633 3.22-.276.074-.566.114-.865.114-.21 0-.41-.02-.61-.058. [...]
+
+<symbol id="youtube" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414"><path d="M0 7.345c0-1.294.16-2.59.16-2.59s.156-1.1.636-1.587c.608-.637 1.408-.617 1.764-.684C3.84 2.36 8 2.324 8 2.324s3.362.004 5.6.166c.314.038.996.04 1.604.678.48.486.636 1.588.636 1.588S16 6.05 16 7.346v1.258c0 1.296-.16 2.59-.16 2.59s-.156 1.102-.636 1.588c-.608.638-1.29.64-1.604.678-2.238.162-5.6.166-5.6.166s-4.16-.037-5.44-.16c-.356-.067-1.156-.047-1.764-.684-.48-.487-.63 [...]
+
+<symbol id="mastodon" fill-rule="evenodd" clip-rule="evenodd" stroke-linejoin="round" stroke-miterlimit="1.414">
+ <path transform="scale(0.07)" d="M211.80734 139.0875c-3.18125 16.36625-28.4925 34.2775-57.5625 37.74875-15.15875 1.80875-30.08375 3.47125-45.99875 2.74125-26.0275-1.1925-46.565-6.2125-46.565-6.2125 0 2.53375.15625 4.94625.46875 7.2025 3.38375 25.68625 25.47 27.225 46.39125 27.9425 21.11625.7225 39.91875-5.20625 39.91875-5.20625l.8675 19.09s-14.77 7.93125-41.08125 9.39c-14.50875.7975-32.52375-.365-53.50625-5.91875C9.23234 213.82 1.40609 165.31125.20859 116.09125c-.365-14.61375-.14-28.39 [...]
+ <path transform="scale(0.07)" fill="#FFF" d="M177.50984 80.077v60.94125h-24.14375v-59.15c0-12.46875-5.24625-18.7975-15.74-18.7975-11.6025 0-17.4175 7.5075-17.4175 22.3525v32.37625H96.20734V85.42325c0-14.845-5.81625-22.3525-17.41875-22.3525-10.49375 0-15.74 6.32875-15.74 18.7975v59.15H38.90484V80.077c0-12.455 3.17125-22.3525 9.54125-29.675 6.56875-7.3225 15.17125-11.07625 25.85-11.07625 12.355 0 21.71125 4.74875 27.8975 14.2475l6.01375 10.08125 6.015-10.08125c6.185-9.49875 15.54125-14.2 [...]
+</symbol>
+
+
+</svg>
diff --git a/blog/2019/02/03/datafusion-donation/index.html b/blog/2019/02/03/datafusion-donation/index.html
new file mode 100644
index 0000000..90aca93
--- /dev/null
+++ b/blog/2019/02/03/datafusion-donation/index.html
@@ -0,0 +1,178 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>DataFusion: A Rust-native Query Engine for Apache Arrow | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="DataFusion: A Rust-native Query Engine for Apache Arrow" />
+<meta name="author" content="agrove" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="We are excited to announce that DataFusion has been donated to the Apache Arrow project. DataFusion is an in-memory query engine for the Rust implementation of Apache Arrow." />
+<meta property="og:description" content="We are excited to announce that DataFusion has been donated to the Apache Arrow project. DataFusion is an in-memory query engine for the Rust implementation of Apache Arrow." />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2019/02/03/datafusion-donation/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2019/02/03/datafusion-donation/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2019-02-03T23:00:00-07:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="DataFusion: A Rust-native Query Engine for Apache Arrow" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"agrove"},"dateModified":"2019-02-03T23:00:00-07:00","datePublished":"2019-02-03T23:00:00-07:00","description":"We are excited to announce that DataFusion has been donated to the Apache Arrow project. DataFusion is an in-memory query engine for the Rust implementation of Apache Arrow.","headline":"DataFusion: A Rust-native Query Engine for Apache Arrow","mainEntityOfPage":{"@type":"WebPage","@id":"ht [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">DataFusion: A Rust-native Query Engine for Apache Arrow</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2019-02-03T23:00:00-07:00" itemprop="datePublished">Feb 3, 2019
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">agrove</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<p>We are excited to announce that <a href="https://github.com/apache/arrow-datafusion">DataFusion</a> has been donated to the Apache Arrow project. DataFusion is an in-memory query engine for the Rust implementation of Apache Arrow.</p>
+
+<p>Although DataFusion was started two years ago, it was recently re-implemented to be Arrow-native and currently has limited capabilities but does support SQL queries against iterators of RecordBatch and has support for CSV files. There are plans to <a href="https://issues.apache.org/jira/browse/ARROW-4466">add support for Parquet files</a>.</p>
+
+<p>SQL support is limited to projection (<code class="language-plaintext highlighter-rouge">SELECT</code>), selection (<code class="language-plaintext highlighter-rouge">WHERE</code>), and simple aggregates (<code class="language-plaintext highlighter-rouge">MIN</code>, <code class="language-plaintext highlighter-rouge">MAX</code>, <code class="language-plaintext highlighter-rouge">SUM</code>) with an optional <code class="language-plaintext highlighter-rouge">GROUP BY</code> clause.</p>
+
+<p>Supported expressions are identifiers, literals, simple math operations (<code class="language-plaintext highlighter-rouge">+</code>, <code class="language-plaintext highlighter-rouge">-</code>, <code class="language-plaintext highlighter-rouge">*</code>, <code class="language-plaintext highlighter-rouge">/</code>), binary expressions (<code class="language-plaintext highlighter-rouge">AND</code>, <code class="language-plaintext highlighter-rouge">OR</code>), equality and comparison o [...]
+
+<h2 id="example">Example</h2>
+
+<p>The following example demonstrates running a simple aggregate SQL query against a CSV file.</p>
+
+<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c1">// create execution context</span>
+<span class="k">let</span> <span class="k">mut</span> <span class="n">ctx</span> <span class="o">=</span> <span class="nn">ExecutionContext</span><span class="p">::</span><span class="nf">new</span><span class="p">();</span>
+
+<span class="c1">// define schema for data source (csv file)</span>
+<span class="k">let</span> <span class="n">schema</span> <span class="o">=</span> <span class="nn">Arc</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="nn">Schema</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="nd">vec!</span><span class="p">[</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c1"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Utf8</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c2"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">UInt32</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c3"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Int8</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c4"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Int16</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c5"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Int32</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c6"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Int64</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c7"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">UInt8</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c8"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">UInt16</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c9"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">UInt32</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c10"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">UInt64</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c11"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Float32</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c12"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Float64</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+ <span class="nn">Field</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"c13"</span><span class="p">,</span> <span class="nn">DataType</span><span class="p">::</span><span class="n">Utf8</span><span class="p">,</span> <span class="kc">false</span><span class="p">),</span>
+<span class="p">]));</span>
+
+<span class="c1">// register csv file with the execution context</span>
+<span class="k">let</span> <span class="n">csv_datasource</span> <span class="o">=</span>
+ <span class="nn">CsvDataSource</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="s">"test/data/aggregate_test_100.csv"</span><span class="p">,</span> <span class="n">schema</span><span class="nf">.clone</span><span class="p">(),</span> <span class="mi">1024</span><span class="p">);</span>
+<span class="n">ctx</span><span class="nf">.register_datasource</span><span class="p">(</span><span class="s">"aggregate_test_100"</span><span class="p">,</span> <span class="nn">Rc</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="nn">RefCell</span><span class="p">::</span><span class="nf">new</span><span class="p">(</span><span class="n">csv_datasource</span><span class="p">)));</span>
+
+<span class="k">let</span> <span class="n">sql</span> <span class="o">=</span> <span class="s">"SELECT c1, MIN(c12), MAX(c12) FROM aggregate_test_100 WHERE c11 > 0.1 AND c11 < 0.9 GROUP BY c1"</span><span class="p">;</span>
+
+<span class="c1">// execute the query</span>
+<span class="k">let</span> <span class="n">relation</span> <span class="o">=</span> <span class="n">ctx</span><span class="nf">.sql</span><span class="p">(</span><span class="o">&</span><span class="n">sql</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
+<span class="k">let</span> <span class="k">mut</span> <span class="n">results</span> <span class="o">=</span> <span class="n">relation</span><span class="nf">.borrow_mut</span><span class="p">();</span>
+
+<span class="c1">// iterate over the results</span>
+<span class="k">while</span> <span class="k">let</span> <span class="nf">Some</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span> <span class="o">=</span> <span class="n">results</span><span class="nf">.next</span><span class="p">()</span><span class="nf">.unwrap</span><span class="p">()</span> <span class="p">{</span>
+ <span class="nd">println!</span><span class="p">(</span>
+ <span class="s">"RecordBatch has {} rows and {} columns"</span><span class="p">,</span>
+ <span class="n">batch</span><span class="nf">.num_rows</span><span class="p">(),</span>
+ <span class="n">batch</span><span class="nf">.num_columns</span><span class="p">()</span>
+ <span class="p">);</span>
+
+ <span class="k">let</span> <span class="n">c1</span> <span class="o">=</span> <span class="n">batch</span>
+ <span class="nf">.column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+ <span class="nf">.as_any</span><span class="p">()</span>
+ <span class="py">.downcast_ref</span><span class="p">::</span><span class="o"><</span><span class="n">BinaryArray</span><span class="o">></span><span class="p">()</span>
+ <span class="nf">.unwrap</span><span class="p">();</span>
+
+ <span class="k">let</span> <span class="n">min</span> <span class="o">=</span> <span class="n">batch</span>
+ <span class="nf">.column</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+ <span class="nf">.as_any</span><span class="p">()</span>
+ <span class="py">.downcast_ref</span><span class="p">::</span><span class="o"><</span><span class="n">Float64Array</span><span class="o">></span><span class="p">()</span>
+ <span class="nf">.unwrap</span><span class="p">();</span>
+
+ <span class="k">let</span> <span class="n">max</span> <span class="o">=</span> <span class="n">batch</span>
+ <span class="nf">.column</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
+ <span class="nf">.as_any</span><span class="p">()</span>
+ <span class="py">.downcast_ref</span><span class="p">::</span><span class="o"><</span><span class="n">Float64Array</span><span class="o">></span><span class="p">()</span>
+ <span class="nf">.unwrap</span><span class="p">();</span>
+
+ <span class="k">for</span> <span class="n">i</span> <span class="k">in</span> <span class="mi">0</span><span class="o">..</span><span class="n">batch</span><span class="nf">.num_rows</span><span class="p">()</span> <span class="p">{</span>
+ <span class="k">let</span> <span class="n">c1_value</span><span class="p">:</span> <span class="nb">String</span> <span class="o">=</span> <span class="nn">String</span><span class="p">::</span><span class="nf">from_utf8</span><span class="p">(</span><span class="n">c1</span><span class="nf">.value</span><span class="p">(</span><span class="n">i</span><span class="p">)</span><span class="nf">.to_vec</span><span class="p">())</span><span class="nf">.unwrap</span><span class="p">() [...]
+ <span class="nd">println!</span><span class="p">(</span><span class="s">"{}, Min: {}, Max: {}"</span><span class="p">,</span> <span class="n">c1_value</span><span class="p">,</span> <span class="n">min</span><span class="nf">.value</span><span class="p">(</span><span class="n">i</span><span class="p">),</span> <span class="n">max</span><span class="nf">.value</span><span class="p">(</span><span class="n">i</span><span class="p">),);</span>
+ <span class="p">}</span>
+<span class="p">}</span>
+</code></pre></div></div>
+
+<h2 id="roadmap">Roadmap</h2>
+
+<p>The roadmap for DataFusion will depend on interest from the Rust community, but here are some of the short term items that are planned:</p>
+
+<ul>
+ <li>Extending test coverage of the existing functionality</li>
+ <li>Adding support for Parquet data sources</li>
+ <li>Implementing more SQL features such as <code class="language-plaintext highlighter-rouge">JOIN</code>, <code class="language-plaintext highlighter-rouge">ORDER BY</code> and <code class="language-plaintext highlighter-rouge">LIMIT</code></li>
+ <li>Implement a DataFrame API as an alternative to SQL</li>
+ <li>Adding support for partitioning and parallel query execution using Rust’s async and await functionality</li>
+ <li>Creating a Docker image to make it easy to use DataFusion as a standalone query tool for interactive and batch queries</li>
+</ul>
+
+<h2 id="contributors-welcome">Contributors Welcome!</h2>
+
+<p>If you are excited about being able to use Rust for data science and would like to contribute to this work then there are many ways to get involved. The simplest way to get started is to try out DataFusion against your own data sources and file bug reports for any issues that you find. You could also check out the current <a href="https://cwiki.apache.org/confluence/display/ARROW/Rust+JIRA+Dashboard">list of issues</a> and have a go at fixing one. You can also join the <a href="http:/ [...]
+
+ </div><a class="u-url" href="/blog/blog/2019/02/03/datafusion-donation/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2021/04/12/ballista-donation/index.html b/blog/2021/04/12/ballista-donation/index.html
new file mode 100644
index 0000000..9f74cb9
--- /dev/null
+++ b/blog/2021/04/12/ballista-donation/index.html
@@ -0,0 +1,129 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Ballista: A Distributed Scheduler for Apache Arrow | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Ballista: A Distributed Scheduler for Apache Arrow" />
+<meta name="author" content="agrove" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="We are excited to announce that Ballista has been donated to the Apache Arrow project. Ballista is a distributed scheduler for the Rust implementation of Apache Arrow." />
+<meta property="og:description" content="We are excited to announce that Ballista has been donated to the Apache Arrow project. Ballista is a distributed scheduler for the Rust implementation of Apache Arrow." />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2021/04/12/ballista-donation/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2021/04/12/ballista-donation/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2021-04-12T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Ballista: A Distributed Scheduler for Apache Arrow" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"agrove"},"dateModified":"2021-04-12T00:00:00-06:00","datePublished":"2021-04-12T00:00:00-06:00","description":"We are excited to announce that Ballista has been donated to the Apache Arrow project. Ballista is a distributed scheduler for the Rust implementation of Apache Arrow.","headline":"Ballista: A Distributed Scheduler for Apache Arrow","mainEntityOfPage":{"@type":"WebPage","@id":"https://dataf [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Ballista: A Distributed Scheduler for Apache Arrow</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2021-04-12T00:00:00-06:00" itemprop="datePublished">Apr 12, 2021
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">agrove</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<p>We are excited to announce that <a href="https://github.com/apache/arrow-datafusion/tree/master/ballista">Ballista</a> has been donated
+to the Apache Arrow project.</p>
+
+<p>Ballista is a distributed compute platform primarily implemented in Rust, and powered by Apache Arrow. It is built
+on an architecture that allows other programming languages (such as Python, C++, and Java) to be supported as
+first-class citizens without paying a penalty for serialization costs.</p>
+
+<p>The foundational technologies in Ballista are:</p>
+
+<ul>
+ <li><a href="https://arrow.apache.org/">Apache Arrow</a> memory model and compute kernels for efficient processing of data.</li>
+ <li><a href="https://github.com/apache/arrow-datafusion">Apache Arrow DataFusion</a> query planning and
+execution framework, extended by Ballista to provide distributed planning and execution.</li>
+ <li><a href="https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/">Apache Arrow Flight Protocol</a> for efficient
+data transfer between processes.</li>
+ <li><a href="https://developers.google.com/protocol-buffers">Google Protocol Buffers</a> for serializing query plans.</li>
+ <li><a href="https://www.docker.com/">Docker</a> for packaging up executors along with user-defined code.</li>
+</ul>
+
+<p>Ballista can be deployed as a standalone cluster and also supports <a href="https://kubernetes.io/">Kubernetes</a>. In either
+case, the scheduler can be configured to use <a href="https://etcd.io/">etcd</a> as a backing store to (eventually) provide
+redundancy in the case of a scheduler failing.</p>
+
+<h2 id="status">Status</h2>
+
+<p>The Ballista project is at an early stage of development. However, it is capable of running complex analytics queries
+in a distributed cluster with reasonable performance (comparable to more established distributed query frameworks).</p>
+
+<p>One of the benefits of Ballista being part of the Arrow codebase is that there is now an opportunity to push parts of
+the scheduler down to DataFusion so that is possible to seamlessly scale across cores in DataFusion, and across nodes
+in Ballista, using the same unified query scheduler.</p>
+
+<h2 id="contributors-welcome">Contributors Welcome!</h2>
+
+<p>If you are excited about being able to use Rust for distributed compute and ETL and would like to contribute to this
+work then there are many ways to get involved. The simplest way to get started is to try out Ballista against your own
+datasets and file bug reports for any issues that you find. You could also check out the current
+<a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20component%20%3D%20%22Rust%20-%20Ballista%22">list of issues</a> and have a go at fixing one.</p>
+
+<p>The <a href="https://github.com/apache/arrow/blob/master/rust/README.md#arrow-rust-community">Arrow Rust Community</a>
+section of the Rust README provides information on other ways to interact with the Ballista contributors and
+maintainers.</p>
+
+
+ </div><a class="u-url" href="/blog/blog/2021/04/12/ballista-donation/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2021/08/18/ballista-0.5.0/index.html b/blog/2021/08/18/ballista-0.5.0/index.html
new file mode 100644
index 0000000..8c2aa4e
--- /dev/null
+++ b/blog/2021/08/18/ballista-0.5.0/index.html
@@ -0,0 +1,144 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow Ballista 0.5.0 Release | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow Ballista 0.5.0 Release" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2021/08/18/ballista-0.5.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2021/08/18/ballista-0.5.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2021-08-18T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow Ballista 0.5.0 Release" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2021-08-18T00:00:00-06:00","datePublished":"2021-08-18T00:00:00-06:00","description":"<!–","headline":"Apache Arrow Ballista 0.5.0 Release","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2021/08/18/ballista-0.5.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwhite_ [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow Ballista 0.5.0 Release</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2021-08-18T00:00:00-06:00" itemprop="datePublished">Aug 18, 2021
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<p>Ballista extends DataFusion to provide support for distributed queries. This is the first release of Ballista since
+the project was <a href="https://arrow.apache.org/blog/2021/04/12/ballista-donation/">donated</a> to the Apache Arrow project
+and includes 80 commits from 11 contributors.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>git shortlog -sn 4.0.0..5.0.0 ballista/rust/client ballista/rust/core ballista/rust/executor ballista/rust/scheduler
+ 27 Andy Grove
+ 15 Jiayu Liu
+ 12 Andrew Lamb
+ 8 Ximo Guanter
+ 6 Daniël Heres
+ 5 QP Hou
+ 2 Jorge Leitao
+ 1 Javier Goday
+ 1 K.I. (Dennis) Jung
+ 1 Mike Seddon
+ 1 sathis
+</code></pre></div></div>
+
+<!--
+$ git log --pretty=oneline 4.0.0..5.0.0 ballista/rust/client ballista/rust/core ballista/rust/executor ballista/rust/scheduler ballista-examples/ | wc -l
+80
+-->
+
+<p>The release notes below are not exhaustive and only expose selected highlights of the release. Many other bug fixes
+and improvements have been made: we refer you to the <a href="https://github.com/apache/arrow-datafusion/blob/5.0.0/ballista/CHANGELOG.md">complete changelog</a>.</p>
+
+<h1 id="performance-and-scalability">Performance and Scalability</h1>
+
+<p>Ballista is now capable of running complex SQL queries at scale and supports scalable distributed joins. We have been
+benchmarking using individual queries from the TPC-H benchmark at scale factors up to 1000 (1 TB). When running against
+CSV files, performance is generally very close to DataFusion, and significantly faster in some cases due to the fact
+that the scheduler limits the number of concurrent tasks that run at any given time. Performance against large Parquet
+datasets is currently non ideal due to some issues (<a href="https://github.com/apache/arrow-datafusion/issues/867">#867</a>,
+<a href="https://github.com/apache/arrow-datafusion/issues/868">#868</a>) that we hope to resolve for the next release.</p>
+
+<h1 id="new-features">New Features</h1>
+
+<p>The main new features in this release are:</p>
+
+<ul>
+ <li>Ballista queries can now be executed by calling DataFrame.collect()</li>
+ <li>The shuffle mechanism has been re-implemented</li>
+ <li>Distributed hash-partitioned joins are now supported</li>
+ <li>Keda autoscaling is supported</li>
+</ul>
+
+<p>To get started with Ballista, refer to the <a href="https://docs.rs/ballista/0.5.0/ballista/">crate documentation</a>.</p>
+
+<p>Now that the basic functionality is in place, the focus for the next release will be to improve the performance and
+scalability as well as improving the documentation.</p>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to Ballista, we would love to have you! You
+can help by trying out Ballista on some of your own data and projects and filing bug reports and helping to
+improve the documentation, or contribute to the documentation, tests or code. A list of open issues suitable for
+beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>
+and the full list is <a href="https://github.com/apache/arrow-datafusion/issues">here</a>.</p>
+
+ </div><a class="u-url" href="/blog/blog/2021/08/18/ballista-0.5.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2021/08/18/datafusion-5.0.0/index.html b/blog/2021/08/18/datafusion-5.0.0/index.html
new file mode 100644
index 0000000..f58d3b2
--- /dev/null
+++ b/blog/2021/08/18/datafusion-5.0.0/index.html
@@ -0,0 +1,175 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 5.0.0 Release | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 5.0.0 Release" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2021/08/18/datafusion-5.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2021/08/18/datafusion-5.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2021-08-18T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 5.0.0 Release" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2021-08-18T00:00:00-06:00","datePublished":"2021-08-18T00:00:00-06:00","description":"<!–","headline":"Apache Arrow DataFusion 5.0.0 Release","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2021/08/18/datafusion-5.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwh [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 5.0.0 Release</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2021-08-18T00:00:00-06:00" itemprop="datePublished">Aug 18, 2021
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<p>The Apache Arrow team is pleased to announce the DataFusion 5.0.0 release. This covers 4 months of development work
+and includes 211 commits from the following 31 distinct contributors.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>$ git shortlog -sn 4.0.0..5.0.0 datafusion datafusion-cli datafusion-examples
+ 61 Jiayu Liu
+ 47 Andrew Lamb
+ 27 Daniël Heres
+ 13 QP Hou
+ 13 Andy Grove
+ 4 Javier Goday
+ 4 sathis
+ 3 Ruan Pearce-Authers
+ 3 Raphael Taylor-Davies
+ 3 Jorge Leitao
+ 3 Cui Wenzheng
+ 3 Mike Seddon
+ 3 Edd Robinson
+ 2 思维
+ 2 Liang-Chi Hsieh
+ 2 Michael Lu
+ 2 Parth Sarthy
+ 2 Patrick More
+ 2 Rich
+ 1 Charlie Evans
+ 1 Gang Liao
+ 1 Agata Naomichi
+ 1 Ritchie Vink
+ 1 Evan Chan
+ 1 Ruihang Xia
+ 1 Todd Treece
+ 1 Yichen Wang
+ 1 baishen
+ 1 Nga Tran
+ 1 rdettai
+ 1 Marco Neumann
+</code></pre></div></div>
+
+<!--
+$ git log --pretty=oneline 4.0.0..5.0.0 datafusion datafusion-cli datafusion-examples | wc -l
+ 211
+-->
+
+<p>The release notes below are not exhaustive and only expose selected highlights of the release. Many other bug fixes
+and improvements have been made: we refer you to the complete
+<a href="https://github.com/apache/arrow-datafusion/blob/5.0.0/datafusion/CHANGELOG.md">changelog</a>.</p>
+
+<h1 id="performance">Performance</h1>
+
+<p>There have been numerous performance improvements in this release. The following chart shows the relative
+performance of individual TPC-H queries compared to the previous release.</p>
+
+<p><i>TPC-H @ scale factor 100, in parquet format. Concurrency 24.</i></p>
+
+<p><img src="/blog/img/2021-08-18-datafusion500perf.png" /></p>
+
+<p>We also extended support for more TPC-H queries: q7, q8, q9 and q13 are running successfully in DataFusion 5.0.</p>
+
+<h1 id="new-features">New Features</h1>
+
+<ul>
+ <li>Initial support for SQL-99 Analytics (WINDOW functions)</li>
+ <li>Improved JOIN support: cross join, semi-join, anti join, and fixes to null handling</li>
+ <li>Improved EXPLAIN support</li>
+ <li>Initial implementation of metrics in the physical plan</li>
+ <li>Support for SELECT DISTINCT</li>
+ <li>Support for Json and NDJson formatted inputs</li>
+ <li>Query column with relations</li>
+ <li>Added more datetime related functions: <code class="language-plaintext highlighter-rouge">now</code>, <code class="language-plaintext highlighter-rouge">date_trunc</code>, <code class="language-plaintext highlighter-rouge">to_timestamp_millis</code>, <code class="language-plaintext highlighter-rouge">to_timestamp_micros</code>, <code class="language-plaintext highlighter-rouge">to_timestamp_seconds</code></li>
+ <li>Streaming Dataframe.collect</li>
+ <li>Support table column aliases</li>
+ <li>Answer count(*), min() and max() queries using only statistics</li>
+ <li>Non-equi-join filters in JOIN conditions</li>
+ <li>Modulus operation</li>
+ <li>Support group by column positions</li>
+ <li>Added constant folding query optimizer</li>
+ <li>Hash partitioned aggregation</li>
+ <li>Added <code class="language-plaintext highlighter-rouge">random</code> SQL function</li>
+ <li>Implemented count distinct for floats and dictionary types</li>
+ <li>Re-exported arrow and parquet crates in Datafusion</li>
+ <li>General row group pruning logic that’s agnostic to storage format</li>
+</ul>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, we would love to have you! You
+can help by trying out DataFusion on some of your own data and projects and filing bug reports and helping to
+improve the documentation, or contribute to the documentation, tests or code. A list of open issues suitable for
+beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>
+and the full list is <a href="https://github.com/apache/arrow-datafusion/issues">here</a>.</p>
+
+ </div><a class="u-url" href="/blog/blog/2021/08/18/datafusion-5.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2021/11/19/datafusion-6.0.0/index.html b/blog/2021/11/19/datafusion-6.0.0/index.html
new file mode 100644
index 0000000..78005c8
--- /dev/null
+++ b/blog/2021/11/19/datafusion-6.0.0/index.html
@@ -0,0 +1,225 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 6.0.0 Release | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 6.0.0 Release" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2021/11/19/datafusion-6.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2021/11/19/datafusion-6.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2021-11-19T00:00:00-07:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 6.0.0 Release" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2021-11-19T00:00:00-07:00","datePublished":"2021-11-19T00:00:00-07:00","description":"<!–","headline":"Apache Arrow DataFusion 6.0.0 Release","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2021/11/19/datafusion-6.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwh [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 6.0.0 Release</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2021-11-19T00:00:00-07:00" itemprop="datePublished">Nov 19, 2021
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an embedded
+query engine which leverages the unique features of
+<a href="https://www.rust-lang.org/">Rust</a> and <a href="https://arrow.apache.org/">Apache
+Arrow</a> to provide a system that is high
+performance, easy to connect, easy to embed, and high quality.</p>
+
+<p>The Apache Arrow team is pleased to announce the DataFusion 6.0.0 release. This covers 4 months of development work
+and includes 134 commits from the following 28 distinct contributors.</p>
+
+<!--
+git log --pretty=oneline 5.0.0..6.0.0 datafusion datafusion-cli datafusion-examples | wc -l
+ 134
+
+git shortlog -sn 5.0.0..6.0.0 datafusion datafusion-cli datafusion-examples | wc -l
+ 29
+
+ Carlos and xudong963 are same individual
+-->
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 28 Andrew Lamb
+ 26 Jiayu Liu
+ 13 xudong963
+ 9 rdettai
+ 9 QP Hou
+ 6 Matthew Turner
+ 5 Daniël Heres
+ 4 Guillaume Balaine
+ 3 Francis Du
+ 3 Marco Neumann
+ 3 Jon Mease
+ 3 Nga Tran
+ 2 Yijie Shen
+ 2 Ruihang Xia
+ 2 Liang-Chi Hsieh
+ 2 baishen
+ 2 Andy Grove
+ 2 Jason Tianyi Wang
+ 1 Nan Zhu
+ 1 Antoine Wendlinger
+ 1 Krisztián Szűcs
+ 1 Mike Seddon
+ 1 Conner Murphy
+ 1 Patrick More
+ 1 Taehoon Moon
+ 1 Tiphaine Ruy
+ 1 adsharma
+ 1 lichuan6
+</code></pre></div></div>
+
+<p>The release notes below are not exhaustive and only expose selected highlights of the release. Many other bug fixes
+and improvements have been made: we refer you to the complete
+<a href="https://github.com/apache/arrow-datafusion/blob/6.0.0/datafusion/CHANGELOG.md">changelog</a>.</p>
+
+<h1 id="new-website">New Website</h1>
+
+<p>Befitting a growing project, DataFusion now has its
+<a href="https://arrow.apache.org/datafusion/">own website</a> hosted as part of the
+main <a href="https://arrow.apache.org">Apache Arrow Website</a></p>
+
+<h1 id="roadmap">Roadmap</h1>
+
+<p>The community worked to gather their thoughts about where we are
+taking DataFusion into a public
+<a href="https://arrow.apache.org/datafusion/specification/roadmap.html">Roadmap</a>
+for the first time</p>
+
+<h1 id="new-features">New Features</h1>
+
+<ul>
+ <li>Runtime operator metrics collection framework</li>
+ <li>Object store abstraction for unified access to local or remote storage</li>
+ <li>Hive style table partitioning support, for Parquet, CSV, Avro and Json files</li>
+ <li>DataFrame API support for: <code class="language-plaintext highlighter-rouge">except</code>, <code class="language-plaintext highlighter-rouge">intersect</code>, <code class="language-plaintext highlighter-rouge">show</code>, <code class="language-plaintext highlighter-rouge">limit</code> and window functions</li>
+ <li>SQL
+ <ul>
+ <li><code class="language-plaintext highlighter-rouge">EXPLAIN ANALYZE</code> with runtime metrics</li>
+ <li><code class="language-plaintext highlighter-rouge">trim ( [ LEADING | TRAILING | BOTH ] [ FROM ] string text [, characters text ] )</code> syntax</li>
+ <li>Postgres style regular expression matching operators <code class="language-plaintext highlighter-rouge">~</code>, <code class="language-plaintext highlighter-rouge">~*</code>, <code class="language-plaintext highlighter-rouge">!~</code>, and <code class="language-plaintext highlighter-rouge">!~*</code></li>
+ <li>SQL set operators <code class="language-plaintext highlighter-rouge">UNION</code>, <code class="language-plaintext highlighter-rouge">INTERSECT</code>, and <code class="language-plaintext highlighter-rouge">EXCEPT</code></li>
+ <li><code class="language-plaintext highlighter-rouge">cume_dist</code>, <code class="language-plaintext highlighter-rouge">percent_rank</code> window functions</li>
+ <li><code class="language-plaintext highlighter-rouge">digest</code>, <code class="language-plaintext highlighter-rouge">blake2s</code>, <code class="language-plaintext highlighter-rouge">blake2b</code>, <code class="language-plaintext highlighter-rouge">blake3</code> crypto functions</li>
+ <li>HyperLogLog based <code class="language-plaintext highlighter-rouge">approx_distinct</code></li>
+ <li><code class="language-plaintext highlighter-rouge">is distinct from</code> and <code class="language-plaintext highlighter-rouge">is not distinct from</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE TABLE AS SELECT</code></li>
+ <li>Accessing elements of nested <code class="language-plaintext highlighter-rouge">Struct</code> and <code class="language-plaintext highlighter-rouge">List</code> columns (e.g. <code class="language-plaintext highlighter-rouge">SELECT struct_column['field_name'], array_column[0] FROM ...</code>)</li>
+ <li>Boolean expressions in <code class="language-plaintext highlighter-rouge">CASE</code> statement</li>
+ <li><code class="language-plaintext highlighter-rouge">DROP TABLE</code></li>
+ <li><code class="language-plaintext highlighter-rouge">VALUES</code> List</li>
+ <li>Postgres regex match operators</li>
+ </ul>
+ </li>
+ <li>Support for Avro format</li>
+ <li>Support for <code class="language-plaintext highlighter-rouge">ScalarValue::Struct</code></li>
+ <li>Automatic schema inference for CSV files</li>
+ <li>Better interactive editing support in <code class="language-plaintext highlighter-rouge">datafusion-cli</code> as well as <code class="language-plaintext highlighter-rouge">psql</code> style commands such as <code class="language-plaintext highlighter-rouge">\d</code>, <code class="language-plaintext highlighter-rouge">\?</code>, and <code class="language-plaintext highlighter-rouge">\q</code></li>
+ <li>Generic constant evaluation and simplification framework</li>
+ <li>Added common subexpression eliminate query plan optimization rule</li>
+ <li>Python binding 0.4.0 with all Datafusion 6.0.0 features</li>
+</ul>
+
+<p>With these new features, we are also now passing TPC-H queries 8, 13 and 21.</p>
+
+<p>For the full list of new features with their relevant PRs, see the
+<a href="https://github.com/apache/arrow-datafusion/blob/6.0.0/datafusion/CHANGELOG.md">enhancements section</a>
+in the changelog.</p>
+
+<h1 id="async-planning-and-decoupling-file-format-from-table-layout"><code class="language-plaintext highlighter-rouge">async</code> planning and decoupling file format from table layout</h1>
+
+<p>Driven by the need to support Hive style table partitioning, @rdettai
+introduced the following design change to the Datafusion core.</p>
+
+<ul>
+ <li>The code for reading specific file formats (<code class="language-plaintext highlighter-rouge">Parquet</code>, <code class="language-plaintext highlighter-rouge">Avro</code>, <code class="language-plaintext highlighter-rouge">CSV</code>, and
+<code class="language-plaintext highlighter-rouge">JSON</code>) was separated from the logic that handles grouping sets of
+files into execution partitions.</li>
+ <li>The query planning process was made <code class="language-plaintext highlighter-rouge">async</code>.</li>
+</ul>
+
+<p>As a result, we are able to replace the old <code class="language-plaintext highlighter-rouge">Parquet</code>, <code class="language-plaintext highlighter-rouge">CSV</code> and <code class="language-plaintext highlighter-rouge">JSON</code> table
+providers with a single <code class="language-plaintext highlighter-rouge">ListingTable</code> table provider.</p>
+
+<p>This also sets up DataFusion and its plug-in ecosystem to
+supporting a wide range of catalogs and various object store implementations.
+You can read more about this change in the
+<a href="https://docs.google.com/document/d/1Bd4-PLLH-pHj0BquMDsJ6cVr_awnxTuvwNJuWsTHxAQ">design document</a>
+and on the <a href="https://github.com/apache/arrow-datafusion/pull/1010">arrow-datafusion#1010 PR</a>.</p>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, we would love to have you! You
+can help by trying out DataFusion on some of your own data and projects and filing bug reports and helping to
+improve the documentation, or contribute to the documentation, tests or code. A list of open issues suitable for
+beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>
+and the full list is <a href="https://github.com/apache/arrow-datafusion/issues">here</a>.</p>
+
+<p>Check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+ </div><a class="u-url" href="/blog/blog/2021/11/19/datafusion-6.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2022/02/28/datafusion-7.0.0/index.html b/blog/2022/02/28/datafusion-7.0.0/index.html
new file mode 100644
index 0000000..88b61e3
--- /dev/null
+++ b/blog/2022/02/28/datafusion-7.0.0/index.html
@@ -0,0 +1,261 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 7.0.0 Release | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 7.0.0 Release" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2022/02/28/datafusion-7.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2022/02/28/datafusion-7.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-02-28T00:00:00-07:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 7.0.0 Release" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2022-02-28T00:00:00-07:00","datePublished":"2022-02-28T00:00:00-07:00","description":"<!–","headline":"Apache Arrow DataFusion 7.0.0 Release","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2022/02/28/datafusion-7.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwh [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 7.0.0 Release</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2022-02-28T00:00:00-07:00" itemprop="datePublished">Feb 28, 2022
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible query execution framework, written in Rust, that uses Apache Arrow as its in-memory format.</p>
+
+<p>When you want to extend your Rust project with <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a>, a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth checking out.</p>
+
+<p>DataFusion’s SQL, <code class="language-plaintext highlighter-rouge">DataFrame</code>, and manual <code class="language-plaintext highlighter-rouge">PlanBuilder</code> API let users access a sophisticated query optimizer and execution engine capable of fast, resource efficient, and parallel execution that takes optimal advantage of todays multicore hardware. Being written in Rust means DataFusion can offer <em>both</em> the safety of dynamic languages as well as the resource efficien [...]
+
+<p>The Apache Arrow team is pleased to announce the DataFusion 7.0.0 release. This covers 4 months of development work
+and includes 195 commits from the following 37 distinct contributors.</p>
+
+<!--
+git log --pretty=oneline 5.0.0..6.0.0 datafusion datafusion-cli datafusion-examples | wc -l
+ 134
+
+git shortlog -sn 5.0.0..6.0.0 datafusion datafusion-cli datafusion-examples | wc -l
+ 29
+
+ Carlos and xudong963 are same individual
+-->
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 44 Andrew Lamb
+ 24 Kun Liu
+ 23 Jiayu Liu
+ 17 xudong.w
+ 11 Yijie Shen
+ 9 Matthew Turner
+ 7 Liang-Chi Hsieh
+ 5 Lin Ma
+ 4 Stephen Carman
+ 4 James Katz
+ 4 Dmitry Patsura
+ 4 QP Hou
+ 3 dependabot[bot]
+ 3 Remzi Yang
+ 3 Yang
+ 3 ic4y
+ 3 Daniël Heres
+ 2 Andy Grove
+ 2 Raphael Taylor-Davies
+ 2 Jason Tianyi Wang
+ 2 Dan Harris
+ 2 Sergey Melnychuk
+ 1 Nitish Tiwari
+ 1 Dom
+ 1 Eduard Karacharov
+ 1 Javier Goday
+ 1 Boaz
+ 1 Marko Mikulicic
+ 1 Max Burke
+ 1 Carol (Nichols || Goulding)
+ 1 Phillip Cloud
+ 1 Rich
+ 1 Toby Hede
+ 1 Will Jones
+ 1 r.4ntix
+ 1 rdettai
+</code></pre></div></div>
+
+<p>The following section highlights some of the improvements in this release. Of course, many other bug fixes and improvements have also been made and we refer you to the complete <a href="https://github.com/apache/arrow-datafusion/blob/7.0.0/datafusion/CHANGELOG.md">changelog</a> for the full detail.</p>
+
+<h1 id="summary">Summary</h1>
+
+<ul>
+ <li>DataFusion Crate
+ <ul>
+ <li>The DataFusion crate is being split into multiple crates to decrease compilation times and improve the development experience. Initially, <code class="language-plaintext highlighter-rouge">datafusion-common</code> (the core DataFusion components) and <code class="language-plaintext highlighter-rouge">datafusion-expr</code> (DataFusion expressions, functions, and operators) have been split out. There will be additional splits after the 7.0 release.</li>
+ </ul>
+ </li>
+ <li>Performance Improvements and Optimizations
+ <ul>
+ <li>Arrow’s dyn scalar kernels are now used to enable efficient operations on <code class="language-plaintext highlighter-rouge">DictionaryArray</code>s <a href="https://github.com/apache/arrow-datafusion/pull/1685">#1685</a></li>
+ <li>Switch from <code class="language-plaintext highlighter-rouge">std::sync::Mutex</code> to <code class="language-plaintext highlighter-rouge">parking_lot::Mutex</code> <a href="https://github.com/apache/arrow-datafusion/pull/1720">#1720</a></li>
+ </ul>
+ </li>
+ <li>New Features
+ <ul>
+ <li>Support for memory tracking and spilling to disk
+ <ul>
+ <li>MemoryMananger and DiskManager <a href="https://github.com/apache/arrow-datafusion/pull/1526">#1526</a></li>
+ <li>Out of core sort <a href="https://github.com/apache/arrow-datafusion/pull/1526">#1526</a></li>
+ <li>New metrics
+ <ul>
+ <li><code class="language-plaintext highlighter-rouge">Gauge</code> and <code class="language-plaintext highlighter-rouge">CurrentMemoryUsage</code> <a href="https://github.com/apache/arrow-datafusion/pull/1682">#1682</a></li>
+ <li><code class="language-plaintext highlighter-rouge">Spill_count</code> and <code class="language-plaintext highlighter-rouge">spilled_bytes</code> <a href="https://github.com/apache/arrow-datafusion/pull/1641">#1641</a></li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li>New math functions
+ <ul>
+ <li><code class="language-plaintext highlighter-rouge">Approx_quantile</code> <a href="https://github.com/apache/arrow-datafusion/pull/1539">#1529</a></li>
+ <li><code class="language-plaintext highlighter-rouge">stddev</code> and <code class="language-plaintext highlighter-rouge">variance</code> (sample and population) <a href="https://github.com/apache/arrow-datafusion/pull/1525">#1525</a></li>
+ <li><code class="language-plaintext highlighter-rouge">corr</code> <a href="https://github.com/apache/arrow-datafusion/pull/1561">#1561</a></li>
+ </ul>
+ </li>
+ <li>Support decimal type <a href="https://github.com/apache/arrow-datafusion/pull/1394">#1394</a><a href="https://github.com/apache/arrow-datafusion/pull/1407">#1407</a><a href="https://github.com/apache/arrow-datafusion/pull/1408">#1408</a><a href="https://github.com/apache/arrow-datafusion/pull/1431">#1431</a><a href="https://github.com/apache/arrow-datafusion/pull/1483">#1483</a><a href="https://github.com/apache/arrow-datafusion/pull/1554">#1554</a><a href="https://github.com/a [...]
+ <li>Support for reading Parquet files with evolved schemas <a href="https://github.com/apache/arrow-datafusion/pull/1622">#1622</a><a href="https://github.com/apache/arrow-datafusion/pull/1709">#1709</a></li>
+ <li>Support for registering <code class="language-plaintext highlighter-rouge">DataFrame</code> as table <a href="https://github.com/apache/arrow-datafusion/pull/1699">#1699</a></li>
+ <li>Support for the <code class="language-plaintext highlighter-rouge">substring</code> function <a href="https://github.com/apache/arrow-datafusion/pull/1621">#1621</a></li>
+ <li>Support <code class="language-plaintext highlighter-rouge">array_agg(distinct ...)</code> <a href="https://github.com/apache/arrow-datafusion/pull/1579">#1579</a></li>
+ <li>Support <code class="language-plaintext highlighter-rouge">sort</code> on unprojected columns <a href="https://github.com/apache/arrow-datafusion/pull/1415">#1415</a></li>
+ </ul>
+ </li>
+ <li>Additional Integration Points
+ <ul>
+ <li>A new public Expression simplification API <a href="https://github.com/apache/arrow-datafusion/pull/1717">#1717</a></li>
+ </ul>
+ </li>
+ <li><a href="https://github.com/datafusion-contrib">DataFusion-Contrib</a>
+ <ul>
+ <li>A new GitHub organization created as a home for both <code class="language-plaintext highlighter-rouge">DataFusion</code> extensions and as a testing ground for new features.
+ <ul>
+ <li>Extensions
+ <ul>
+ <li><a href="https://github.com/datafusion-contrib/datafusion-python">DataFusion-Python</a></li>
+ <li><a href="https://github.com/datafusion-contrib/datafusion-java">DataFusion-Java</a></li>
+ <li><a href="https://github.com/datafusion-contrib/datafusion-hdfs-native">DataFusion-hdsfs-native</a></li>
+ <li><a href="https://github.com/datafusion-contrib/datafusion-objectstore-s3">DataFusion-ObjectStore-s3</a></li>
+ </ul>
+ </li>
+ <li>New Features
+ <ul>
+ <li><a href="https://github.com/datafusion-contrib/datafusion-streams">DataFusion-Streams</a></li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li><a href="https://github.com/jorgecarleitao/arrow2">Arrow2</a>
+ <ul>
+ <li>An <a href="https://github.com/apache/arrow-datafusion/tree/arrow2">Arrow2 Branch</a> has been created. There are ongoing discussions in <a href="https://github.com/apache/arrow-datafusion/issues/1532">DataFusion</a> and <a href="https://github.com/apache/arrow-rs/issues/1176">arrow-rs</a> about migrating <code class="language-plaintext highlighter-rouge">DataFusion</code> to <code class="language-plaintext highlighter-rouge">Arrow2</code></li>
+ </ul>
+ </li>
+</ul>
+
+<h1 id="documentation-and-roadmap">Documentation and Roadmap</h1>
+
+<p>We are working to consolidate the documentation into the <a href="https://arrow.apache.org/datafusion">official site</a>. You can find more details there on topics such as the <a href="https://arrow.apache.org/datafusion/user-guide/sql/index.html">SQL status</a> and a <a href="https://arrow.apache.org/datafusion/user-guide/introduction.html#introduction">user guide</a>. This is also an area we would love to get help from the broader community <a href="https://github.com/apache/arrow [...]
+
+<p>To provide transparency on DataFusion’s priorities to users and developers a three month roadmap will be published at the beginning of each quarter. This can be found here <a href="https://arrow.apache.org/datafusion/specification/roadmap.html">here</a>.</p>
+
+<h1 id="upcoming-attractions">Upcoming Attractions</h1>
+
+<ul>
+ <li>Ballista is gaining momentum, and several groups are now evaluating and contributing to the project.
+ <ul>
+ <li>Some of the proposed improvements
+ <ul>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/1701">Improvements Overview</a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/1675">Extensibility</a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/1702">File system access</a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/1704">Cluster state</a></li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li>Continued improvements for working with limited resources and large datasets
+ <ul>
+ <li>Memory limited joins<a href="https://github.com/apache/arrow-datafusion/issues/1599">#1599</a></li>
+ <li>Sort-merge join<a href="https://github.com/apache/arrow-datafusion/issues/141">#141</a><a href="https://github.com/apache/arrow-datafusion/pull/1776">#1776</a></li>
+ <li>Introduce row based bytes representation <a href="https://github.com/apache/arrow-datafusion/pull/1708">#1708</a></li>
+ </ul>
+ </li>
+</ul>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, and learning about state of
+the art query processing, we would love to have you join us on the journey! You
+can help by trying out DataFusion on some of your own data and projects and let us know how it goes or contribute a PR with documentation, tests or code. A list of open issues suitable for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a></p>
+
+<p>Check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+ </div><a class="u-url" href="/blog/blog/2022/02/28/datafusion-7.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2022/03/21/datafusion-contrib/index.html b/blog/2022/03/21/datafusion-contrib/index.html
new file mode 100644
index 0000000..6d738d4
--- /dev/null
+++ b/blog/2022/03/21/datafusion-contrib/index.html
@@ -0,0 +1,276 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Introducing Apache Arrow DataFusion Contrib | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Introducing Apache Arrow DataFusion Contrib" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2022/03/21/datafusion-contrib/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2022/03/21/datafusion-contrib/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-03-21T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Introducing Apache Arrow DataFusion Contrib" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2022-03-21T00:00:00-06:00","datePublished":"2022-03-21T00:00:00-06:00","description":"<!–","headline":"Introducing Apache Arrow DataFusion Contrib","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2022/03/21/datafusion-contrib/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Introducing Apache Arrow DataFusion Contrib</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2022-03-21T00:00:00-06:00" itemprop="datePublished">Mar 21, 2022
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p>Apache Arrow <a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible query execution framework, written in Rust, that uses <a href="https://arrow.apache.org">Apache Arrow</a> as its in-memory format.</p>
+
+<p>When you want to extend your Rust project with <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a>, a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth checking out. DataFusion’s pluggable design makes creating extensions at various points particular easy to build.</p>
+
+<p>DataFusion’s SQL, <code class="language-plaintext highlighter-rouge">DataFrame</code>, and manual <code class="language-plaintext highlighter-rouge">PlanBuilder</code> API let users access a sophisticated query optimizer and execution engine capable of fast, resource efficient, and parallel execution that takes optimal advantage of todays multicore hardware. Being written in Rust means DataFusion can offer <em>both</em> the safety of dynamic languages as well as the resource efficien [...]
+
+<p>The DataFusion team is pleased to announce the creation of the <a href="https://github.com/datafusion-contrib">DataFusion-Contrib</a> GitHub organization to support and accelerate other projects. While the core DataFusion library remains under Apache governance, the contrib organization provides a more flexible testing ground for new DataFusion features and a home for DataFusion extensions. With this announcement, we are pleased to introduce the following inaugural DataFusion-Contri [...]
+
+<h2 id="datafusion-python">DataFusion-Python</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-python">project</a> provides Python bindings to the core Rust implementation of DataFusion, which allows users to:</p>
+
+<ul>
+ <li>Work with familiar SQL or DataFrame APIs to run queries in a safe, multi-threaded environment, returning results in Python</li>
+ <li>Create User Defined Functions and User Defined Aggregate Functions for complex operations</li>
+ <li>Pay no overhead to copy between Python and underlying Rust execution engine (by way of Apache Arrow arrays)</li>
+</ul>
+
+<h3 id="upcoming-enhancements">Upcoming enhancements</h3>
+
+<p>The team is focusing on exposing more features from the underlying Rust implementation of DataFusion and improving documentation.</p>
+
+<h3 id="how-to-install">How to install</h3>
+
+<p>From <code class="language-plaintext highlighter-rouge">pip</code></p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>pip <span class="nb">install </span>datafusion
+</code></pre></div></div>
+
+<p>Or</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>python <span class="nt">-m</span> pip <span class="nb">install </span>datafusion
+</code></pre></div></div>
+
+<h2 id="datafusion-objectstore-s3">DataFusion-ObjectStore-S3</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-objectstore-s3">crate</a> provides an <code class="language-plaintext highlighter-rouge">ObjectStore</code> implementation for querying data stored in S3 or S3 compatible storage. This makes it almost as easy to query data that lives on S3 as lives in local files</p>
+
+<ul>
+ <li>Ability to create <code class="language-plaintext highlighter-rouge">S3FileSystem</code> to register as part of DataFusion <code class="language-plaintext highlighter-rouge">ExecutionContext</code></li>
+ <li>Register files or directories stored on S3 with <code class="language-plaintext highlighter-rouge">ctx.register_listing_table</code></li>
+</ul>
+
+<h3 id="upcoming-enhancements-1">Upcoming enhancements</h3>
+
+<p>The current priority is adding python bindings for <code class="language-plaintext highlighter-rouge">S3FileSystem</code>. After that there will be async improvements as DataFusion adopts more of that functionality and we are looking into S3 Select functionality.</p>
+
+<h3 id="how-to-install-1">How to Install</h3>
+
+<p>Add the below to your <code class="language-plaintext highlighter-rouge">Cargo.toml</code> in your Rust Project with DataFusion.</p>
+
+<div class="language-toml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="py">datafusion-objectstore-s3</span> <span class="p">=</span> <span class="s">"0.1.0"</span>
+</code></pre></div></div>
+
+<h2 id="datafusion-substrait">DataFusion-Substrait</h2>
+
+<p><a href="https://substrait.io/">Substrait</a> is an emerging standard that provides a cross-language serialization format for relational algebra (e.g. expressions and query plans).</p>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-substrait">crate</a> provides a Substrait producer and consumer for DataFusion. A producer converts a DataFusion logical plan into a Substrait protobuf and a consumer does the reverse.</p>
+
+<p>Examples of how to use this crate can be found <a href="https://github.com/datafusion-contrib/datafusion-substrait/blob/main/src/lib.rs">here</a>.</p>
+
+<h3 id="potential-use-cases">Potential Use Cases</h3>
+
+<ul>
+ <li>Replace custom DataFusion protobuf serialization.</li>
+ <li>Make it easier to pass query plans over FFI boundaries, such as from Python to Rust</li>
+ <li>Allow Apache Calcite query plans to be executed in DataFusion</li>
+</ul>
+
+<h2 id="datafusion-bigtable">DataFusion-BigTable</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-bigtable">crate</a> implements <a href="https://cloud.google.com/bigtable">Bigtable</a> as a data source and physical executor for DataFusion queries. It currently supports both UTF-8 string and 64-bit big-endian signed integers in Bigtable. From a SQL perspective it supports both simple and composite row keys with <code class="language-plaintext highlighter-rouge">=</code>, <code class="language-plaintext highlighter-ro [...]
+
+<h3 id="upcoming-enhancements-2">Upcoming Enhancements</h3>
+
+<ul>
+ <li>Predicate pushdown
+ <ul>
+ <li>Value range</li>
+ <li>Value Regex</li>
+ <li>Timestamp range</li>
+ </ul>
+ </li>
+ <li>Multithreaded</li>
+ <li>Partition aware execution</li>
+ <li>Production ready</li>
+</ul>
+
+<h3 id="how-to-install-2">How to Install</h3>
+
+<p>Add the below to your <code class="language-plaintext highlighter-rouge">Cargo.toml</code> in your Rust Project with DataFusion.</p>
+
+<div class="language-toml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="py">datafusion-bigtable</span> <span class="p">=</span> <span class="s">"0.1.0"</span>
+</code></pre></div></div>
+
+<h2 id="datafusion-hdfs">DataFusion-HDFS</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-objectstore-hdfs">crate</a> introduces <code class="language-plaintext highlighter-rouge">HadoopFileSystem</code> as a remote <code class="language-plaintext highlighter-rouge">ObjectStore</code> which provides the ability to query HDFS files. For HDFS access the <a href="https://github.com/yahoNanJing/fs-hdfs">fs-hdfs</a> library is used.</p>
+
+<h2 id="datafusion-tokomak">DataFusion-Tokomak</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-tokomak">crate</a> provides an e-graph based DataFusion optimization framework based on the Rust <a href="https://egraphs-good.github.io">egg</a> library. An e-graph is a data structure that powers the equality saturation optimization technique.</p>
+
+<p>As context, the optimizer framework within DataFusion is currently <a href="https://github.com/apache/arrow-datafusion/issues/1972">under review</a> with the objective of implementing a more strategic long term solution that is more efficient and simpler to develop.</p>
+
+<p>Some of the benefits of using <code class="language-plaintext highlighter-rouge">egg</code> within DataFusion are:</p>
+
+<ul>
+ <li>Implements optimized algorithms that are hard to match with manually written optimization passes</li>
+ <li>Makes it easy and less verbose to add optimization rules</li>
+ <li>Plugin framework to add more complex optimizations</li>
+ <li>Egg does not depend on rule order and can lead to a higher level of optimization by being able to apply multiple rules at the same time until it converges</li>
+ <li>Allows for cost-based optimizations</li>
+</ul>
+
+<p>This is an exciting new area for DataFusion with lots of opportunity for community involvement!</p>
+
+<h2 id="datafusion-tui">DataFusion-Tui</h2>
+
+<p><a href="https://github.com/datafusion-contrib/datafusion-tui">DataFusion-tui</a> aka <code class="language-plaintext highlighter-rouge">dft</code> provides a feature rich terminal application for using DataFusion. It has drawn inspiration and several features from <code class="language-plaintext highlighter-rouge">datafusion-cli</code>. In contrast to <code class="language-plaintext highlighter-rouge">datafusion-cli</code> the objective of this tool is to provide a light SQL IDE ex [...]
+
+<ul>
+ <li>Tab Management to provide clean and structured organization of DataFusion queries, results, <code class="language-plaintext highlighter-rouge">ExecutionContext</code> information, and logs
+ <ul>
+ <li>SQL Editor
+ <ul>
+ <li>Text editor for writing SQL queries</li>
+ </ul>
+ </li>
+ <li>Query History
+ <ul>
+ <li>History of executed queries, their execution time, and the number of returned rows</li>
+ </ul>
+ </li>
+ <li><code class="language-plaintext highlighter-rouge">ExecutionContext</code> information
+ <ul>
+ <li>Expose information on which physical optimizers are used and which <code class="language-plaintext highlighter-rouge">ExecutionConfig</code> settings are set</li>
+ </ul>
+ </li>
+ <li>Logs
+ <ul>
+ <li>Logs from <code class="language-plaintext highlighter-rouge">dft</code>, DataFusion, and any dependent libraries</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li>Support for custom <code class="language-plaintext highlighter-rouge">ObjectStore</code>s
+ <ul>
+ <li>S3</li>
+ </ul>
+ </li>
+ <li>Preload DDL from <code class="language-plaintext highlighter-rouge">~/.datafusionrc</code> to enable having local “database” available at startup</li>
+</ul>
+
+<h3 id="upcoming-enhancements-3">Upcoming Enhancements</h3>
+
+<ul>
+ <li>SQL Editor
+ <ul>
+ <li>Command to write query results to file</li>
+ <li>Multiple SQL editor tabs</li>
+ </ul>
+ </li>
+ <li>Expose more information from <code class="language-plaintext highlighter-rouge">ExecutionContext</code></li>
+ <li>A help tab that provides information on functions</li>
+ <li>Query custom <code class="language-plaintext highlighter-rouge">TableProvider</code>s such as <a href="https://github.com/delta-io/delta-rs">DeltaTable</a> or <a href="https://github.com/datafusion-contrib/datafusion-bigtable">BigTable</a></li>
+</ul>
+
+<h2 id="datafusion-streams">DataFusion-Streams</h2>
+
+<p><a href="https://github.com/datafusion-contrib/datafusion-streams">DataFusion-Stream</a> is a new testing ground for creating a <code class="language-plaintext highlighter-rouge">StreamProvider</code> in DataFusion that will enable querying streaming data sources such as Apache Kafka. The implementation for this feature is currently being designed and is under active review. Once the design is finalized the trait and attendant data structures will be added back to the core DataFusio [...]
+
+<h2 id="datafusion-java">DataFusion-Java</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-java">project</a> created an initial set of Java bindings to DataFusion. The project is currently in maintenance mode and is looking for maintainers to drive future development.</p>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, and learning about state of
+the art query processing, we would love to have you join us on the journey! You
+can help by trying out DataFusion on some of your own data and projects and let us know how it goes or contribute a PR with documentation, tests or code. A list of open issues suitable for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a></p>
+
+<p>The best way to find out about creating new extensions within DataFusion-Contrib is reaching out on the <code class="language-plaintext highlighter-rouge">#arrow-rust</code> channel of the Apache Software Foundation <a href="https://join.slack.com/t/the-asf/shared_invite/zt-vlfbf7ch-HkbNHiU_uDlcH_RvaHv9gQ">Slack</a> workspace.</p>
+
+<p>You can also check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more ways to engage with the community.</p>
+
+<p>Links for each DataFusion-Contrib repository are provided above if you would like to contribute to those.</p>
+
+ </div><a class="u-url" href="/blog/blog/2022/03/21/datafusion-contrib/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2022/05/16/datafusion-8.0.0/index.html b/blog/2022/05/16/datafusion-8.0.0/index.html
new file mode 100644
index 0000000..bcfd7a7
--- /dev/null
+++ b/blog/2022/05/16/datafusion-8.0.0/index.html
@@ -0,0 +1,266 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 8.0.0 Release | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 8.0.0 Release" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2022/05/16/datafusion-8.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2022/05/16/datafusion-8.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-05-16T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 8.0.0 Release" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2022-05-16T00:00:00-06:00","datePublished":"2022-05-16T00:00:00-06:00","description":"<!–","headline":"Apache Arrow DataFusion 8.0.0 Release","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2022/05/16/datafusion-8.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwh [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 8.0.0 Release</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2022-05-16T00:00:00-06:00" itemprop="datePublished">May 16, 2022
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible query execution framework, written in Rust, that
+uses Apache Arrow as its in-memory format.</p>
+
+<p>When you want to extend your Rust project with <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a>,
+a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth
+checking out.</p>
+
+<p>DataFusion’s SQL, <code class="language-plaintext highlighter-rouge">DataFrame</code>, and manual <code class="language-plaintext highlighter-rouge">PlanBuilder</code> API let users access a sophisticated query optimizer and
+execution engine capable of fast, resource efficient, and parallel execution that takes optimal advantage of
+today’s multicore hardware. Being written in Rust means DataFusion can offer <em>both</em> the safety of a dynamic language and
+the resource efficiency of a compiled language.</p>
+
+<p>The Apache Arrow team is pleased to announce the DataFusion 8.0.0 release (and also the release of version 0.7.0 of
+the Ballista subproject). This covers 3 months of development work and includes 279 commits from the following 49
+distinct contributors.</p>
+
+<!--
+$ git log --pretty=oneline 7.0.0..8.0.0 datafusion datafusion-cli datafusion-examples ballista ballista-cli ballista-examples | wc -l
+279
+
+$ git shortlog -sn 7.0.0..8.0.0 datafusion datafusion-cli datafusion-examples ballista ballista-cli ballista-examples | wc -l
+49
+
+(feynman han, feynman.h, Feynman Han were assumed to be the same person)
+-->
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 39 Andy Grove
+ 33 Andrew Lamb
+ 21 DuRipeng
+ 20 Yijie Shen
+ 19 Yang Jiang
+ 17 Raphael Taylor-Davies
+ 11 Dan Harris
+ 11 Matthew Turner
+ 11 yahoNanJing
+ 9 dependabot[bot]
+ 8 jakevin
+ 6 Kun Liu
+ 5 Jiayu Liu
+ 4 Daniël Heres
+ 4 mingmwang
+ 4 xudong.w
+ 3 Carol (Nichols || Goulding)
+ 3 Dmitry Patsura
+ 3 Eduard Karacharov
+ 3 Jeremy Dyer
+ 3 Kaushik
+ 3 Rich
+ 3 comphead
+ 3 gaojun2048
+ 3 Feynman Han
+ 2 Jie Han
+ 2 Jon Mease
+ 2 Tim Van Wassenhove
+ 2 Yt
+ 2 Zhang Li
+ 2 silence-coding
+ 1 Alexander Spies
+ 1 George Andronchik
+ 1 Guillaume Balaine
+ 1 Hao Xin
+ 1 Jiacai Liu
+ 1 Jörn Horstmann
+ 1 Liang-Chi Hsieh
+ 1 Max Burke
+ 1 NaincyKumariKnoldus
+ 1 Nga Tran
+ 1 Patrick More
+ 1 Pierre Zemb
+ 1 Remzi Yang
+ 1 Sergey Melnychuk
+ 1 Stephen Carman
+ 1 doki
+</code></pre></div></div>
+
+<p>The following sections highlight some of the changes in this release. Of course, many other bug fixes and
+improvements have been made and we encourage you to check out the
+<a href="https://github.com/apache/arrow-datafusion/blob/8.0.0/datafusion/CHANGELOG.md">changelog</a> for full details.</p>
+
+<h1 id="summary">Summary</h1>
+
+<h2 id="ddl-support">DDL Support</h2>
+
+<p>DDL support has been expanded to include the following commands for creating databases, schemas, and views. This
+allows DataFusion to be used more effectively from the CLI.</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">CREATE DATABASE</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE VIEW</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE SCHEMA</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE EXTERNAL TABLE</code> now supports JSON files, <code class="language-plaintext highlighter-rouge">IF NOT EXISTS</code>, and partition columns</li>
+</ul>
+
+<h2 id="sql-support">SQL Support</h2>
+
+<p>The SQL query planner now supports a number of new SQL features, including:</p>
+
+<ul>
+ <li><em>Subqueries</em>: when used via <code class="language-plaintext highlighter-rouge">IN</code>, <code class="language-plaintext highlighter-rouge">EXISTS</code>, and as scalars</li>
+ <li><em>Grouping Sets</em>: <code class="language-plaintext highlighter-rouge">CUBE</code> and <code class="language-plaintext highlighter-rouge">ROLLUP</code> grouping sets.</li>
+ <li><em>Aggregate functions</em>: <code class="language-plaintext highlighter-rouge">approx_percentile</code>, <code class="language-plaintext highlighter-rouge">approx_percentile_cont</code>, <code class="language-plaintext highlighter-rouge">approx_percentile_cont_with_weight</code>, <code class="language-plaintext highlighter-rouge">approx_distinct</code>, <code class="language-plaintext highlighter-rouge">approx_median</code> and <code class="language-plaintext highlighter-rouge">a [...]
+ <li><em><code class="language-plaintext highlighter-rouge">null</code> literals</em></li>
+ <li><em>bitwise operations</em>: for example ‘<code class="language-plaintext highlighter-rouge">|</code>’</li>
+</ul>
+
+<p>There are also many bug fixes and improvements around normalizing identifiers consistently.</p>
+
+<p>We continue our tradition of incrementally releasing support for new
+features as they are developed. Thus, while the physical plan may not yet
+support all new features, it gets more complete each release. These
+changes also make DataFusion an increasingly compelling choice for
+projects looking for a SQL parser and query planner that can produce
+optimized logical plans that can be translated to
+their own execution engine.</p>
+
+<h2 id="query-execution--internals">Query Execution & Internals</h2>
+
+<p>There are several notable improvements and new features in the query execution engine:</p>
+
+<ul>
+ <li>The <code class="language-plaintext highlighter-rouge">ExecutionContext</code> has been renamed to <code class="language-plaintext highlighter-rouge">SessionContext</code> and now supports multi-tenancy</li>
+ <li>The <code class="language-plaintext highlighter-rouge">ExecutionPlan</code> trait is no longer <code class="language-plaintext highlighter-rouge">async</code></li>
+ <li>A new serialization API for serializing plans to bytes (based on protobuf)</li>
+</ul>
+
+<p>In addition, we have added several foundational features to drive even
+more advanced query processing into DataFusion, focusing on running
+arbitrary queries larger than available memory, and pushing the
+envelope for performance of sorting, grouping, and joining even
+further:</p>
+
+<ul>
+ <li>Morsel-Driven Scheduler based on <a href="https://15721.courses.cs.cmu.edu/spring2016/papers/p743-leis.pdf">“Morsel-Driven Parallelism: A NUMA-Aware Query
+Evaluation Framework for the Many-Core Age”</a></li>
+ <li>Consolidated object store implementation and integration with parquet decoding</li>
+ <li>Memory Limited Spilling sort operator</li>
+ <li>Memory Limited Sort-Merge join operator</li>
+ <li>High performance JIT code generation for tuple comparisons</li>
+ <li>Memory efficient Row Format</li>
+</ul>
+
+<h2 id="improved-file-support">Improved file support</h2>
+
+<p>DataFusion now supports JSON, both for reading and writing. There are also new DataFrame methods for writing query
+results to files in CSV, Parquet, and JSON format.</p>
+
+<h2 id="ballista">Ballista</h2>
+
+<p>Ballista continues to mature and now supports a wider range of operators and expressions. There are also improvements
+to the scheduler to support UDFs, and there are some robustness improvements, such as cleaning up work directories
+and persisting session configs to allow schedulers to restart and continue processing in-flight jobs.</p>
+
+<h2 id="upcoming-work">Upcoming Work</h2>
+
+<p>Here are some of the initiatives that the community plans on working on prior to the next release.</p>
+
+<ul>
+ <li>There is a <a href="https://docs.google.com/document/d/1jNRbadyStSrV5kifwn0khufAwq6OnzGczG4z8oTQJP4/edit?usp=sharing">proposal to move Ballista to its own top-level arrow-ballista repository</a>
+ to decouple DataFusion and Ballista releases and to allow each project to have documentation better targeted at
+its particular audience.</li>
+ <li>We plan on increasing the frequency of DataFusion releases, with monthly releases now instead of quarterly. This
+is driven by requests from the increasing number of projects that now depend on DataFusion.</li>
+ <li>There is ongoing work to implement new optimizer rules to rewrite queries containing subquery expressions as
+joins, to support a wider range of queries.</li>
+ <li>The new scheduler based on morsel-driven execution will continue to evolve in this next release, with work to
+refine IO abstractions to improve performance and integration with the new scheduler.</li>
+ <li>Improved performance for Sort, Grouping and Joins</li>
+</ul>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, and learning about state-of-the-art query processing, we would
+love to have you join us on the journey! You can help by trying out DataFusion on some of your own data and projects
+and let us know how it goes or contribute a PR with documentation, tests or code. A list of open issues suitable
+for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a></p>
+
+<p>Check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+ </div><a class="u-url" href="/blog/blog/2022/05/16/datafusion-8.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2022/10/25/datafusion-13.0.0/index.html b/blog/2022/10/25/datafusion-13.0.0/index.html
new file mode 100644
index 0000000..0005909
--- /dev/null
+++ b/blog/2022/10/25/datafusion-13.0.0/index.html
@@ -0,0 +1,306 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 13.0.0 Project Update | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 13.0.0 Project Update" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2022/10/25/datafusion-13.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2022/10/25/datafusion-13.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-10-25T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 13.0.0 Project Update" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2022-10-25T00:00:00-06:00","datePublished":"2022-10-25T00:00:00-06:00","description":"<!–","headline":"Apache Arrow DataFusion 13.0.0 Project Update","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2022/10/25/datafusion-13.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/im [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 13.0.0 Project Update</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2022-10-25T00:00:00-06:00" itemprop="datePublished">Oct 25, 2022
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> <a href="https://crates.io/crates/datafusion"><code class="language-plaintext highlighter-rouge">13.0.0</code></a> is released, and this blog contains an update on the project for the 5 months since our <a href="https://arrow.apache.org/blog/2022/05/16/datafusion-8.0.0/">last update in May 2022</a>.</p>
+
+<p>DataFusion is an extensible and embeddable query engine, written in Rust used to create modern, fast and efficient data pipelines, ETL processes, and database systems. You may want to check out DataFusion to extend your Rust project to:</p>
+
+<ul>
+ <li>Support <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a></li>
+ <li>Support <a href="https://docs.rs/datafusion/13.0.0/datafusion/dataframe/struct.DataFrame.html">DataFrame API</a></li>
+ <li>Support a Domain Specific Query Language</li>
+ <li>Easily and quickly read and process Parquet, JSON, Avro or CSV data.</li>
+ <li>Read from remote object stores such as AWS S3, Azure Blob Storage, GCP.</li>
+</ul>
+
+<p>Even though DataFusion is 4 years “young,” it has seen significant community growth in the last few months and the momentum continues to accelerate.</p>
+
+<h1 id="background">Background</h1>
+
+<p>DataFusion is used as the engine in <a href="https://github.com/apache/arrow-datafusion#known-uses">many open source and commercial projects</a> and was one of the early open source projects to provide this capability. 2022 has validated our belief in the need for such a <a href="https://docs.google.com/presentation/d/1iNX_35sWUakee2q3zMFPyHE4IV2nC3lkCK_H6Y2qK84/edit#slide=id.p">“LLVM for database and AI systems”</a><a href="https://www.slideshare.net/AndrewLamb32/20220623-apache-arro [...]
+
+<p>While Velox and Acero focus on execution engines, DataFusion provides the entire suite of components needed to build most analytic systems, including a SQL frontend, a dataframe API, and extension points for just about everything. Some <a href="https://github.com/apache/arrow-datafusion#known-uses">DataFusion users</a> use a subset of the features such as the frontend (e.g. <a href="https://dask-sql.readthedocs.io/en/latest/">dask-sql</a>) or the execution engine, (e.g. <a href="htt [...]
+
+<p>One of DataFusion’s advantages is its implementation in <a href="https://www.rust-lang.org/">Rust</a> and thus its easy integration with the broader Rust ecosystem. Rust continues to be a major source of benefit, from the <a href="https://www.influxdata.com/blog/using-rustlangs-async-tokio-runtime-for-cpu-bound-tasks/">ease of parallelization with the high quality and standardized <code class="language-plaintext highlighter-rouge">async</code> ecosystem</a> , as well as its modern dep [...]
+<!--While we haven’t invested in the benchmarking ratings game datafusion continues to be quite speedy (todo quantity this, with some evidence) – maybe clickbench?--></p>
+
+<!--
+Maybe we can do this un a future post
+# DataFusion in Action
+
+While DataFusion really shines as an embeddable query engine, if you want to try it out and get a feel for its power, you can use the basic[`datafusion-cli`](https://docs.rs/datafusion-cli/13.0.0/datafusion_cli/) tool to get a sense for what is possible to add in your application
+
+(TODO example here of using datafusion-cli to query from local parquet files on disk)
+
+TODO: also mention you can use the same thing to query data from S3
+-->
+
+<h1 id="summary">Summary</h1>
+
+<p>We have increased the frequency of DataFusion releases to monthly instead of quarterly. This
+makes it easier for the increasing number of projects that now depend on DataFusion.</p>
+
+<p>We have also completed the “graduation” of <a href="https://github.com/apache/arrow-ballista">Ballista to its own top-level arrow-ballista repository</a>
+which decouples the two projects and allows each project to move even faster.</p>
+
+<p>Along with numerous other bug fixes and smaller improvements, here are some of the major advances:</p>
+
+<h1 id="improved-support-for-cloud-object-stores">Improved Support for Cloud Object Stores</h1>
+
+<p>DataFusion now supports many major cloud object stores (Amazon S3, Azure Blob Storage, and Google Cloud Storage) “out of the box” via the <a href="https://crates.io/crates/object_store">object_store</a> crate. Using this integration, DataFusion optimizes reading parquet files by reading only the parts of the files that are needed.</p>
+
+<h2 id="advanced-sql">Advanced SQL</h2>
+
+<p>DataFusion now supports correlated subqueries, by rewriting them as joins. See the <a href="https://arrow.apache.org/datafusion/user-guide/sql/subqueries.html">Subquery</a> page in the User Guide for more information.</p>
+
+<p>In addition to numerous other small improvements, the following SQL features are now supported:</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">ROWS</code>, <code class="language-plaintext highlighter-rouge">RANGE</code>, <code class="language-plaintext highlighter-rouge">PRECEDING</code> and <code class="language-plaintext highlighter-rouge">FOLLOWING</code> in <code class="language-plaintext highlighter-rouge">OVER</code> clauses <a href="https://github.com/apache/arrow-datafusion/issues/3570">#3570</a></li>
+ <li><code class="language-plaintext highlighter-rouge">ROLLUP</code> and <code class="language-plaintext highlighter-rouge">CUBE</code> grouping set expressions <a href="https://github.com/apache/arrow-datafusion/issues/2446">#2446</a></li>
+ <li><code class="language-plaintext highlighter-rouge">SUM DISTINCT</code> aggregate support <a href="https://github.com/apache/arrow-datafusion/issues/2405">#2405</a></li>
+ <li><code class="language-plaintext highlighter-rouge">IN</code> and <code class="language-plaintext highlighter-rouge">NOT IN</code> Subqueries by rewriting them to <code class="language-plaintext highlighter-rouge">SEMI</code> / <code class="language-plaintext highlighter-rouge">ANTI</code> <a href="https://github.com/apache/arrow-datafusion/issues/2885">#2421</a></li>
+ <li>Non equality predicates in <code class="language-plaintext highlighter-rouge">ON</code> clause of <code class="language-plaintext highlighter-rouge">LEFT</code>, <code class="language-plaintext highlighter-rouge">RIGHT, </code>and <code class="language-plaintext highlighter-rouge">FULL</code> joins <a href="https://github.com/apache/arrow-datafusion/issues/2591">#2591</a></li>
+ <li>Exact <code class="language-plaintext highlighter-rouge">MEDIAN</code> <a href="https://github.com/apache/arrow-datafusion/issues/3009">#3009</a></li>
+ <li><code class="language-plaintext highlighter-rouge">GROUPING SETS</code>/<code class="language-plaintext highlighter-rouge">CUBE</code>/<code class="language-plaintext highlighter-rouge">ROLLUP</code> <a href="https://github.com/apache/arrow-datafusion/issues/2716">#2716</a></li>
+</ul>
+
+<h1 id="more-ddl-support">More DDL Support</h1>
+
+<p>Just as it is important to query, it is also important to give users the ability to define their data sources. We have added:</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">CREATE VIEW</code> <a href="https://github.com/apache/arrow-datafusion/issues/2279">#2279</a></li>
+ <li><code class="language-plaintext highlighter-rouge">DESCRIBE <table></code> <a href="https://github.com/apache/arrow-datafusion/issues/2642">#2642</a></li>
+ <li>Custom / Dynamic table provider factories <a href="https://github.com/apache/arrow-datafusion/issues/3311">#3311</a></li>
+ <li><code class="language-plaintext highlighter-rouge">SHOW CREATE TABLE</code> for support for views <a href="https://github.com/apache/arrow-datafusion/issues/2830">#2830</a></li>
+</ul>
+
+<h1 id="faster-execution">Faster Execution</h1>
+<p>Performance is always an important goal for DataFusion, and there are a number of significant new optimizations such as</p>
+
+<ul>
+ <li>Optimizations of TopK (queries with a <code class="language-plaintext highlighter-rouge">LIMIT</code> or <code class="language-plaintext highlighter-rouge">OFFSET</code> clause): <a href="https://github.com/apache/arrow-datafusion/issues/3527">#3527</a>, <a href="https://github.com/apache/arrow-datafusion/issues/2521">#2521</a></li>
+ <li>Reduce <code class="language-plaintext highlighter-rouge">left</code>/<code class="language-plaintext highlighter-rouge">right</code>/<code class="language-plaintext highlighter-rouge">full</code> joins to <code class="language-plaintext highlighter-rouge">inner</code> join <a href="https://github.com/apache/arrow-datafusion/issues/2750">#2750</a></li>
+ <li>Convert cross joins to inner joins when possible <a href="https://github.com/apache/arrow-datafusion/issues/3482">#3482</a></li>
+ <li>Sort preserving <code class="language-plaintext highlighter-rouge">SortMergeJoin</code> <a href="https://github.com/apache/arrow-datafusion/issues/2699">#2699</a></li>
+ <li>Improvements in group by and sort performance <a href="https://github.com/apache/arrow-datafusion/issues/2375">#2375</a></li>
+ <li>Adaptive <code class="language-plaintext highlighter-rouge">regex_replace</code> implementation <a href="https://github.com/apache/arrow-datafusion/issues/3518">#3518</a></li>
+</ul>
+
+<h1 id="optimizer-enhancements">Optimizer Enhancements</h1>
+<p>Internally the optimizer has been significantly enhanced as well.</p>
+
+<ul>
+ <li>Casting / coercion now happens during logical planning <a href="https://github.com/apache/arrow-datafusion/issues/3396">#3185</a> <a href="https://github.com/apache/arrow-datafusion/issues/3636">#3636</a></li>
+ <li>More sophisticated expression analysis and simplification is available</li>
+</ul>
+
+<h1 id="parquet">Parquet</h1>
+<ul>
+ <li>The parquet reader can now read directly from parquet files on remote object storage <a href="https://github.com/apache/arrow-datafusion/issues/2677">#2489</a> <a href="https://github.com/apache/arrow-datafusion/issues/3051">#3051</a></li>
+ <li>Experimental support for “predicate pushdown” with late materialization after filtering during the scan (another blog post on this topic is coming soon).</li>
+ <li>Support reading directly from AWS S3 and other object stores via <code class="language-plaintext highlighter-rouge">datafusion-cli </code> <a href="https://github.com/apache/arrow-datafusion/issues/3631">#3631</a></li>
+</ul>
+
+<h1 id="datatype-support">DataType Support</h1>
+<ul>
+ <li>Support for <code class="language-plaintext highlighter-rouge">TimestampTz</code> <a href="https://github.com/apache/arrow-datafusion/issues/3660">#3660</a></li>
+ <li>Expanded support for the <code class="language-plaintext highlighter-rouge">Decimal</code> type, including <code class="language-plaintext highlighter-rouge">IN</code> list and better built in coercion.</li>
+ <li>Expanded support for date/time manipulation such as <code class="language-plaintext highlighter-rouge">date_bin</code> built-in function , timestamp <code class="language-plaintext highlighter-rouge">+/-</code> interval, <code class="language-plaintext highlighter-rouge">TIME</code> literal values <a href="https://github.com/apache/arrow-datafusion/issues/3010">#3010</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3110">#3110</a>, <a href="https://github.com/apache [...]
+ <li>Binary operations (<code class="language-plaintext highlighter-rouge">AND</code>, <code class="language-plaintext highlighter-rouge">XOR</code>, etc): <a href="https://github.com/apache/arrow-datafusion/issues/1619">#3037</a> <a href="https://github.com/apache/arrow-datafusion/issues/3430">#3420</a></li>
+ <li><code class="language-plaintext highlighter-rouge">IS TRUE/FALSE</code> and <code class="language-plaintext highlighter-rouge">IS [NOT] UNKNOWN</code> <a href="https://github.com/apache/arrow-datafusion/issues/3235">#3235</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3246">#3246</a></li>
+</ul>
+
+<h2 id="upcoming-work">Upcoming Work</h2>
+<p>With the community growing and code accelerating, there is so much great stuff on the horizon. Some features we expect to land in the next few months:</p>
+
+<ul>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/3462">Complete Parquet Pushdown</a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/3148">Additional date/time support</a></li>
+ <li>Cost models, Nested Join Optimizations, analysis framework <a href="https://github.com/apache/arrow-datafusion/issues/128">#128</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3843">#3843</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3845">#3845</a></li>
+</ul>
+
+<h1 id="community-growth">Community Growth</h1>
+
+<p>The DataFusion 9.0.0 and 13.0.0 releases consists of 433 PRs from 64 distinct contributors. This does not count all the work that goes into our dependencies such as <a href="https://crates.io/crates/arrow">arrow</a>, <a href="https://crates.io/crates/parquet">parquet</a>, and <a href="https://crates.io/crates/object_store">object_store</a>, that much of the same community helps nurture.</p>
+
+<!--
+$ git log --pretty=oneline 9.0.0..13.0.0 . | wc -l
+433
+
+$ git shortlog -sn 9.0.0..13.0.0 . | wc -l
+65
+-->
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>Kudos to everyone in the community who contributed ideas, discussions, bug reports, documentation and code. It is exciting to be building something so cool together!</p>
+
+<p>If you are interested in contributing to DataFusion, we would love to
+have you join us on our journey to create the most advanced open
+source query engine. You can try out DataFusion on some of your own
+data and projects and let us know how it goes or contribute a PR with
+documentation, tests or code. A list of open issues suitable for
+beginners is
+<a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>Check out our <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+<h2 id="appendix-contributor-shoutout">Appendix: Contributor Shoutout</h2>
+
+<p>To give a sense of the number of people who contribute to this project regularly, we present for your consideration the following list derived from <code class="language-plaintext highlighter-rouge">git shortlog -sn 9.0.0..13.0.0 .</code> Thank you all again!</p>
+
+<!-- Note: combined kmitchener and Kirk Mitchener -->
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 87 Andy Grove
+ 71 Andrew Lamb
+ 29 Kun Liu
+ 29 Kirk Mitchener
+ 17 Wei-Ting Kuo
+ 14 Yang Jiang
+ 12 Raphael Taylor-Davies
+ 11 Batuhan Taskaya
+ 10 Brent Gardner
+ 10 Remzi Yang
+ 10 comphead
+ 10 xudong.w
+ 8 AssHero
+ 7 Ruihang Xia
+ 6 Dan Harris
+ 6 Daniël Heres
+ 6 Ian Alexander Joiner
+ 6 Mike Roberts
+ 6 askoa
+ 4 BaymaxHWY
+ 4 gorkem
+ 4 jakevin
+ 3 George Andronchik
+ 3 Sarah Yurick
+ 3 Stuart Carnie
+ 2 Dalton Modlin
+ 2 Dmitry Patsura
+ 2 JasonLi
+ 2 Jon Mease
+ 2 Marco Neumann
+ 2 yahoNanJing
+ 1 Adilet Sarsembayev
+ 1 Ayush Dattagupta
+ 1 Dezhi Wu
+ 1 Dhamotharan Sritharan
+ 1 Eduard Karacharov
+ 1 Francis Du
+ 1 Harbour Zheng
+ 1 Ismaël Mejía
+ 1 Jack Klamer
+ 1 Jeremy Dyer
+ 1 Jiayu Liu
+ 1 Kamil Konior
+ 1 Liang-Chi Hsieh
+ 1 Martin Grigorov
+ 1 Matthijs Brobbel
+ 1 Mehmet Ozan Kabak
+ 1 Metehan Yıldırım
+ 1 Morgan Cassels
+ 1 Nitish Tiwari
+ 1 Renjie Liu
+ 1 Rito Takeuchi
+ 1 Robert Pack
+ 1 Thomas Cameron
+ 1 Vrishabh
+ 1 Xin Hao
+ 1 Yijie Shen
+ 1 byteink
+ 1 kamille
+ 1 mateuszkj
+ 1 nvartolomei
+ 1 yourenawo
+ 1 Özgür Akkurt
+</code></pre></div></div>
+
+
+ </div><a class="u-url" href="/blog/blog/2022/10/25/datafusion-13.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2022/10/28/ballista-0.9.0/index.html b/blog/2022/10/28/ballista-0.9.0/index.html
new file mode 100644
index 0000000..6b800ae
--- /dev/null
+++ b/blog/2022/10/28/ballista-0.9.0/index.html
@@ -0,0 +1,172 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow Ballista 0.9.0 Release | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow Ballista 0.9.0 Release" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2022/10/28/ballista-0.9.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2022/10/28/ballista-0.9.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-10-28T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow Ballista 0.9.0 Release" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2022-10-28T00:00:00-06:00","datePublished":"2022-10-28T00:00:00-06:00","description":"<!–","headline":"Apache Arrow Ballista 0.9.0 Release","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2022/10/28/ballista-0.9.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwhite_ [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow Ballista 0.9.0 Release</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2022-10-28T00:00:00-06:00" itemprop="datePublished">Oct 28, 2022
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://github.com/apache/arrow-ballista">Ballista</a> is an Arrow-native distributed SQL query engine implemented in Rust.</p>
+
+<p>Ballista 0.9.0 is now available and is the most significant release since the project was <a href="http://arrow.apache.org/blog/2021/04/12/ballista-donation/">donated</a> to Apache
+Arrow in 2021.</p>
+
+<p>This release represents 4 weeks of work, with 66 commits from 14 contributors:</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 22 Andy Grove
+ 12 yahoNanJing
+ 6 Daniël Heres
+ 4 Brent Gardner
+ 4 dependabot[bot]
+ 4 r.4ntix
+ 3 Stefan Stanciulescu
+ 3 mingmwang
+ 2 Ken Suenobu
+ 2 Yang Jiang
+ 1 Metehan Yıldırım
+ 1 Trent Feda
+ 1 askoa
+ 1 yangzhong
+</code></pre></div></div>
+
+<h2 id="release-highlights">Release Highlights</h2>
+
+<p>The release notes below are not exhaustive and only expose selected highlights of the release. Many other bug fixes
+and improvements have been made: we refer you to the <a href="https://github.com/apache/arrow-ballista/blob/0.9.0-rc2/ballista/CHANGELOG.md">complete changelog</a>.</p>
+
+<h3 id="support-for-cloud-object-stores-and-distributed-file-systems">Support for Cloud Object Stores and Distributed File Systems</h3>
+
+<p>This is the first release of Ballista to have documented support for querying data from distributed file systems and
+object stores. Currently, S3 and HDFS are supported. Support for Google Cloud Storage and Azure Blob Storage is planned
+for the next release.</p>
+
+<h3 id="flight-sql--jdbc-support">Flight SQL & JDBC support</h3>
+
+<p>The Ballista scheduler now implements the <a href="https://arrow.apache.org/blog/2022/02/16/introducing-arrow-flight-sql/">Flight SQL protocol</a>, enabling any compliant Flight SQL client
+to connect to and run queries against a Ballista cluster.</p>
+
+<p>The Apache Arrow Flight SQL JDBC driver can be used to connect Business Intelligence tools to a Ballista cluster.</p>
+
+<h3 id="python-bindings">Python Bindings</h3>
+
+<p>It is now possible to connect to a Ballista cluster from Python and execute queries using both the DataFrame and SQL
+interfaces.</p>
+
+<h3 id="scheduler-web-user-interface-and-rest-api">Scheduler Web User Interface and REST API</h3>
+
+<p>The scheduler now has a web user interface for monitoring queries. It is also possible to view graphical query plans
+that show how the query was executed, along with metrics.</p>
+
+<p><img src="/blog/img/2022-10-28-ballista-web-ui.png" width="800" /></p>
+
+<p>The REST API that powers the user interface can also be accessed directly.</p>
+
+<h3 id="simplified-kubernetes-deployment">Simplified Kubernetes Deployment</h3>
+
+<p>Ballista now provides a <a href="https://github.com/apache/arrow-ballista/tree/master/helm">Helm chart</a> for simplified Kubernetes deployment.</p>
+
+<h3 id="user-guide">User Guide</h3>
+
+<p>The user guide is published at <a href="https://arrow.apache.org/ballista/">https://arrow.apache.org/ballista/</a> and provides
+deployment instructions for Docker, Docker Compose, and Kubernetes, as well as references for configuring and
+tuning Ballista.</p>
+
+<h2 id="roadmap">Roadmap</h2>
+
+<p>The Ballista community is currently focused on the following tasks for the next release:</p>
+
+<ul>
+ <li>Support for Azure Blob Storage and Google Cloud Storage</li>
+ <li>Improve benchmark performance by implementing more query optimizations</li>
+ <li>Improve scheduler web user interface</li>
+ <li>Publish Docker images to GitHub Container Registry</li>
+</ul>
+
+<p>The detailed list of issues planned for the 0.10.0 release can be found in the <a href="https://github.com/apache/arrow-ballista/issues/361">tracking issue</a>.</p>
+
+<h2 id="getting-involved">Getting Involved</h2>
+
+<p>Ballista has a friendly community and we welcome contributions. A good place to start is to following the instructions
+in the <a href="https://arrow.apache.org/ballista/">user guide</a> and try using Ballista with your own SQL queries and ETL pipelines, and file issues
+for any bugs or feature suggestions.</p>
+
+
+ </div><a class="u-url" href="/blog/blog/2022/10/28/ballista-0.9.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2023/01/19/datafusion-16.0.0/index.html b/blog/2023/01/19/datafusion-16.0.0/index.html
new file mode 100644
index 0000000..2962410
--- /dev/null
+++ b/blog/2023/01/19/datafusion-16.0.0/index.html
@@ -0,0 +1,326 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 16.0.0 Project Update | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 16.0.0 Project Update" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2023/01/19/datafusion-16.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2023/01/19/datafusion-16.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2023-01-19T00:00:00-07:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 16.0.0 Project Update" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2023-01-19T00:00:00-07:00","datePublished":"2023-01-19T00:00:00-07:00","description":"<!–","headline":"Apache Arrow DataFusion 16.0.0 Project Update","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2023/01/19/datafusion-16.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/im [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 16.0.0 Project Update</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2023-01-19T00:00:00-07:00" itemprop="datePublished">Jan 19, 2023
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible
+query execution framework, written in <a href="https://www.rust-lang.org/">Rust</a>,
+that uses <a href="https://arrow.apache.org">Apache Arrow</a> as its
+in-memory format. It is targeted primarily at developers creating data
+intensive analytics, and offers mature
+<a href="https://arrow.apache.org/datafusion/user-guide/sql/index.html">SQL support</a>,
+a DataFrame API, and many extension points.</p>
+
+<p>Systems based on DataFusion perform very well in benchmarks,
+especially considering they operate directly on parquet files rather
+than first loading into a specialized format. Some recent highlights
+include <a href="https://benchmark.clickhouse.com/">clickbench</a> and the
+<a href="https://www.cloudfuse.io/dashboards/standalone-engines">Cloudfuse.io standalone query
+engines</a> page.</p>
+
+<p>DataFusion is also part of a longer term trend, articulated clearly by
+<a href="http://www.cs.cmu.edu/~pavlo/">Andy Pavlo</a> in his <a href="https://ottertune.com/blog/2022-databases-retrospective/">2022 Databases
+Retrospective</a>.
+Database frameworks are proliferating and it is likely that all OLAP
+DBMSs and other data heavy applications, such as machine learning,
+will <strong>require</strong> a vectorized, highly performant query engine in the next
+5 years to remain relevant. The only practical way to make such
+technology so widely available without many millions of dollars of
+investment is though open source engine such as DataFusion or
+<a href="https://github.com/facebookincubator/velox">Velox</a>.</p>
+
+<p>The rest of this post describes the improvements made to DataFusion
+over the last three months and some hints of where we are heading.</p>
+
+<h2 id="community-growth">Community Growth</h2>
+
+<p>We again saw significant growth in the DataFusion community since <a href="https://arrow.apache.org/blog/2022/10/25/datafusion-13.0.0/">our last update</a>. There are some interesting metrics on <a href="https://ossrank.com/p/1573-apache-arrow-datafusion">OSSRank</a>.</p>
+
+<p>The DataFusion 16.0.0 release consists of 543 PRs from 73 distinct contributors, not including all the work that goes into dependencies such as <a href="https://crates.io/crates/arrow">arrow</a>, <a href="https://crates.io/crates/parquet">parquet</a>, and <a href="https://crates.io/crates/object_store">object_store</a>, that much of the same community helps support. Thank you all for your help</p>
+
+<!--
+$ git log --pretty=oneline 13.0.0..16.0.0 . | wc -l
+ 543
+
+$ git shortlog -sn 13.0.0..16.0.0 . | wc -l
+ 73
+-->
+<p>Several <a href="https://github.com/apache/arrow-datafusion#known-uses">new systems based on DataFusion</a> were recently added:</p>
+
+<ul>
+ <li><a href="https://github.com/GreptimeTeam/greptimedb">Greptime DB</a></li>
+ <li><a href="https://synnada.ai/">Synnada</a></li>
+ <li><a href="https://github.com/PRQL/prql-query">PRQL</a></li>
+ <li><a href="https://github.com/parseablehq/parseable">Parseable</a></li>
+ <li><a href="https://github.com/splitgraph/seafowl">SeaFowl</a></li>
+</ul>
+
+<h2 id="performance-">Performance 🚀</h2>
+
+<p>Performance and efficiency are core values for
+DataFusion. While there is still a gap between DataFusion and the best of
+breed, tightly integrated systems such as <a href="https://duckdb.org">DuckDB</a>
+and <a href="https://www.pola.rs/">Polars</a>, DataFusion is
+closing the gap quickly. Performance highlights from the last three
+months:</p>
+
+<ul>
+ <li>Up to 30% Faster Sorting and Merging using the new <a href="https://arrow.apache.org/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">Row Format</a></li>
+ <li><a href="https://arrow.apache.org/blog/2022/12/26/querying-parquet-with-millisecond-latency/">Advanced predicate pushdown</a>, directly on parquet, directly from object storage, enabling sub millisecond filtering. <!-- Andrew nots: we should really get this turned on by default --></li>
+ <li><code class="language-plaintext highlighter-rouge">70%</code> faster <code class="language-plaintext highlighter-rouge">IN</code> expressions evaluation (<a href="https://github.com/apache/arrow-datafusion/issues/4057">#4057</a>)</li>
+ <li>Sort and partition aware optimizations (<a href="https://github.com/apache/arrow-datafusion/issues/3969">#3969</a> and <a href="https://github.com/apache/arrow-datafusion/issues/4691">#4691</a>)</li>
+ <li>Filter selectivity analysis (<a href="https://github.com/apache/arrow-datafusion/issues/3868">#3868</a>)</li>
+</ul>
+
+<h2 id="runtime-resource-limits">Runtime Resource Limits</h2>
+
+<p>Previously, DataFusion could potentially use unbounded amounts of memory for certain queries that included Sorts, Grouping or Joins.</p>
+
+<p>In version 16.0.0, it is possible to limit DataFusion’s memory usage for Sorting and Grouping. We are looking for help adding similar limiting for Joins as well as expanding our algorithms to optionally spill to secondary storage. See <a href="https://github.com/apache/arrow-datafusion/issues/3941">#3941</a> for more detail.</p>
+
+<h2 id="sql-window-functions">SQL Window Functions</h2>
+
+<p><a href="https://en.wikipedia.org/wiki/Window_function_(SQL)">SQL Window Functions</a> are useful for a variety of analysis and DataFusion’s implementation support expanded significantly:</p>
+
+<ul>
+ <li>Custom window frames such as <code class="language-plaintext highlighter-rouge">... OVER (ORDER BY ... RANGE BETWEEN 0.2 PRECEDING AND 0.2 FOLLOWING)</code></li>
+ <li>Unbounded window frames such as <code class="language-plaintext highlighter-rouge">... OVER (ORDER BY ... RANGE UNBOUNDED ROWS PRECEDING)</code></li>
+ <li>Support for the <code class="language-plaintext highlighter-rouge">NTILE</code> window function (<a href="https://github.com/apache/arrow-datafusion/issues/4676">#4676</a>)</li>
+ <li>Support for <code class="language-plaintext highlighter-rouge">GROUPS</code> mode (<a href="https://github.com/apache/arrow-datafusion/issues/4155">#4155</a>)</li>
+</ul>
+
+<h1 id="improved-joins">Improved Joins</h1>
+
+<p>Joins are often the most complicated operations to handle well in
+analytics systems and DataFusion 16.0.0 offers significant improvements
+such as</p>
+
+<ul>
+ <li>Cost based optimizer (CBO) automatically reorders join evaluations, selects algorithms (Merge / Hash), and pick build side based on available statistics and join type (<code class="language-plaintext highlighter-rouge">INNER</code>, <code class="language-plaintext highlighter-rouge">LEFT</code>, etc) (<a href="https://github.com/apache/arrow-datafusion/issues/4219">#4219</a>)</li>
+ <li>Fast non <code class="language-plaintext highlighter-rouge">column=column</code> equijoins such as <code class="language-plaintext highlighter-rouge">JOIN ON a.x + 5 = b.y</code></li>
+ <li>Better performance on non-equijoins (<a href="https://github.com/apache/arrow-datafusion/issues/4562">#4562</a>) <!-- TODO is this a good thing to mention as any time this is usd the query is going to go slow or the data size is small --></li>
+</ul>
+
+<h1 id="streaming-execution">Streaming Execution</h1>
+
+<p>One emerging use case for Datafusion is as a foundation for
+streaming-first data platforms. An important prerequisite
+is support for incremental execution for queries that can be computed
+incrementally.</p>
+
+<p>With this release, DataFusion now supports the following streaming features:</p>
+
+<ul>
+ <li>Data ingestion from infinite files such as FIFOs (<a href="https://github.com/apache/arrow-datafusion/issues/4694">#4694</a>),</li>
+ <li>Detection of pipeline-breaking queries in streaming use cases (<a href="https://github.com/apache/arrow-datafusion/issues/4694">#4694</a>),</li>
+ <li>Automatic input swapping for joins so probe side is a data stream (<a href="https://github.com/apache/arrow-datafusion/issues/4694">#4694</a>),</li>
+ <li>Intelligent elision of pipeline-breaking sort operations whenever possible (<a href="https://github.com/apache/arrow-datafusion/issues/4691">#4691</a>),</li>
+ <li>Incremental execution for more types of queries; e.g. queries involving finite window frames (<a href="https://github.com/apache/arrow-datafusion/issues/4777">#4777</a>).</li>
+</ul>
+
+<p>These are a major steps forward, and we plan even more improvements over the next few releases.</p>
+
+<h1 id="better-support-for-distributed-catalogs">Better Support for Distributed Catalogs</h1>
+
+<p>16.0.0 has been enhanced support for asynchronous catalogs (<a href="https://github.com/apache/arrow-datafusion/issues/4607">#4607</a>)
+to better support distributed metadata stores such as
+<a href="https://delta.io/">Delta.io</a> and <a href="https://iceberg.apache.org/">Apache
+Iceberg</a> which require asynchronous I/O
+during planning to access remote catalogs. Previously, DataFusion
+required synchronous access to all relevant catalog information.</p>
+
+<h1 id="additional-sql-support">Additional SQL Support</h1>
+<p>SQL support continues to improve, including some of these highlights:</p>
+
+<ul>
+ <li>Add TPC-DS query planning regression tests <a href="https://github.com/apache/arrow-datafusion/issues/4719">#4719</a></li>
+ <li>Support for <code class="language-plaintext highlighter-rouge">PREPARE</code> statement <a href="https://github.com/apache/arrow-datafusion/issues/4490">#4490</a></li>
+ <li>Automatic coercions ast between Date and Timestamp <a href="https://github.com/apache/arrow-datafusion/issues/4726">#4726</a></li>
+ <li>Support type coercion for timestamp and utf8 <a href="https://github.com/apache/arrow-datafusion/issues/4312">#4312</a></li>
+ <li>Full support for time32 and time64 literal values (<code class="language-plaintext highlighter-rouge">ScalarValue</code>) <a href="https://github.com/apache/arrow-datafusion/issues/4156">#4156</a></li>
+ <li>New functions, incuding <code class="language-plaintext highlighter-rouge">uuid()</code> <a href="https://github.com/apache/arrow-datafusion/issues/4041">#4041</a>, <code class="language-plaintext highlighter-rouge">current_time</code> <a href="https://github.com/apache/arrow-datafusion/issues/4054">#4054</a>, <code class="language-plaintext highlighter-rouge">current_date</code> <a href="https://github.com/apache/arrow-datafusion/issues/4022">#4022</a></li>
+ <li>Compressed CSV/JSON support <a href="https://github.com/apache/arrow-datafusion/issues/3642">#3642</a></li>
+</ul>
+
+<p>The community has also invested in new <a href="https://github.com/apache/arrow-datafusion/blob/master/datafusion/core/tests/sqllogictests/README.md">sqllogic based</a> tests to keep improving DataFusion’s quality with less effort.</p>
+
+<h1 id="plan-serialization-and-substrait">Plan Serialization and Substrait</h1>
+
+<p>DataFusion now supports serialization of physical plans, with a custom protocol buffers format. In addition, we are adding initial support for <a href="https://substrait.io/">Substrait</a>, a Cross-Language Serialization for Relational Algebra</p>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>Kudos to everyone in the community who contributed ideas, discussions, bug reports, documentation and code. It is exciting to be building something so cool together!</p>
+
+<p>If you are interested in contributing to DataFusion, we would love to
+have you join us. You can try out DataFusion on some of your own
+data and projects and let us know how it goes or contribute a PR with
+documentation, tests or code. A list of open issues suitable for
+beginners is
+<a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>Check out our <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+<h2 id="appendix-contributor-shoutout">Appendix: Contributor Shoutout</h2>
+
+<p>Here is a list of people who have contributed PRs to this project over the last three releases, derived from <code class="language-plaintext highlighter-rouge">git shortlog -sn 13.0.0..16.0.0 .</code> Thank you all!</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 113 Andrew Lamb
+ 58 jakevin
+ 46 Raphael Taylor-Davies
+ 30 Andy Grove
+ 19 Batuhan Taskaya
+ 19 Remzi Yang
+ 17 ygf11
+ 16 Burak
+ 16 Jeffrey
+ 16 Marco Neumann
+ 14 Kun Liu
+ 12 Yang Jiang
+ 10 mingmwang
+ 9 Daniël Heres
+ 9 Mustafa akur
+ 9 comphead
+ 9 mvanschellebeeck
+ 9 xudong.w
+ 7 dependabot[bot]
+ 7 yahoNanJing
+ 6 Brent Gardner
+ 5 AssHero
+ 4 Jiayu Liu
+ 4 Wei-Ting Kuo
+ 4 askoa
+ 3 André Calado Coroado
+ 3 Jie Han
+ 3 Jon Mease
+ 3 Metehan Yıldırım
+ 3 Nga Tran
+ 3 Ruihang Xia
+ 3 baishen
+ 2 Berkay Şahin
+ 2 Dan Harris
+ 2 Dongyan Zhou
+ 2 Eduard Karacharov
+ 2 Kikkon
+ 2 Liang-Chi Hsieh
+ 2 Marko Milenković
+ 2 Martin Grigorov
+ 2 Roman Nozdrin
+ 2 Tim Van Wassenhove
+ 2 r.4ntix
+ 2 unconsolable
+ 2 unvalley
+ 1 Ajaya Agrawal
+ 1 Alexander Spies
+ 1 ArkashaJavelin
+ 1 Artjoms Iskovs
+ 1 BoredPerson
+ 1 Christian Salvati
+ 1 Creampanda
+ 1 Data Psycho
+ 1 Francis Du
+ 1 Francis Le Roy
+ 1 LFC
+ 1 Marko Grujic
+ 1 Matt Willian
+ 1 Matthijs Brobbel
+ 1 Max Burke
+ 1 Mehmet Ozan Kabak
+ 1 Rito Takeuchi
+ 1 Roman Zeyde
+ 1 Vrishabh
+ 1 Zhang Li
+ 1 ZuoTiJia
+ 1 byteink
+ 1 cfraz89
+ 1 nbr
+ 1 xxchan
+ 1 yujie.zhang
+ 1 zembunia
+ 1 哇呜哇呜呀咦耶
+</code></pre></div></div>
+
+
+ </div><a class="u-url" href="/blog/blog/2023/01/19/datafusion-16.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2023/06/24/datafusion-25.0.0/index.html b/blog/2023/06/24/datafusion-25.0.0/index.html
new file mode 100644
index 0000000..f0ad19d
--- /dev/null
+++ b/blog/2023/06/24/datafusion-25.0.0/index.html
@@ -0,0 +1,303 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 26.0.0 | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 26.0.0" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2023/06/24/datafusion-25.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2023/06/24/datafusion-25.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2023-06-24T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 26.0.0" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2023-06-24T00:00:00-06:00","datePublished":"2023-06-24T00:00:00-06:00","description":"<!–","headline":"Apache Arrow DataFusion 26.0.0","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2023/06/24/datafusion-25.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwhite_or [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 26.0.0</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2023-06-24T00:00:00-06:00" itemprop="datePublished">Jun 24, 2023
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<p>It has been a whirlwind 6 months of DataFusion development since <a href="https://arrow.apache.org/blog/2023/01/19/datafusion-16.0.0">our
+last update</a>: the community has grown, many features have been added,
+performance improved and we are <a href="https://github.com/apache/arrow-datafusion/discussions/6475">discussing</a> branching out to our own
+top level Apache Project.</p>
+
+<h2 id="background">Background</h2>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> is an extensible query engine and database
+toolkit, written in <a href="https://www.rust-lang.org/">Rust</a>, that uses <a href="https://arrow.apache.org">Apache Arrow</a> as its in-memory
+format.</p>
+
+<p>DataFusion, along with <a href="https://calcite.apache.org">Apache Calcite</a>, Facebook’s <a href="https://github.com/facebookincubator/velox">Velox</a> and
+similar technology are part of the next generation “<a href="https://www.usenix.org/publications/login/winter2018/khurana">Deconstructed
+Database</a>” architectures, where new systems are built on a foundation
+of fast, modular components, rather as a single tightly integrated
+system.</p>
+
+<p>While single tightly integrated systems such as <a href="https://spark.apache.org/">Spark</a>, <a href="https://duckdb.org">DuckDB</a> and
+<a href="https://www.pola.rs/">Pola.rs</a> are great pieces of technology, our community believes that
+anyone developing new data heavy application, such as those common in
+machine learning in the next 5 years, will <strong>require</strong> a high
+performance, vectorized, query engine to remain relevant. The only
+practical way to gain access to such technology without investing many
+millions of dollars to build a new tightly integrated engine, is
+though open source projects like DataFusion and similar enabling
+technologies such as <a href="https://arrow.apache.org">Apache Arrow</a> and <a href="https://www.rust-lang.org/">Rust</a>.</p>
+
+<p>DataFusion is targeted primarily at developers creating other data
+intensive analytics, and offers:</p>
+
+<ul>
+ <li>High performance, native, parallel streaming execution engine</li>
+ <li>Mature <a href="https://arrow.apache.org/datafusion/user-guide/sql/index.html">SQL support</a>, featuring subqueries, window functions, grouping sets, and more</li>
+ <li>Built in support for Parquet, Avro, CSV, JSON and Arrow formats and easy extension for others</li>
+ <li>Native DataFrame API and <a href="https://arrow.apache.org/datafusion-python/">python bindings</a></li>
+ <li><a href="https://docs.rs/datafusion/latest/datafusion/index.html">Well documented</a> source code and architecture, designed to be customized to suit downstream project needs</li>
+ <li>High quality, easy to use code <a href="https://crates.io/crates/datafusion/versions">released every 2 weeks to crates.io</a></li>
+ <li>Welcoming, open community, governed by the highly regarded and well understood <a href="https://www.apache.org/">Apache Software Foundation</a></li>
+</ul>
+
+<p>The rest of this post highlights some of the improvements we have made
+to DataFusion over the last 6 months and a preview of where we are
+heading. You can see a list of all changes in the detailed
+<a href="https://github.com/apache/arrow-datafusion/blob/main/datafusion/CHANGELOG.md">CHANGELOG</a>.</p>
+
+<h2 id="even-better-performance">(Even) Better Performance</h2>
+
+<p><a href="https://voltrondata.com/resources/speeds-and-feeds-hardware-and-software-matter">Various</a> benchmarks show DataFusion to be quite close or <a href="https://github.com/tustvold/access-log-bench">even
+faster</a> to the state of the art in analytic performance (at the moment
+this seems to be DuckDB). We continually work on improving performance
+(see <a href="https://github.com/apache/arrow-datafusion/issues/5546">#5546</a> for a list) and would love additional help in this area.</p>
+
+<p>DataFusion now reads single large Parquet files significantly faster by
+<a href="https://github.com/apache/arrow-datafusion/pull/5057">parallelizing across multiple cores</a>. Native speeds for reading JSON
+and CSV files are also up to 2.5x faster thanks to improvements
+upstream in arrow-rs <a href="https://github.com/apache/arrow-rs/pull/3479#issuecomment-1384353159">JSON reader</a> and <a href="https://github.com/apache/arrow-rs/pull/3365">CSV reader</a>.</p>
+
+<p>Also, we have integrated the <a href="https://arrow.apache.org/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">arrow-rs Row Format</a> into DataFusion resulting in up to <a href="https://github.com/apache/arrow-datafusion/pull/6163">2-3x faster sorting and merging</a>.</p>
+
+<h2 id="improved-documentation-and-website">Improved Documentation and Website</h2>
+
+<p>Part of growing the DataFusion community is ensuring that DataFusion’s
+features are understood and that it is easy to contribute and
+participate. To that end the <a href="https://arrow.apache.org/datafusion/">website</a> has been cleaned up, <a href="https://docs.rs/datafusion/latest/datafusion/index.html#architecture">the
+architecture guide</a> expanded, the <a href="https://arrow.apache.org/datafusion/contributor-guide/roadmap.html">roadmap</a> updated, and several
+overview talks created:</p>
+
+<ul>
+ <li>Apr 2023 <em>Query Engine</em>: <a href="https://youtu.be/NVKujPxwSBA">recording</a> and <a href="https://docs.google.com/presentation/d/1D3GDVas-8y0sA4c8EOgdCvEjVND4s2E7I6zfs67Y4j8/edit#slide=id.p">slides</a></li>
+ <li>April 2023 <em>Logical Plan and Expressions</em>: <a href="https://youtu.be/EzZTLiSJnhY">recording</a> and <a href="https://docs.google.com/presentation/d/1ypylM3-w60kVDW7Q6S99AHzvlBgciTdjsAfqNP85K30">slides</a></li>
+ <li>April 2023 <em>Physical Plan and Execution</em>: <a href="https://youtu.be/2jkWU3_w6z0">recording</a> and <a href="https://docs.google.com/presentation/d/1cA2WQJ2qg6tx6y4Wf8FH2WVSm9JQ5UgmBWATHdik0hg">slides</a></li>
+</ul>
+
+<h2 id="new-features">New Features</h2>
+
+<h3 id="more-streaming-less-memory">More Streaming, Less Memory</h3>
+
+<p>We have made significant progress on the <a href="https://github.com/apache/arrow-datafusion/issues/4285">streaming execution roadmap</a>
+such as <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/trait.ExecutionPlan.html#method.unbounded_output">unbounded datasources</a>, <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/aggregates/enum.GroupByOrderMode.html">streaming group by</a>, sophisticated
+<a href="https://docs.rs/datafusion/latest/datafusion/physical_optimizer/global_sort_selection/index.html">sort</a> and <a href="https://docs.rs/datafusion/latest/datafusion/physical_optimizer/repartition/index.html">repartitioning</a> improvements in the optimizer, and support
+for <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/joins/struct.SymmetricHashJoinExec.html">symmetric hash join</a> (read more about that in the great <a href="https://www.synnada.ai/blog/general-purpose-stream-joins-via-pruning-symmetric-hash-joins">Synnada
+Blog Post</a> on the topic). Together, these features both 1) make it
+easier to build streaming systems using DataFusion that can
+incrementally generate output before (or ever) seeing the end of the
+input and 2) allow general queries to use less memory and generate their
+results faster.</p>
+
+<p>We have also improved the runtime <a href="https://docs.rs/datafusion/latest/datafusion/execution/memory_pool/index.html">memory management</a> system so that
+DataFusion now stays within its declared memory budget <a href="https://github.com/apache/arrow-datafusion/issues/3941">generate
+runtime errors</a>.</p>
+
+<h3 id="dml-support-insert-delete-update-etc">DML Support (<code class="language-plaintext highlighter-rouge">INSERT</code>, <code class="language-plaintext highlighter-rouge">DELETE</code>, <code class="language-plaintext highlighter-rouge">UPDATE</code>, etc)</h3>
+
+<p>Part of building high performance data systems includes writing data,
+and DataFusion supports several features for creating new files:</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">INSERT INTO</code> and <code class="language-plaintext highlighter-rouge">SELECT ... INTO </code> support for memory backed and CSV tables</li>
+ <li>New <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/insert/trait.DataSink.html">API for writing data into TableProviders</a></li>
+</ul>
+
+<p>We are working on easier to use <a href="https://github.com/apache/arrow-datafusion/issues/5654">COPY INTO</a> syntax, better support
+for writing parquet, JSON, and AVRO, and more – see our <a href="https://github.com/apache/arrow-datafusion/issues/6569">tracking epic</a>
+for more details.</p>
+
+<h3 id="timestamp-and-intervals">Timestamp and Intervals</h3>
+
+<p>One mark of the maturity of a SQL engine is how it handles the tricky
+world of timestamp, date, times and interval arithmetic. DataFusion is
+feature complete in this area and behaves as you would expect,
+supporting queries such as</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="n">now</span><span class="p">()</span> <span class="o">+</span> <span class="s1">'1 month'</span> <span class="k">FROM</span> <span class="n">my_table</span><span class="p">;</span>
+</code></pre></div></div>
+
+<p>We still have a long tail of <a href="https://github.com/apache/arrow-datafusion/issues/3148">date and time improvements</a>, which we are working on as well.</p>
+
+<h3 id="querying-structured-types-list-and-structs">Querying Structured Types (<code class="language-plaintext highlighter-rouge">List</code> and <code class="language-plaintext highlighter-rouge">Struct</code>s)</h3>
+
+<p>Arrow and Parquet <a href="https://arrow.apache.org/blog/2022/10/08/arrow-parquet-encoding-part-2/">support nested data</a> well and DataFusion lets you
+easily query such <code class="language-plaintext highlighter-rouge">Struct</code> and <code class="language-plaintext highlighter-rouge">List</code>. For example, you can use
+DataFusion to read and query the <a href="https://data.mendeley.com/datasets/ct8f9skv97">JSON Datasets for Exploratory OLAP -
+Mendeley Data</a> like this:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c1">----------</span>
+<span class="c1">-- Explore structured data using SQL</span>
+<span class="c1">----------</span>
+<span class="k">SELECT</span> <span class="k">delete</span> <span class="k">FROM</span> <span class="s1">'twitter-sample-head-100000.parquet'</span> <span class="k">WHERE</span> <span class="k">delete</span> <span class="k">IS</span> <span class="k">NOT</span> <span class="k">NULL</span> <span class="k">limit</span> <span class="mi">10</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">---------------------------------------------------------------------------------------------------------------------------+</span>
+<span class="o">|</span> <span class="k">delete</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">---------------------------------------------------------------------------------------------------------------------------+</span>
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">135037425050320896</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">135037425050320896</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134703982051463168</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134703982051463168</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134773741740765184</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134773741740765184</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">132543659655704576</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">132543659655704576</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">133786431926697984</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">133786431926697984</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134619093570560002</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134619093570560002</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134019857527214080</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134019857527214080</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">133931546469076993</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">133931546469076993</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134397743350296576</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134397743350296576</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">127833661767823360</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">127833661767823360</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">+</span><span class="c1">---------------------------------------------------------------------------------------------------------------------------+</span>
+
+<span class="c1">----------</span>
+<span class="c1">-- Select some deeply nested fields</span>
+<span class="c1">----------</span>
+<span class="k">SELECT</span>
+ <span class="k">delete</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'id'</span><span class="p">][</span><span class="s1">'$numberLong'</span><span class="p">]</span> <span class="k">as</span> <span class="n">delete_id</span><span class="p">,</span>
+ <span class="k">delete</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'user_id'</span><span class="p">]</span> <span class="k">as</span> <span class="n">delete_user_id</span>
+<span class="k">FROM</span> <span class="s1">'twitter-sample-head-100000.parquet'</span> <span class="k">WHERE</span> <span class="k">delete</span> <span class="k">IS</span> <span class="k">NOT</span> <span class="k">NULL</span> <span class="k">LIMIT</span> <span class="mi">10</span><span class="p">;</span>
+
+<span class="o">+</span><span class="c1">--------------------+----------------+</span>
+<span class="o">|</span> <span class="n">delete_id</span> <span class="o">|</span> <span class="n">delete_user_id</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">--------------------+----------------+</span>
+<span class="o">|</span> <span class="mi">135037425050320896</span> <span class="o">|</span> <span class="mi">334902461</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134703982051463168</span> <span class="o">|</span> <span class="mi">405383453</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134773741740765184</span> <span class="o">|</span> <span class="mi">64823441</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">132543659655704576</span> <span class="o">|</span> <span class="mi">45917834</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">133786431926697984</span> <span class="o">|</span> <span class="mi">67229952</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134619093570560002</span> <span class="o">|</span> <span class="mi">182430773</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134019857527214080</span> <span class="o">|</span> <span class="mi">257396311</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">133931546469076993</span> <span class="o">|</span> <span class="mi">124539548</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134397743350296576</span> <span class="o">|</span> <span class="mi">139836391</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">127833661767823360</span> <span class="o">|</span> <span class="mi">244442687</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">--------------------+----------------+</span>
+</code></pre></div></div>
+
+<h3 id="subqueries-all-the-way-down">Subqueries All the Way Down</h3>
+
+<p>DataFusion can run many different subqueries by rewriting them to
+joins. It has been able to run the full suite of TPC-H queries for at
+least the last year, but recently we have implemented significant
+improvements to this logic, sufficient to run almost all queries in
+the TPC-DS benchmark as well.</p>
+
+<h2 id="community-and-project-growth">Community and Project Growth</h2>
+
+<p>The six months since <a href="https://arrow.apache.org/blog/2023/01/19/datafusion-16.0.0">our last update</a> saw significant growth in
+the DataFusion community. Between versions <code class="language-plaintext highlighter-rouge">17.0.0</code> and <code class="language-plaintext highlighter-rouge">26.0.0</code>,
+DataFusion merged 711 PRs from 107 distinct contributors, not
+including all the work that goes into our core dependencies such as
+<a href="https://crates.io/crates/arrow">arrow</a>,
+<a href="https://crates.io/crates/parquet">parquet</a>, and
+<a href="https://crates.io/crates/object_store">object_store</a>, that much of
+the same community helps support.</p>
+
+<p>In addition, we have added 7 new committers and 1 new PMC member to
+the Apache Arrow project, largely focused on DataFusion, and we
+learned about some of the cool <a href="https://arrow.apache.org/datafusion/user-guide/introduction.html#known-users">new systems</a> which are using
+DataFusion. Given the growth of the community and interest in the
+project, we also clarified the <a href="https://github.com/apache/arrow-datafusion/discussions/6441">mission statement</a> and are
+<a href="https://github.com/apache/arrow-datafusion/discussions/6475">discussing</a> “graduate”ing DataFusion to a new top level
+Apache Software Foundation project.</p>
+
+<!--
+$ git log --pretty=oneline 17.0.0..26.0.0 . | wc -l
+ 711
+
+$ git shortlog -sn 17.0.0..26.0.0 . | wc -l
+ 107
+-->
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>Kudos to everyone in the community who has contributed ideas,
+discussions, bug reports, documentation and code. It is exciting to be
+innovating on the next generation of database architectures together!</p>
+
+<p>If you are interested in contributing to DataFusion, we would love to
+have you join us. You can try out DataFusion on some of your own
+data and projects and let us know how it goes or contribute a PR with
+documentation, tests or code. A list of open issues suitable for
+beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>Check out our <a href="https://arrow.apache.org/datafusion/contributor-guide/communication.html">Communication Doc</a> for more ways to engage with the
+community.</p>
+
+
+ </div><a class="u-url" href="/blog/blog/2023/06/24/datafusion-25.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2023/08/05/datafusion_fast_grouping/index.html b/blog/2023/08/05/datafusion_fast_grouping/index.html
new file mode 100644
index 0000000..7e50e4f
--- /dev/null
+++ b/blog/2023/08/05/datafusion_fast_grouping/index.html
@@ -0,0 +1,460 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0 | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0" />
+<meta name="author" content="alamb, Dandandan, tustvold" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2023/08/05/datafusion_fast_grouping/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2023/08/05/datafusion_fast_grouping/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2023-08-05T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"alamb, Dandandan, tustvold"},"dateModified":"2023-08-05T00:00:00-06:00","datePublished":"2023-08-05T00:00:00-06:00","description":"<!–","headline":"Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2023/08/05/datafusion_fast_grouping/"},"publisher":{"@type":"Organization","logo":{"@type":"Ima [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2023-08-05T00:00:00-06:00" itemprop="datePublished">Aug 5, 2023
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">alamb, Dandandan, tustvold</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<!-- Converted from Google Docs using https://www.buymeacoffee.com/docstomarkdown -->
+
+<h2 id="aggregating-millions-of-groups-fast-in-apache-arrow-datafusion">Aggregating Millions of Groups Fast in Apache Arrow DataFusion</h2>
+
+<p>Andrew Lamb, Daniël Heres, Raphael Taylor-Davies,</p>
+
+<p><em>Note: this article was originally published on the <a href="https://www.influxdata.com/blog/aggregating-millions-groups-fast-apache-arrow-datafusion">InfluxData Blog</a></em></p>
+
+<h2 id="tldr">TLDR</h2>
+
+<p>Grouped aggregations are a core part of any analytic tool, creating understandable summaries of huge data volumes. <a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a>’s parallel aggregation capability is 2-3x faster in the <a href="https://crates.io/crates/datafusion/28.0.0">newly released version <code class="language-plaintext highlighter-rouge">28.0.0</code></a> for queries with a large number (10,000 or more) of groups.</p>
+
+<p>Improving aggregation performance matters to all users of DataFusion. For example, both InfluxDB, a <a href="https://github.com/influxdata/influxdb">time series data platform</a> and Coralogix, a <a href="https://coralogix.com/?utm_source=InfluxDB&utm_medium=Blog&utm_campaign=organic">full-stack observability</a> platform, aggregate vast amounts of raw data to monitor and create insights for our customers. Improving DataFusion’s performance lets us provide better user experien [...]
+
+<p>With the new optimizations, DataFusion’s grouping speed is now close to DuckDB, a system that regularly reports <a href="https://duckdblabs.github.io/db-benchmark/">great</a> <a href="https://duckdb.org/2022/03/07/aggregate-hashtable.html#experiments">grouping</a> benchmark performance numbers. Figure 1 contains a representative sample of <a href="https://github.com/ClickHouse/ClickBench/tree/main">ClickBench</a> on a single Parquet file, and the full results are at the end of this ar [...]
+
+<p><img src="/blog/assets/datafusion_fast_grouping/summary.png" width="700" /></p>
+
+<p><strong>Figure 1</strong>: Query performance for ClickBench queries on queries 16, 17, 18 and 19 on a single Parquet file for DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code>.</p>
+
+<h2 id="introduction-to-high-cardinality-grouping">Introduction to high cardinality grouping</h2>
+
+<p>Aggregation is a fancy word for computing summary statistics across many rows that have the same value in one or more columns. We call the rows with the same values <em>groups</em> and “high cardinality” means there are a large number of distinct groups in the dataset. At the time of writing, a “large” number of groups in analytic engines is around 10,000.</p>
+
+<p>For example the <a href="https://github.com/ClickHouse/ClickBench">ClickBench</a> <em>hits</em> dataset contains 100 million anonymized user clicks across a set of websites. ClickBench Query 17 is:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="nv">"UserID"</span><span class="p">,</span> <span class="nv">"SearchPhrase"</span><span class="p">,</span> <span class="k">COUNT</span><span class="p">(</span><span class="o">*</span><span class="p">)</span>
+<span class="k">FROM</span> <span class="n">hits</span>
+<span class="k">GROUP</span> <span class="k">BY</span> <span class="nv">"UserID"</span><span class="p">,</span> <span class="nv">"SearchPhrase"</span>
+<span class="k">ORDER</span> <span class="k">BY</span> <span class="k">COUNT</span><span class="p">(</span><span class="o">*</span><span class="p">)</span>
+<span class="k">DESC</span> <span class="k">LIMIT</span> <span class="mi">10</span><span class="p">;</span>
+</code></pre></div></div>
+
+<p>In English, this query finds “the top ten (user, search phrase) combinations, across all clicks” and produces the following results (there are no search phrases for the top ten users):</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+---------------------+--------------+-----------------+
+| UserID | SearchPhrase | COUNT(UInt8(1)) |
++---------------------+--------------+-----------------+
+| 1313338681122956954 | | 29097 |
+| 1907779576417363396 | | 25333 |
+| 2305303682471783379 | | 10597 |
+| 7982623143712728547 | | 6669 |
+| 7280399273658728997 | | 6408 |
+| 1090981537032625727 | | 6196 |
+| 5730251990344211405 | | 6019 |
+| 6018350421959114808 | | 5990 |
+| 835157184735512989 | | 5209 |
+| 770542365400669095 | | 4906 |
++---------------------+--------------+-----------------+
+</code></pre></div></div>
+
+<p>The ClickBench dataset contains</p>
+
+<ul>
+ <li>99,997,497 total rows<sup id="fnref:1" role="doc-noteref"><a href="#fn:1" class="footnote" rel="footnote">1</a></sup></li>
+ <li>17,630,976 different users (distinct UserIDs)<sup id="fnref:2" role="doc-noteref"><a href="#fn:2" class="footnote" rel="footnote">2</a></sup></li>
+ <li>6,019,103 different search phrases<sup id="fnref:3" role="doc-noteref"><a href="#fn:3" class="footnote" rel="footnote">3</a></sup></li>
+ <li>24,070,560 distinct combinations<sup id="fnref:4" role="doc-noteref"><a href="#fn:4" class="footnote" rel="footnote">4</a></sup> of (UserID, SearchPhrase)
+Thus, to answer the query, DataFusion must map each of the 100M different input rows into one of the <strong>24 million different groups</strong>, and keep count of how many such rows there are in each group.</li>
+</ul>
+
+<h2 id="the-solution">The solution</h2>
+
+<p>Like most concepts in databases and other analytic systems, the basic ideas of this algorithm are straightforward and taught in introductory computer science courses. You could compute the query with a program such as this<sup id="fnref:5" role="doc-noteref"><a href="#fn:5" class="footnote" rel="footnote">5</a></sup>:</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kn">import</span> <span class="n">pandas</span> <span class="k">as</span> <span class="n">pd</span>
+<span class="kn">from</span> <span class="n">collections</span> <span class="kn">import</span> <span class="n">defaultdict</span>
+<span class="kn">from</span> <span class="n">operator</span> <span class="kn">import</span> <span class="n">itemgetter</span>
+
+<span class="c1"># read file
+</span><span class="n">hits</span> <span class="o">=</span> <span class="n">pd</span><span class="p">.</span><span class="nf">read_parquet</span><span class="p">(</span><span class="sh">'</span><span class="s">hits.parquet</span><span class="sh">'</span><span class="p">,</span> <span class="n">engine</span><span class="o">=</span><span class="sh">'</span><span class="s">pyarrow</span><span class="sh">'</span><span class="p">)</span>
+
+<span class="c1"># build groups
+</span><span class="n">counts</span> <span class="o">=</span> <span class="nf">defaultdict</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
+<span class="k">for</span> <span class="n">index</span><span class="p">,</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">hits</span><span class="p">.</span><span class="nf">iterrows</span><span class="p">():</span>
+ <span class="n">group</span> <span class="o">=</span> <span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="sh">'</span><span class="s">UserID</span><span class="sh">'</span><span class="p">],</span> <span class="n">row</span><span class="p">[</span><span class="sh">'</span><span class="s">SearchPhrase</span><span class="sh">'</span><span class="p">]);</span>
+ <span class="c1"># update the dict entry for the corresponding key
+</span> <span class="n">counts</span><span class="p">[</span><span class="n">group</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
+
+<span class="c1"># Print the top 10 values
+</span><span class="nf">print </span><span class="p">(</span><span class="nf">dict</span><span class="p">(</span><span class="nf">sorted</span><span class="p">(</span><span class="n">counts</span><span class="p">.</span><span class="nf">items</span><span class="p">(),</span> <span class="n">key</span><span class="o">=</span><span class="nf">itemgetter</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">reverse</span><span class="o">=</span><sp [...]
+</code></pre></div></div>
+
+<p>This approach, while simple, is both slow and very memory inefficient. It requires over 40 seconds to compute the results for less than 1% of the dataset<sup id="fnref:6" role="doc-noteref"><a href="#fn:6" class="footnote" rel="footnote">6</a></sup>. Both DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code> compute results in under 10 seconds for the <em>entire</em> dataset.</p>
+
+<p>To answer this query quickly and efficiently, you have to write your code such that it:</p>
+
+<ol>
+ <li>Keeps all cores busy aggregating via parallelized computation</li>
+ <li>Updates aggregate values quickly, using vectorizable loops that are easy for compilers to translate into the high performance <a href="https://en.wikipedia.org/wiki/Single_instruction,_multiple_data">SIMD</a> instructions available in modern CPUs.</li>
+</ol>
+
+<p>The rest of this article explains how grouping works in DataFusion and the improvements we made in <code class="language-plaintext highlighter-rouge">28.0.0</code>.</p>
+
+<h3 id="two-phase-parallel-partitioned-grouping">Two phase parallel partitioned grouping</h3>
+
+<p>Both DataFusion <code class="language-plaintext highlighter-rouge">27.0.</code> and <code class="language-plaintext highlighter-rouge">28.0.0</code> use state-of-the-art, two phase parallel hash partitioned grouping, similar to other high-performance vectorized engines like <a href="https://duckdb.org/2022/03/07/aggregate-hashtable.html">DuckDB’s Parallel Grouped Aggregates</a>. In pictures this looks like:</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ▲ ▲
+ │ │
+ │ │
+ │ │
+┌───────────────────────┐ ┌───────────────────┐
+│ GroupBy │ │ GroupBy │ Step 4
+│ (Final) │ │ (Final) │
+└───────────────────────┘ └───────────────────┘
+ ▲ ▲
+ │ │
+ └────────────┬───────────┘
+ │
+ │
+ ┌─────────────────────────┐
+ │ Repartition │ Step 3
+ │ HASH(x) │
+ └─────────────────────────┘
+ ▲
+ │
+ ┌────────────┴──────────┐
+ │ │
+ │ │
+ ┌────────────────────┐ ┌─────────────────────┐
+ │ GroupyBy │ │ GroupBy │ Step 2
+ │ (Partial) │ │ (Partial) │
+ └────────────────────┘ └─────────────────────┘
+ ▲ ▲
+ ┌──┘ └─┐
+ │ │
+ .─────────. .─────────.
+ ,─' '─. ,─' '─.
+; Input : ; Input : Step 1
+: Stream 1 ; : Stream 2 ;
+ ╲ ╱ ╲ ╱
+ '─. ,─' '─. ,─'
+ `───────' `───────'
+</code></pre></div></div>
+
+<p><strong>Figure 2</strong>: Two phase repartitioned grouping: data flows from bottom (source) to top (results) in two phases. First (Steps 1 and 2), each core reads the data into a core-specific hash table, computing intermediate aggregates without any cross-core coordination. Then (Steps 3 and 4) DataFusion divides the data (“repartitions”) into distinct subsets by group value, and each subset is sent to a specific core which computes the final aggregate.</p>
+
+<p>The two phases are critical for keeping cores busy in a multi-core system. Both phases use the same hash table approach (explained in the next section), but differ in how the groups are distributed and the partial results emitted from the accumulators. The first phase aggregates data as soon as possible after it is produced. However, as shown in Figure 2, the groups can be anywhere in any input, so the same group is often found on many different cores. The second phase uses a hash fun [...]
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌─────┐ ┌─────┐
+ │ 1 │ │ 3 │
+ │ 2 │ │ 4 │ 2. After Repartitioning: each
+ └─────┘ └─────┘ group key appears in exactly
+ ┌─────┐ ┌─────┐ one partition
+ │ 1 │ │ 3 │
+ │ 2 │ │ 4 │
+ └─────┘ └─────┘
+
+─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
+
+ ┌─────┐ ┌─────┐
+ │ 2 │ │ 2 │
+ │ 1 │ │ 2 │
+ │ 3 │ │ 3 │
+ │ 4 │ │ 1 │
+ └─────┘ └─────┘ 1. Input Stream: groups
+ ... ... values are spread
+ ┌─────┐ ┌─────┐ arbitrarily over each input
+ │ 1 │ │ 4 │
+ │ 4 │ │ 3 │
+ │ 1 │ │ 1 │
+ │ 4 │ │ 3 │
+ │ 3 │ │ 2 │
+ │ 2 │ │ 2 │
+ │ 2 │ └─────┘
+ └─────┘
+
+ Core A Core B
+
+</code></pre></div></div>
+
+<p><strong>Figure 3</strong>: Group value distribution across 2 cores during aggregation phases. In the first phase, every group value <code class="language-plaintext highlighter-rouge">1</code>, <code class="language-plaintext highlighter-rouge">2</code>, <code class="language-plaintext highlighter-rouge">3</code>, <code class="language-plaintext highlighter-rouge">4</code>, is present in the input stream processed by each core. In the second phase, after repartitioning, the group value [...]
+
+<p>There are some additional subtleties in the <a href="https://github.com/apache/arrow-datafusion/blob/main/datafusion/core/src/physical_plan/aggregates/row_hash.rs">DataFusion implementation</a> not mentioned above due to space constraints, such as:</p>
+
+<ol>
+ <li>The policy of when to emit data from the first phase’s hash table (e.g. because the data is partially sorted)</li>
+ <li>Handling specific filters per aggregate (due to the <code class="language-plaintext highlighter-rouge">FILTER</code> SQL clause)</li>
+ <li>Data types of intermediate values (which may not be the same as the final output for some aggregates such as <code class="language-plaintext highlighter-rouge">AVG</code>).</li>
+ <li>Action taken when memory use exceeds its budget.</li>
+</ol>
+
+<h3 id="hash-grouping">Hash grouping</h3>
+
+<p>DataFusion queries can compute many different aggregate functions for each group, both <a href="https://arrow.apache.org/datafusion/user-guide/sql/aggregate_functions.html">built in</a> and/or user defined <a href="https://docs.rs/datafusion/latest/datafusion/logical_expr/struct.AggregateUDF.html"><code class="language-plaintext highlighter-rouge">AggregateUDFs</code></a>. The state for each aggregate function, called an <em>accumulator</em>, is tracked with a hash table (DataFusion u [...]
+
+<h3 id="hash-grouping-in-2700">Hash grouping in <code class="language-plaintext highlighter-rouge">27.0.0</code></h3>
+
+<p>As shown in Figure 3, DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code> stores the data in a <a href="https://github.com/apache/arrow-datafusion/blob/4d93b6a3802151865b68967bdc4c7d7ef425b49a/datafusion/core/src/physical_plan/aggregates/utils.rs#L38-L50"><code class="language-plaintext highlighter-rouge">GroupState</code></a> structure which, unsurprisingly, tracks the state for each group. The state for each group consists of:</p>
+
+<ol>
+ <li>The actual value of the group columns, in <a href="https://docs.rs/arrow-row/latest/arrow_row/index.html">Arrow Row</a> format.</li>
+ <li>In-progress accumulations (e.g. the running counts for the <code class="language-plaintext highlighter-rouge">COUNT</code> aggregate) for each group, in one of two possible formats (<a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/expr/src/accumulator.rs#L24-L49"><code class="language-plaintext highlighter-rouge">Accumulator</code></a> or <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b42 [...]
+ <li>Scratch space for tracking which rows match each aggregate in each batch.</li>
+</ol>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌──────────────────────────────────────┐
+ │ │
+ │ ... │
+ │ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ │
+ │ ┃ ┃ │
+ ┌─────────┐ │ ┃ ┌──────────────────────────────┐ ┃ │
+ │ │ │ ┃ │group values: OwnedRow │ ┃ │
+ │ ┌─────┐ │ │ ┃ └──────────────────────────────┘ ┃ │
+ │ │ 5 │ │ │ ┃ ┌──────────────────────────────┐ ┃ │
+ │ ├─────┤ │ │ ┃ │Row accumulator: │ ┃ │
+ │ │ 9 │─┼────┐ │ ┃ │Vec<u8> │ ┃ │
+ │ ├─────┤ │ │ │ ┃ └──────────────────────────────┘ ┃ │
+ │ │ ... │ │ │ │ ┃ ┌──────────────────────┐ ┃ │
+ │ ├─────┤ │ │ │ ┃ │┌──────────────┐ │ ┃ │
+ │ │ 1 │ │ │ │ ┃ ││Accumulator 1 │ │ ┃ │
+ │ ├─────┤ │ │ │ ┃ │└──────────────┘ │ ┃ │
+ │ │ ... │ │ │ │ ┃ │┌──────────────┐ │ ┃ │
+ │ └─────┘ │ │ │ ┃ ││Accumulator 2 │ │ ┃ │
+ │ │ │ │ ┃ │└──────────────┘ │ ┃ │
+ └─────────┘ │ │ ┃ │ Box<dyn Accumulator> │ ┃ │
+ Hash Table │ │ ┃ └──────────────────────┘ ┃ │
+ │ │ ┃ ┌─────────────────────────┐ ┃ │
+ │ │ ┃ │scratch indices: Vec<u32>│ ┃ │
+ │ │ ┃ └─────────────────────────┘ ┃ │
+ │ │ ┃ GroupState ┃ │
+ └─────▶ │ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ │
+ │ │
+ Hash table tracks an │ ... │
+ index into group_states │ │
+ └──────────────────────────────────────┘
+ group_states: Vec<GroupState>
+
+ There is one GroupState PER GROUP
+
+</code></pre></div></div>
+
+<p><strong>Figure 4</strong>: Hash group operator structure in DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>. A hash table maps each group to a GroupState which contains all the per-group states.</p>
+
+<p>To compute the aggregate, DataFusion performs the following steps for each input batch:</p>
+
+<ol>
+ <li>Calculate hash using <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/physical-expr/src/hash_utils.rs#L264-L307">efficient vectorized code</a>, specialized for each data type.</li>
+ <li>Determine group indexes for each input row using the hash table (creating new entries for newly seen groups).</li>
+ <li><a href="https://github.com/apache/arrow-datafusion/blob/4ab8be57dee3bfa72dd105fbd7b8901b873a4878/datafusion/core/src/physical_plan/aggregates/row_hash.rs#L562-L602">Update Accumulators for each group that had input rows,</a> assembling the rows into a contiguous range for vectorized accumulator if there are a sufficient number of them.</li>
+</ol>
+
+<p>DataFusion also stores the hash values in the table to avoid potentially costly hash recomputation when resizing the hash table.</p>
+
+<p>This scheme works very well for a relatively small number of distinct groups: all accumulators are efficiently updated with large contiguous batches of rows.</p>
+
+<p>However, this scheme is not ideal for high cardinality grouping due to:</p>
+
+<ol>
+ <li><strong>Multiple allocations per group</strong> for the group value row format, as well as for the <code class="language-plaintext highlighter-rouge">RowAccumulator</code>s and each <code class="language-plaintext highlighter-rouge">Accumulator</code>. The <code class="language-plaintext highlighter-rouge">Accumulator</code> may have additional allocations within it as well.</li>
+ <li><strong>Non-vectorized updates:</strong> Accumulator updates often fall back to a slower non-vectorized form because the number of distinct groups is large (and thus number of values per group is small) in each input batch.</li>
+</ol>
+
+<h3 id="hash-grouping-in-2800">Hash grouping in <code class="language-plaintext highlighter-rouge">28.0.0</code></h3>
+
+<p>For <code class="language-plaintext highlighter-rouge">28.0.0</code>, we rewrote the core group by implementation following traditional system optimization principles: fewer allocations, type specialization, and aggressive vectorization.</p>
+
+<p>DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> uses the same RawTable and still stores group indexes. The major differences, as shown in Figure 4, are:</p>
+
+<ol>
+ <li>Group values are stored either
+ <ol>
+ <li>Inline in the <code class="language-plaintext highlighter-rouge">RawTable</code> (for single columns of primitive types), where the conversion to Row format costs more than its benefit</li>
+ <li>In a separate <a href="https://docs.rs/arrow-row/latest/arrow_row/struct.Row.html">Rows</a> structure with a single contiguous allocation for all groups values, rather than an allocation per group. Accumulators manage the state for all the groups internally, so the code to update intermediate values is a tight type specialized loop. The new <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/physical-expr/src/aggregate/gr [...]
+ </ol>
+ </li>
+</ol>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>┌───────────────────────────────────┐ ┌───────────────────────┐
+│ ┌ ─ ─ ─ ─ ─ ┐ ┌─────────────────┐│ │ ┏━━━━━━━━━━━━━━━━━━━┓ │
+│ │ ││ │ ┃ ┌──────────────┐ ┃ │
+│ │ │ │ ┌ ─ ─ ┐┌─────┐ ││ │ ┃ │┌───────────┐ │ ┃ │
+│ │ X │ 5 │ ││ │ ┃ ││ value1 │ │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ │ ┃ │└───────────┘ │ ┃ │
+│ │ Q │ 9 │──┼┼──┐ │ ┃ │ ... │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ └──┼─╋─▶│ │ ┃ │
+│ │ ... │ ... │ ││ │ ┃ │┌───────────┐ │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ │ ┃ ││ valueN │ │ ┃ │
+│ │ H │ 1 │ ││ │ ┃ │└───────────┘ │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ │ ┃ │values: Vec<T>│ ┃ │
+│ Rows │ ... │ ... │ ││ │ ┃ └──────────────┘ ┃ │
+│ │ │ │ └ ─ ─ ┘└─────┘ ││ │ ┃ ┃ │
+│ ─ ─ ─ ─ ─ ─ │ ││ │ ┃ GroupsAccumulator ┃ │
+│ └─────────────────┘│ │ ┗━━━━━━━━━━━━━━━━━━━┛ │
+│ Hash Table │ │ │
+│ │ │ ... │
+└───────────────────────────────────┘ └───────────────────────┘
+ GroupState Accumulators
+
+
+Hash table value stores group_indexes One GroupsAccumulator
+and group values. per aggregate. Each
+ stores the state for
+Group values are stored either inline *ALL* groups, typically
+in the hash table or in a single using a native Vec<T>
+allocation using the arrow Row format
+</code></pre></div></div>
+
+<p><strong>Figure 5</strong>: Hash group operator structure in DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code>. Group values are stored either directly in the hash table, or in a single allocation using the arrow Row format. The hash table contains group indexes. A single <code class="language-plaintext highlighter-rouge">GroupsAccumulator</code> stores the per-aggregate state for <em>all</em> groups.</p>
+
+<p>This new structure improves performance significantly for high cardinality groups due to:</p>
+
+<ol>
+ <li><strong>Reduced allocations</strong>: There are no longer any individual allocations per group.</li>
+ <li><strong>Contiguous native accumulator states</strong>: Type-specialized accumulators store the values for all groups in a single contiguous allocation using a <a href="https://doc.rust-lang.org/std/vec/struct.Vec.html">Rust Vec<T></a> of some native type.</li>
+ <li><strong>Vectorized state update</strong>: The inner aggregate update loops, which are type-specialized and in terms of native <code class="language-plaintext highlighter-rouge">Vec</code>s, are well-vectorized by the Rust compiler (thanks <a href="https://llvm.org/">LLVM</a>!).</li>
+</ol>
+
+<h3 id="notes">Notes</h3>
+
+<p>Some vectorized grouping implementations store the accumulator state row-wise directly in the hash table, which often uses modern CPU caches efficiently. Managing accumulator state in columnar fashion may sacrifice some cache locality, however it ensures the size of the hash table remains small, even when there are large numbers of groups and aggregates, making it easier for the compiler to vectorize the accumulator update.</p>
+
+<p>Depending on the cost of recomputing hash values, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> may or may not store the hash values in the table. This optimizes the tradeoff between the cost of computing the hash value (which is expensive for strings, for example) vs. the cost of storing it in the hash table.</p>
+
+<p>One subtlety that arises from pushing state updates into GroupsAccumulators is that each accumulator must handle similar variations with/without filtering and with/without nulls in the input. DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> uses a templated <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/physical-expr/src/aggregate/groups_accumulator/accumulate.rs#L28-L54"><code class="language-pla [...]
+
+<p>The code structure is heavily influenced by the fact DataFusion is implemented using <a href="https://www.rust-lang.org/">Rust</a>, a new(ish) systems programming language focused on speed and safety. Rust heavily discourages many of the traditional pointer casting “tricks” used in C/C++ hash grouping implementations. The DataFusion aggregation code is almost entirely <a href="https://doc.rust-lang.org/nomicon/meet-safe-and-unsafe.html#:~:text=Safe%20Rust%20is%20the%20true,Undefined%2 [...]
+
+<h2 id="clickbench-results">ClickBench results</h2>
+
+<p>The full results of running the <a href="https://github.com/ClickHouse/ClickBench/tree/main">ClickBench</a> queries against the single Parquet file with DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code>, and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code> are below. These numbers were run on a GCP <code class="language-plaintext highlighter-rouge">e2-standard-8 [...]
+
+<p>As the industry moves towards data systems assembled from components, it is increasingly important that they exchange data using open standards such as <a href="https://arrow.apache.org/">Apache Arrow</a> and <a href="https://parquet.apache.org/">Parquet</a> rather than custom storage and in-memory formats. Thus, this benchmark uses a single input Parquet file representative of many DataFusion users and aligned with the current trend in analytics of avoiding a costly load/transformati [...]
+
+<p>DataFusion now reaches near-DuckDB-speeds querying Parquet data. While we don’t plan to engage in a benchmarking shootout with a team that literally wrote <a href="https://dl.acm.org/doi/abs/10.1145/3209950.3209955">Fair Benchmarking Considered Difficult</a>, hopefully everyone can agree that DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> is a significant improvement.</p>
+
+<p><img src="/blog/assets/datafusion_fast_grouping/full.png" width="700" /></p>
+
+<p><strong>Figure 6</strong>: Performance of DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code>, and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code> on all 43 ClickBench queries against a single <code class="language-plaintext highlighter-rouge">hits.parquet</code> file. Lower is better.</p>
+
+<h3 id="notes-1">Notes</h3>
+
+<p>DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code> was not able to run several queries due to either planner bugs (Q9, Q11, Q12, 14) or running out of memory (Q33). DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> solves those issues.</p>
+
+<p>DataFusion is faster than DuckDB for query 21 and 22, likely due to optimized implementations of string pattern matching.</p>
+
+<h2 id="conclusion-performance-matters">Conclusion: performance matters</h2>
+
+<p>Improving aggregation performance by more than a factor of two allows developers building products and projects with DataFusion to spend more time on value-added domain specific features. We believe building systems with DataFusion is much faster than trying to build something similar from scratch. DataFusion increases productivity because it eliminates the need to rebuild well-understood, but costly to implement, analytic database technology. While we’re pleased with the improvements [...]
+
+<h2 id="acknowledgments">Acknowledgments</h2>
+
+<p>DataFusion is a <a href="https://arrow.apache.org/datafusion/contributor-guide/communication.html">community effort</a> and this work was not possible without contributions from many in the community. A special shout out to <a href="https://github.com/sunchao">sunchao</a>, <a href="https://github.com/jyshen">yjshen</a>, <a href="https://github.com/yahoNanJing">yahoNanJing</a>, <a href="https://github.com/mingmwang">mingmwang</a>, <a href="https://github.com/ozankabak">ozankabak</a>, < [...]
+
+<h2 id="about-datafusion">About DataFusion</h2>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> is an extensible query engine and database toolkit, written in <a href="https://www.rust-lang.org/">Rust</a>, that uses <a href="https://arrow.apache.org/">Apache Arrow</a> as its in-memory format. DataFusion, along with <a href="https://calcite.apache.org/">Apache Calcite</a>, Facebook’s <a href="https://github.com/facebookincubator/velox">Velox</a>, and similar technology are part of the next generation “<a h [...]
+
+<!-- Footnotes themselves at the bottom. -->
+<h2 id="notes-2">Notes</h2>
+
+<div class="footnotes" role="doc-endnotes">
+ <ol>
+ <li id="fn:1" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(*) FROM 'hits.parquet';</code> <a href="#fnref:1" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:2" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(DISTINCT "UserID") as num_users FROM 'hits.parquet';</code> <a href="#fnref:2" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:3" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(DISTINCT "SearchPhrase") as num_phrases FROM 'hits.parquet';</code> <a href="#fnref:3" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:4" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(*) FROM (SELECT DISTINCT "UserID", "SearchPhrase" FROM 'hits.parquet')</code> <a href="#fnref:4" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:5" role="doc-endnote">
+ <p>Full script at <a href="https://github.com/alamb/datafusion-duckdb-benchmark/blob/main/hash.py">hash.py</a> <a href="#fnref:5" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:6" role="doc-endnote">
+ <p><a href="https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_%7B%7D.parquet">hits_0.parquet</a>, one of the files from the partitioned ClickBench dataset, which has <code class="language-plaintext highlighter-rouge">100,000</code> rows and is 117 MB in size. The entire dataset has <code class="language-plaintext highlighter-rouge">100,000,000</code> rows in a single 14 GB Parquet file. The script did not complete on the entire dataset after 40 minutes, and us [...]
+ </li>
+ </ol>
+</div>
+
+ </div><a class="u-url" href="/blog/blog/2023/08/05/datafusion_fast_grouping/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2024/01/19/datafusion-34.0.0/index.html b/blog/2024/01/19/datafusion-34.0.0/index.html
new file mode 100644
index 0000000..1383382
--- /dev/null
+++ b/blog/2024/01/19/datafusion-34.0.0/index.html
@@ -0,0 +1,358 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024 | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2024/01/19/datafusion-34.0.0/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2024/01/19/datafusion-34.0.0/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2024-01-19T00:00:00-07:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2024-01-19T00:00:00-07:00","datePublished":"2024-01-19T00:00:00-07:00","description":"<!–","headline":"Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2024/01/19/datafusion-34.0.0/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2024-01-19T00:00:00-07:00" itemprop="datePublished">Jan 19, 2024
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>We recently <a href="https://crates.io/crates/datafusion/34.0.0">released DataFusion 34.0.0</a>. This blog highlights some of the major
+improvements since we <a href="https://arrow.apache.org/blog/2023/06/24/datafusion-25.0.0/.">released DataFusion 26.0.0</a> (spoiler alert there are many)
+and a preview of where the community plans to focus in the next 6 months.</p>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> is an extensible query engine, written in <a href="https://www.rust-lang.org/">Rust</a>, that
+uses <a href="https://arrow.apache.org">Apache Arrow</a> as its in-memory format. DataFusion is used by developers to
+create new, fast data centric systems such as databases, dataframe libraries,
+machine learning and streaming applications. While <a href="https://arrow.apache.org/datafusion/user-guide/introduction.html#project-goals">DataFusion’s primary design
+goal</a> is to accelerate creating other data centric systems, it has a
+reasonable experience directly out of the box as a <a href="https://arrow.apache.org/datafusion-python/">dataframe library</a> and
+<a href="https://arrow.apache.org/datafusion/user-guide/cli.html">command line SQL tool</a>.</p>
+
+<p>This may also be our last update on the Apache Arrow Site. Future
+updates will likely be on the DataFusion website as we are working to <a href="https://github.com/apache/arrow-datafusion/discussions/6475">graduate
+to a top level project</a> (Apache Arrow DataFusion → Apache DataFusion!) which
+will help focus governance and project growth. Also exciting, our <a href="https://github.com/apache/arrow-datafusion/discussions/8522">first
+DataFusion in person meetup</a> is planned for March 2024.</p>
+
+<p>DataFusion is very much a community endeavor. Our core thesis is that as a
+community we can build much more advanced technology than any of us as
+individuals or companies could alone. In the last 6 months between <code class="language-plaintext highlighter-rouge">26.0.0</code> and
+<code class="language-plaintext highlighter-rouge">34.0.0</code>, community growth has been strong. We accepted and reviewed over a
+thousand PRs from 124 different committers, created over 650 issues and closed 517
+of them.
+You can find a list of all changes in the detailed <a href="https://github.com/apache/arrow-datafusion/blob/main/datafusion/CHANGELOG.md">CHANGELOG</a>.</p>
+
+<!--
+$ git log --pretty=oneline 26.0.0..34.0.0 . | wc -l
+ 1009
+
+$ git shortlog -sn 26.0.0..34.0.0 . | wc -l
+ 124
+
+https://crates.io/crates/datafusion/26.0.0
+DataFusion 26 released June 7, 2023
+
+https://crates.io/crates/datafusion/34.0.0
+DataFusion 34 released Dec 17, 2023
+
+Issues created in this time: 214 open, 437 closed
+https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+created%3A2023-06-23..2023-12-17
+
+Issues closes: 517
+https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+closed%3A2023-06-23..2023-12-17+
+
+PRs merged in this time 908
+https://github.com/apache/arrow-datafusion/pulls?q=is%3Apr+merged%3A2023-06-23..2023-12-17
+-->
+
+<h1 id="improved-performance-">Improved Performance 🚀</h1>
+
+<p>Performance is a key feature of DataFusion, DataFusion is
+more than 2x faster on <a href="https://benchmark.clickhouse.com/">ClickBench</a> compared to version <code class="language-plaintext highlighter-rouge">25.0.0</code>, as shown below:</p>
+
+<!--
+ Scripts: https://github.com/alamb/datafusion-duckdb-benchmark/tree/datafusion-25-34
+ Spreadsheet: https://docs.google.com/spreadsheets/d/1FtI3652WIJMC5LmJbLfT3G06w0JQIxEPG4yfMafexh8/edit#gid=1879366976
+ Average runtime on 25.0.0: 7.2s (for the queries that actually ran)
+ Average runtime on 34.0.0: 3.6s (for the same queries that ran in 25.0.0)
+-->
+
+<figure style="text-align: center;">
+ <img src="/blog/img/datafusion-34.0.0/compare-new.png" width="100%" class="img-responsive" alt="Fig 1: Adaptive Arrow schema architecture overview." />
+ <figcaption>
+ <b>Figure 1</b>: Performance improvement between <code>25.0.0</code> and <code>34.0.0</code> on ClickBench.
+ Note that DataFusion <code>25.0.0</code>, could not run several queries due to
+ unsupported SQL (Q9, Q11, Q12, Q14) or memory requirements (Q33).
+ </figcaption>
+</figure>
+
+<figure style="text-align: center;">
+ <img src="/blog/img/datafusion-34.0.0/compare.png" width="100%" class="img-responsive" alt="Fig 1: Adaptive Arrow schema architecture overview." />
+ <figcaption>
+ <b>Figure 2</b>: Total query runtime for DataFusion <code>34.0.0</code> and DataFusion <code>25.0.0</code>.
+ </figcaption>
+</figure>
+
+<p>Here are some specific enhancements we have made to improve performance:</p>
+<ul>
+ <li><a href="https://arrow.apache.org/blog/2023/08/05/datafusion_fast_grouping/">2-3x better aggregation performance with many distinct groups</a></li>
+ <li>Partially ordered grouping / streaming grouping</li>
+ <li><a href="https://github.com/apache/arrow-datafusion/pull/7721">Specialized operator for “TopK” <code class="language-plaintext highlighter-rouge">ORDER BY LIMIT XXX</code></a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/pull/7192">Specialized operator for <code class="language-plaintext highlighter-rouge">min(col) GROUP BY .. ORDER by min(col) LIMIT XXX</code></a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/pull/8126">Improved join performance</a></li>
+ <li>Eliminate redundant sorting with sort order aware optimizers</li>
+</ul>
+
+<h1 id="new-features-">New Features ✨</h1>
+
+<h2 id="dml--insert--creating-files">DML / Insert / Creating Files</h2>
+
+<p>DataFusion now supports writing data in parallel, to individual or multiple
+files, using <code class="language-plaintext highlighter-rouge">Parquet</code>, <code class="language-plaintext highlighter-rouge">CSV</code>, <code class="language-plaintext highlighter-rouge">JSON</code>, <code class="language-plaintext highlighter-rouge">ARROW</code> and user defined formats.
+<a href="https://github.com/apache/arrow-datafusion/pull/7655">Benchmark results</a> show improvements up to 5x in some cases.</p>
+
+<p>Similarly to reading, data can now be written to any <a href="https://docs.rs/object_store/0.9.0/object_store/index.html"><code class="language-plaintext highlighter-rouge">ObjectStore</code></a>
+implementation, including AWS S3, Azure Blob Storage, GCP Cloud Storage, local
+files, and user defined implementations. While reading from <a href="https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html#features">hive style
+partitioned tables</a> has long been supported, it is now possible to write to such
+tables as well.</p>
+
+<p>For example, to write to a local file:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">awesome_table</span><span class="p">(</span><span class="n">x</span> <span class="nb">INT</span><span class="p">)</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">PARQUET</span> <span class="k">LOCATION</span> <span class="s1">'/tmp [...]
+<span class="mi">0</span> <span class="k">rows</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">003</span> <span class="n">seconds</span><span class="p">.</span>
+
+<span class="err">❯</span> <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">awesome_table</span> <span class="k">SELECT</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">10</span> <span class="k">FROM</span> <span class="n">my_source_table</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="k">count</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="mi">1</span> <span class="k">row</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">024</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<p>You can also write to files with the <a href="https://arrow.apache.org/datafusion/user-guide/sql/dml.html#copy"><code class="language-plaintext highlighter-rouge">COPY</code></a>, similarly to <a href="https://duckdb.org/docs/sql/statements/copy.html">DuckDB’s <code class="language-plaintext highlighter-rouge">COPY</code></a>:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">COPY</span> <span class="p">(</span><span class="k">SELECT</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">FROM</span> <span class="n">my_source_table</span><span class="p">)</span> <span class="k">TO</span> <span class="s1">'/tmp/output.json'</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="k">count</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="mi">1</span> <span class="k">row</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">014</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">$ </span><span class="nb">cat</span> /tmp/output.json
+<span class="o">{</span><span class="s2">"x"</span>:1<span class="o">}</span>
+<span class="o">{</span><span class="s2">"x"</span>:2<span class="o">}</span>
+<span class="o">{</span><span class="s2">"x"</span>:3<span class="o">}</span>
+</code></pre></div></div>
+
+<h2 id="improved-struct-and-array-support">Improved <code class="language-plaintext highlighter-rouge">STRUCT</code> and <code class="language-plaintext highlighter-rouge">ARRAY</code> support</h2>
+
+<p>DataFusion <code class="language-plaintext highlighter-rouge">34.0.0</code> has much improved <code class="language-plaintext highlighter-rouge">STRUCT</code> and <code class="language-plaintext highlighter-rouge">ARRAY</code>
+support, including a full range of <a href="https://arrow.apache.org/datafusion/user-guide/sql/scalar_functions.html#struct-functions">struct functions</a> and <a href="https://arrow.apache.org/datafusion/user-guide/sql/scalar_functions.html#array-functions">array functions</a>.</p>
+
+<!--
+❯ create table my_table as values ([1,2,3]), ([2]), ([4,5]);
+-->
+
+<p>For example, you can now use <code class="language-plaintext highlighter-rouge">[]</code> syntax and <code class="language-plaintext highlighter-rouge">array_length</code> to access and inspect arrays:</p>
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">SELECT</span> <span class="n">column1</span><span class="p">,</span>
+ <span class="n">column1</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">AS</span> <span class="n">first_element</span><span class="p">,</span>
+ <span class="n">array_length</span><span class="p">(</span><span class="n">column1</span><span class="p">)</span> <span class="k">AS</span> <span class="n">len</span>
+ <span class="k">FROM</span> <span class="n">my_table</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">-----------+---------------+-----+</span>
+<span class="o">|</span> <span class="n">column1</span> <span class="o">|</span> <span class="n">first_element</span> <span class="o">|</span> <span class="n">len</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-----------+---------------+-----+</span>
+<span class="o">|</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">|</span> <span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
+<span class="o">|</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-----------+---------------+-----+</span>
+</code></pre></div></div>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">SELECT</span> <span class="n">column1</span><span class="p">,</span> <span class="n">column1</span><span class="p">[</span><span class="s1">'c0'</span><span class="p">]</span> <span class="k">FROM</span> <span class="n">my_table</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">------------------+----------------------+</span>
+<span class="o">|</span> <span class="n">column1</span> <span class="o">|</span> <span class="n">my_table</span><span class="p">.</span><span class="n">column1</span><span class="p">[</span><span class="n">c0</span><span class="p">]</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">------------------+----------------------+</span>
+<span class="o">|</span> <span class="p">{</span><span class="n">c0</span><span class="p">:</span> <span class="n">foo</span><span class="p">,</span> <span class="n">c1</span><span class="p">:</span> <span class="mi">1</span><span class="p">}</span> <span class="o">|</span> <span class="n">foo</span> <span class="o">|</span>
+<span class="o">|</span> <span class="p">{</span><span class="n">c0</span><span class="p">:</span> <span class="n">bar</span><span class="p">,</span> <span class="n">c1</span><span class="p">:</span> <span class="mi">2</span><span class="p">}</span> <span class="o">|</span> <span class="n">bar</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">------------------+----------------------+</span>
+<span class="mi">2</span> <span class="k">rows</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">002</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<h2 id="other-features">Other Features</h2>
+<p>Other notable features include:</p>
+<ul>
+ <li>Support aggregating datasets that exceed memory size, with <a href="https://github.com/apache/arrow-datafusion/pull/7400">group by spill to disk</a></li>
+ <li>All operators now track and limit their memory consumption, including Joins</li>
+</ul>
+
+<h1 id="building-systems-is-easier-with-datafusion-️">Building Systems is Easier with DataFusion 🛠️</h1>
+
+<h2 id="documentation">Documentation</h2>
+<p>It is easier than ever to get started using DataFusion with the
+new <a href="https://arrow.apache.org/datafusion/library-user-guide/index.html">Library Users Guide</a> as well as significantly improved the <a href="https://docs.rs/datafusion/latest/datafusion/index.html">API documentation</a>.</p>
+
+<h2 id="user-defined-window-and-table-functions">User Defined Window and Table Functions</h2>
+<p>In addition to DataFusion’s <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-a-scalar-udf">User Defined Scalar Functions</a>, and <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-an-aggregate-udf">User Defined Aggregate Functions</a>, DataFusion now supports <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-a-window-udf">User Defined Window Functions</a>
+ and <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-a-user-defined-table-function">User Defined Table Functions</a>.</p>
+
+<p>For example, <a href="https://arrow.apache.org/datafusion/user-guide/cli.html">the <code class="language-plaintext highlighter-rouge">datafusion-cli</code></a> implements a DuckDB style <a href="https://arrow.apache.org/datafusion/user-guide/cli.html#supported-sql"><code class="language-plaintext highlighter-rouge">parquet_metadata</code></a>
+function as a user defined table function (<a href="https://github.com/apache/arrow-datafusion/blob/3f219bc929cfd418b0e3d3501f8eba1d5a2c87ae/datafusion-cli/src/functions.rs#L222-L248">source code here</a>):</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">SELECT</span>
+ <span class="n">path_in_schema</span><span class="p">,</span> <span class="n">row_group_id</span><span class="p">,</span> <span class="n">row_group_num_rows</span><span class="p">,</span> <span class="n">stats_min</span><span class="p">,</span> <span class="n">stats_max</span><span class="p">,</span> <span class="n">total_compressed_size</span>
+<span class="k">FROM</span>
+ <span class="n">parquet_metadata</span><span class="p">(</span><span class="s1">'hits.parquet'</span><span class="p">)</span>
+<span class="k">WHERE</span> <span class="n">path_in_schema</span> <span class="o">=</span> <span class="s1">'"WatchID"'</span>
+<span class="k">LIMIT</span> <span class="mi">3</span><span class="p">;</span>
+
+<span class="o">+</span><span class="c1">----------------+--------------+--------------------+---------------------+---------------------+-----------------------+</span>
+<span class="o">|</span> <span class="n">path_in_schema</span> <span class="o">|</span> <span class="n">row_group_id</span> <span class="o">|</span> <span class="n">row_group_num_rows</span> <span class="o">|</span> <span class="n">stats_min</span> <span class="o">|</span> <span class="n">stats_max</span> <span class="o">|</span> <span class="n">total_compressed_size</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">----------------+--------------+--------------------+---------------------+---------------------+-----------------------+</span>
+<span class="o">|</span> <span class="nv">"WatchID"</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">450560</span> <span class="o">|</span> <span class="mi">4611687214012840539</span> <span class="o">|</span> <span class="mi">9223369186199968220</span> <span class="o">|</span> <span class="mi">3883759</span> <span class="o">|</span>
+<span class="o">|</span> <span class="nv">"WatchID"</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">612174</span> <span class="o">|</span> <span class="mi">4611689135232456464</span> <span class="o">|</span> <span class="mi">9223371478009085789</span> <span class="o">|</span> <span class="mi">5176803</span> <span class="o">|</span>
+<span class="o">|</span> <span class="nv">"WatchID"</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">344064</span> <span class="o">|</span> <span class="mi">4611692774829951781</span> <span class="o">|</span> <span class="mi">9223363791697310021</span> <span class="o">|</span> <span class="mi">3031680</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">----------------+--------------+--------------------+---------------------+---------------------+-----------------------+</span>
+<span class="mi">3</span> <span class="k">rows</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">053</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<h3 id="growth-of-datafusion-">Growth of DataFusion 📈</h3>
+<p>DataFusion has been appearing more publically in the wild. For example</p>
+<ul>
+ <li>New projects built using DataFusion such as <a href="https://lancedb.com/">lancedb</a>, <a href="https://glaredb.com/">GlareDB</a>, <a href="https://www.arroyo.dev/">Arroyo</a>, and <a href="https://github.com/cmu-db/optd">optd</a>.</li>
+ <li>Public talks such as <a href="https://www.youtube.com/watch?v=AJU9rdRNk9I">Apache Arrow Datafusion: Vectorized
+Execution Framework For Maximum Performance</a> in <a href="https://www.bagevent.com/event/8432178">CommunityOverCode Asia 2023</a></li>
+ <li>Blogs posts such as <a href="https://www.synnada.ai/blog/apache-arrow-arrow-datafusion-ai-native-data-infra-an-interview-with-our-ceo-ozan">Apache Arrow, Arrow/DataFusion, AI-native Data Infra</a>,
+<a href="https://www.influxdata.com/blog/flight-datafusion-arrow-parquet-fdap-architecture-influxdb/">Flight, DataFusion, Arrow, and Parquet: Using the FDAP Architecture to build InfluxDB 3.0</a>, and
+<a href="https://www.linkedin.com/pulse/guide-user-defined-functions-apache-arrow-datafusion-dade-aderemi/">A Guide to User-Defined Functions in Apache Arrow DataFusion</a></li>
+</ul>
+
+<p>We have also <a href="https://github.com/apache/arrow-datafusion/issues/6782">submitted a paper</a> to <a href="https://2024.sigmod.org/">SIGMOD 2024</a>, one of the
+premiere database conferences, describing DataFusion in a technically formal
+style and making the case that it is possible to create a modular and extensive query engine
+without sacrificing performance. We hope this paper helps people
+evaluating DataFusion for their needs understand it better.</p>
+
+<h1 id="datafusion-in-2024-">DataFusion in 2024 🥳</h1>
+
+<p>Some major initiatives from contributors we know of this year are:</p>
+
+<ol>
+ <li>
+ <p><em>Modularity</em>: Make DataFusion even more modular, such as <a href="https://github.com/apache/arrow-datafusion/issues/8045">unifying
+built in and user functions</a>, making it easier to customize
+DataFusion’s behavior.</p>
+ </li>
+ <li>
+ <p><em>Community Growth</em>: Graduate to our own top level Apache project, and
+subsequently add more committers and PMC members to keep pace with project
+growth.</p>
+ </li>
+ <li>
+ <p><em>Use case white papers</em>: Write blog posts and videos explaining
+how to use DataFusion for real-world use cases.</p>
+ </li>
+ <li>
+ <p><em>Testing</em>: Improve CI infrastructure and test coverage, more fuzz
+testing, and better functional and performance regression testing.</p>
+ </li>
+ <li>
+ <p><em>Planning Time</em>: Reduce the time taken to plan queries, both <a href="https://github.com/apache/arrow-datafusion/issues/7698">wide
+tables of 1000s of columns</a>, and in <a href="https://github.com/apache/arrow-datafusion/issues/5637">general</a>.</p>
+ </li>
+ <li>
+ <p><em>Aggregate Performance</em>: Improve the speed of <a href="https://github.com/apache/arrow-datafusion/issues/7000">aggregating “high cardinality”</a> data
+when there are many (e.g. millions) of distinct groups.</p>
+ </li>
+ <li>
+ <p><em>Statistics</em>: <a href="https://github.com/apache/arrow-datafusion/issues/8227">Improved statistics handling</a> with an eye towards more
+sophisticated expression analysis and cost models.</p>
+ </li>
+</ol>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion we would love to have you
+join us. You can try out DataFusion on some of your own data and projects and
+let us know how it goes, contribute suggestions, documentation, bug reports, or
+a PR with documentation, tests or code. A list of open issues
+suitable for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>As the community grows, we are also looking to restart biweekly calls /
+meetings. Timezones are always a challenge for such meetings, but we hope to
+have two calls that can work for most attendees. If you are interested
+in helping, or just want to say hi, please drop us a note via one of
+the methods listed in our <a href="https://arrow.apache.org/datafusion/contributor-guide/communication.html">Communication Doc</a>.</p>
+
+
+ </div><a class="u-url" href="/blog/blog/2024/01/19/datafusion-34.0.0/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2024/03/06/comet-donation/index.html b/blog/2024/03/06/comet-donation/index.html
new file mode 100644
index 0000000..3e85745
--- /dev/null
+++ b/blog/2024/03/06/comet-donation/index.html
@@ -0,0 +1,136 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Announcing Apache Arrow DataFusion Comet | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Announcing Apache Arrow DataFusion Comet" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2024/03/06/comet-donation/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2024/03/06/comet-donation/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2024-03-06T00:00:00-07:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Announcing Apache Arrow DataFusion Comet" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2024-03-06T00:00:00-07:00","datePublished":"2024-03-06T00:00:00-07:00","description":"<!–","headline":"Announcing Apache Arrow DataFusion Comet","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2024/03/06/comet-donation/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgw [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Announcing Apache Arrow DataFusion Comet</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2024-03-06T00:00:00-07:00" itemprop="datePublished">Mar 6, 2024
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+<p>The Apache Arrow PMC is pleased to announce the donation of the <a href="https://github.com/apache/arrow-datafusion-comet">Comet project</a>,
+a native Spark SQL Accelerator built on <a href="https://arrow.apache.org/datafusion">Apache Arrow DataFusion</a>.</p>
+
+<p>Comet is an Apache Spark plugin that uses Apache Arrow DataFusion to
+accelerate Spark workloads. It is designed as a drop-in
+replacement for Spark’s JVM based SQL execution engine and offers significant
+performance improvements for some workloads as shown below.</p>
+
+<figure style="text-align: center;">
+ <img src="/blog/img/datafusion-comet/comet-architecture.png" width="100%" class="img-responsive" alt="Fig 1: Adaptive Arrow schema architecture overview." />
+ <figcaption>
+ <b>Figure 1</b>: With Comet, users interact with the same Spark ecosystem, tools
+ and APIs such as Spark SQL. Queries still run through Spark's query optimizer and planner.
+ However, the execution is delegated to Comet,
+ which is significantly faster and more resource efficient than a JVM based
+ implementation.
+</figcaption>
+</figure>
+
+<p>Comet is one of a growing class of projects that aim to accelerate Spark using
+native columnar engines such as the proprietary <a href="https://www.databricks.com/product/photon">Databricks Photon Engine</a> and
+open source projects <a href="https://incubator.apache.org/projects/gluten.html">Gluten</a>, <a href="https://github.com/NVIDIA/spark-rapids">Spark RAPIDS</a>, and <a href="https://github.com/kwai/blaze">Blaze</a> (also built using
+DataFusion).</p>
+
+<p>Comet was originally implemented at Apple and the engineers who worked on the
+project are also significant contributors to Arrow and DataFusion. Bringing
+Comet into the Apache Software Foundation will accelerate its development and
+grow its community of contributors and users.</p>
+
+<h1 id="get-involved">Get Involved</h1>
+<p>Comet is still in the early stages of development and we would love to have you
+join us and help shape the project. We are working on an initial release, and
+expect to post another update with more details at that time.</p>
+
+<p>Before then, here are some ways to get involved:</p>
+
+<ul>
+ <li>
+ <p>Learn more by visiting the <a href="https://github.com/apache/arrow-datafusion-comet">Comet project</a> page and reading the <a href="https://lists.apache.org/thread/0q1rb11jtpopc7vt1ffdzro0omblsh0s">mailing list
+discussion</a> about the initial donation.</p>
+ </li>
+ <li>
+ <p>Help us plan out the <a href="https://github.com/apache/arrow-datafusion-comet/issues/19">roadmap</a></p>
+ </li>
+ <li>
+ <p>Try out the project and provide feedback, file issues, and contribute code.</p>
+ </li>
+</ul>
+
+
+ </div><a class="u-url" href="/blog/blog/2024/03/06/comet-donation/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/blog/2024/05/07/datafusion-tlp/index.html b/blog/2024/05/07/datafusion-tlp/index.html
new file mode 100644
index 0000000..6fc527f
--- /dev/null
+++ b/blog/2024/05/07/datafusion-tlp/index.html
@@ -0,0 +1,135 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Announcing Apache Arrow DataFusion is now Apache DataFusion | Apache DataFusion Project News & Blog</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Announcing Apache Arrow DataFusion is now Apache DataFusion" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="<!–" />
+<meta property="og:description" content="<!–" />
+<link rel="canonical" href="https://datafusion.apache.org/blog/blog/2024/05/07/datafusion-tlp/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/blog/2024/05/07/datafusion-tlp/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2024-05-07T00:00:00-06:00" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Announcing Apache Arrow DataFusion is now Apache DataFusion" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"pmc"},"dateModified":"2024-05-07T00:00:00-06:00","datePublished":"2024-05-07T00:00:00-06:00","description":"<!–","headline":"Announcing Apache Arrow DataFusion is now Apache DataFusion","mainEntityOfPage":{"@type":"WebPage","@id":"https://datafusion.apache.org/blog/blog/2024/05/07/datafusion-tlp/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache. [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
+
+ <header class="post-header">
+ <h1 class="post-title p-name" itemprop="name headline">Announcing Apache Arrow DataFusion is now Apache DataFusion</h1>
+ <p class="post-meta">
+ <time class="dt-published" datetime="2024-05-07T00:00:00-06:00" itemprop="datePublished">May 7, 2024
+ </time>• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">pmc</span></span></p>
+ </header>
+
+ <div class="post-content e-content" itemprop="articleBody">
+ <!--
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>TLDR; <a href="https://arrow.apache.org/">Apache Arrow</a> DataFusion –> <a href="https://datafusion.apache.org/">Apache DataFusion</a></p>
+
+<p>The Arrow PMC and newly created DataFusion PMC are happy to announce that as of
+April 16, 2024 the Apache Arrow DataFusion subproject is now a top level
+<a href="https://www.apache.org/">Apache Software Foundation</a> project.</p>
+
+<h2 id="background">Background</h2>
+
+<p>Apache DataFusion is a fast, extensible query engine for building high-quality
+data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+
+<p>When DataFusion was <a href="https://arrow.apache.org/blog/2019/02/04/datafusion-donation/">donated to the Apache Software Foundation</a> in 2019, the
+DataFusion community was not large enough to stand on its own and the Arrow
+project agreed to help support it. The community has grown significantly since
+2019, benefiting immensely from being part of Arrow and following <a href="https://www.apache.org/theapacheway/">The Apache
+Way</a>.</p>
+
+<h2 id="why-now">Why now?</h2>
+
+<p>The community <a href="https://github.com/apache/datafusion/discussions/6475">discussed graduating to a top level project publicly</a> for almost
+a year, as the project seemed ready to stand on its own and would benefit from
+more focused governance. For example, earlier in DataFusion’s life many
+contributed to both <a href="https://github.com/apache/arrow-rs">arrow-rs</a> and DataFusion, but as DataFusion has matured many
+contributors, committers and PMC members focused more and more exclusively on
+DataFusion.</p>
+
+<h2 id="looking-forward">Looking forward</h2>
+
+<p>The future looks bright. There are now <a href="https://datafusion.apache.org/user-guide/introduction.html#known-users">10s of known projects built with
+DataFusion</a>, and that number continues to grow. We recently held our <a href="https://github.com/apache/datafusion/discussions/8522">first in
+person meetup</a> passed <a href="https://github.com/apache/datafusion/stargazers">5000 stars</a> on GitHub, <a href="https://github.com/apache/datafusion/issues/8373#issuecomment-2025133714">wrote a paper that was accepted
+at SIGMOD 2024</a>, and began work on <a href="https://github.com/apache/datafusion-comet">Comet</a>, an <a href="https://spark.apache.org/">Apache Spark</a> accelerator
+<a href="https://arrow.apache.org/blog/2024/03/06/comet-donation/">initially donated by Apple</a>.</p>
+
+<p>Thank you to everyone in the Arrow community who helped DataFusion grow and
+mature over the years, and we look forward to continuing our collaboration as
+projects. All future blogs and announcements will be posted on the <a href="https://datafusion.apache.org/">Apache
+DataFusion</a> website.</p>
+
+<h2 id="get-involved">Get Involved</h2>
+
+<p>If you are interested in joining the community, we would love to have you join
+us. Get in touch using <a href="https://datafusion.apache.org/contributor-guide/communication.html">Communication Doc</a> and learn how to get involved in the
+<a href="https://datafusion.apache.org/contributor-guide/index.html">Contributor Guide</a>. We welcome everyone to try DataFusion on their
+own data and projects and let us know how it goes, contribute suggestions,
+documentation, bug reports, or a PR with documentation, tests or code.</p>
+
+
+ </div><a class="u-url" href="/blog/blog/2024/05/07/datafusion-tlp/" hidden></a>
+</article>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
diff --git a/feed.xml b/feed.xml
new file mode 100644
index 0000000..0cddb76
--- /dev/null
+++ b/feed.xml
@@ -0,0 +1,1902 @@
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="4.3.3">Jekyll</generator><link href="https://datafusion.apache.org/blog/feed.xml" rel="self" type="application/atom+xml" /><link href="https://datafusion.apache.org/blog/" rel="alternate" type="text/html" /><updated>2024-05-19T08:24:30-06:00</updated><id>https://datafusion.apache.org/blog/feed.xml</id><title type="html">Apache DataFusion Project News &amp; [...]
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>TLDR; <a href="https://arrow.apache.org/">Apache Arrow</a> DataFusion –> <a href="https://datafusion.apache.org/">Apache DataFusion</a></p>
+
+<p>The Arrow PMC and newly created DataFusion PMC are happy to announce that as of
+April 16, 2024 the Apache Arrow DataFusion subproject is now a top level
+<a href="https://www.apache.org/">Apache Software Foundation</a> project.</p>
+
+<h2 id="background">Background</h2>
+
+<p>Apache DataFusion is a fast, extensible query engine for building high-quality
+data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+
+<p>When DataFusion was <a href="https://arrow.apache.org/blog/2019/02/04/datafusion-donation/">donated to the Apache Software Foundation</a> in 2019, the
+DataFusion community was not large enough to stand on its own and the Arrow
+project agreed to help support it. The community has grown significantly since
+2019, benefiting immensely from being part of Arrow and following <a href="https://www.apache.org/theapacheway/">The Apache
+Way</a>.</p>
+
+<h2 id="why-now">Why now?</h2>
+
+<p>The community <a href="https://github.com/apache/datafusion/discussions/6475">discussed graduating to a top level project publicly</a> for almost
+a year, as the project seemed ready to stand on its own and would benefit from
+more focused governance. For example, earlier in DataFusion’s life many
+contributed to both <a href="https://github.com/apache/arrow-rs">arrow-rs</a> and DataFusion, but as DataFusion has matured many
+contributors, committers and PMC members focused more and more exclusively on
+DataFusion.</p>
+
+<h2 id="looking-forward">Looking forward</h2>
+
+<p>The future looks bright. There are now <a href="https://datafusion.apache.org/user-guide/introduction.html#known-users">10s of known projects built with
+DataFusion</a>, and that number continues to grow. We recently held our <a href="https://github.com/apache/datafusion/discussions/8522">first in
+person meetup</a> passed <a href="https://github.com/apache/datafusion/stargazers">5000 stars</a> on GitHub, <a href="https://github.com/apache/datafusion/issues/8373#issuecomment-2025133714">wrote a paper that was accepted
+at SIGMOD 2024</a>, and began work on <a href="https://github.com/apache/datafusion-comet">Comet</a>, an <a href="https://spark.apache.org/">Apache Spark</a> accelerator
+<a href="https://arrow.apache.org/blog/2024/03/06/comet-donation/">initially donated by Apple</a>.</p>
+
+<p>Thank you to everyone in the Arrow community who helped DataFusion grow and
+mature over the years, and we look forward to continuing our collaboration as
+projects. All future blogs and announcements will be posted on the <a href="https://datafusion.apache.org/">Apache
+DataFusion</a> website.</p>
+
+<h2 id="get-involved">Get Involved</h2>
+
+<p>If you are interested in joining the community, we would love to have you join
+us. Get in touch using <a href="https://datafusion.apache.org/contributor-guide/communication.html">Communication Doc</a> and learn how to get involved in the
+<a href="https://datafusion.apache.org/contributor-guide/index.html">Contributor Guide</a>. We welcome everyone to try DataFusion on their
+own data and projects and let us know how it goes, contribute suggestions,
+documentation, bug reports, or a PR with documentation, tests or code.</p>]]></content><author><name>pmc</name></author><category term="subprojects" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Announcing Apache Arrow DataFusion Comet</title><link href="https://datafusion.apache.org/blog/blog/2024/03/06/comet-donation/" rel="alternate" type="text/html" title="Announcing Apache Arrow DataFusion Comet" /><published>2024-03-06T00:00:00-07:00</publishe [...]
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+<p>The Apache Arrow PMC is pleased to announce the donation of the <a href="https://github.com/apache/arrow-datafusion-comet">Comet project</a>,
+a native Spark SQL Accelerator built on <a href="https://arrow.apache.org/datafusion">Apache Arrow DataFusion</a>.</p>
+
+<p>Comet is an Apache Spark plugin that uses Apache Arrow DataFusion to
+accelerate Spark workloads. It is designed as a drop-in
+replacement for Spark’s JVM based SQL execution engine and offers significant
+performance improvements for some workloads as shown below.</p>
+
+<figure style="text-align: center;">
+ <img src="/blog/img/datafusion-comet/comet-architecture.png" width="100%" class="img-responsive" alt="Fig 1: Adaptive Arrow schema architecture overview." />
+ <figcaption>
+ <b>Figure 1</b>: With Comet, users interact with the same Spark ecosystem, tools
+ and APIs such as Spark SQL. Queries still run through Spark's query optimizer and planner.
+ However, the execution is delegated to Comet,
+ which is significantly faster and more resource efficient than a JVM based
+ implementation.
+</figcaption>
+</figure>
+
+<p>Comet is one of a growing class of projects that aim to accelerate Spark using
+native columnar engines such as the proprietary <a href="https://www.databricks.com/product/photon">Databricks Photon Engine</a> and
+open source projects <a href="https://incubator.apache.org/projects/gluten.html">Gluten</a>, <a href="https://github.com/NVIDIA/spark-rapids">Spark RAPIDS</a>, and <a href="https://github.com/kwai/blaze">Blaze</a> (also built using
+DataFusion).</p>
+
+<p>Comet was originally implemented at Apple and the engineers who worked on the
+project are also significant contributors to Arrow and DataFusion. Bringing
+Comet into the Apache Software Foundation will accelerate its development and
+grow its community of contributors and users.</p>
+
+<h1 id="get-involved">Get Involved</h1>
+<p>Comet is still in the early stages of development and we would love to have you
+join us and help shape the project. We are working on an initial release, and
+expect to post another update with more details at that time.</p>
+
+<p>Before then, here are some ways to get involved:</p>
+
+<ul>
+ <li>
+ <p>Learn more by visiting the <a href="https://github.com/apache/arrow-datafusion-comet">Comet project</a> page and reading the <a href="https://lists.apache.org/thread/0q1rb11jtpopc7vt1ffdzro0omblsh0s">mailing list
+discussion</a> about the initial donation.</p>
+ </li>
+ <li>
+ <p>Help us plan out the <a href="https://github.com/apache/arrow-datafusion-comet/issues/19">roadmap</a></p>
+ </li>
+ <li>
+ <p>Try out the project and provide feedback, file issues, and contribute code.</p>
+ </li>
+</ul>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024</title><link href="https://datafusion.apache.org/blog/blog/2024/01/19/datafusion-34.0.0/" rel="alternate" type="text/html" title="Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024" /><published>2024-01-19T00:00:00-07:00</published><updated>2024-01-19T [...]
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>We recently <a href="https://crates.io/crates/datafusion/34.0.0">released DataFusion 34.0.0</a>. This blog highlights some of the major
+improvements since we <a href="https://arrow.apache.org/blog/2023/06/24/datafusion-25.0.0/.">released DataFusion 26.0.0</a> (spoiler alert there are many)
+and a preview of where the community plans to focus in the next 6 months.</p>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> is an extensible query engine, written in <a href="https://www.rust-lang.org/">Rust</a>, that
+uses <a href="https://arrow.apache.org">Apache Arrow</a> as its in-memory format. DataFusion is used by developers to
+create new, fast data centric systems such as databases, dataframe libraries,
+machine learning and streaming applications. While <a href="https://arrow.apache.org/datafusion/user-guide/introduction.html#project-goals">DataFusion’s primary design
+goal</a> is to accelerate creating other data centric systems, it has a
+reasonable experience directly out of the box as a <a href="https://arrow.apache.org/datafusion-python/">dataframe library</a> and
+<a href="https://arrow.apache.org/datafusion/user-guide/cli.html">command line SQL tool</a>.</p>
+
+<p>This may also be our last update on the Apache Arrow Site. Future
+updates will likely be on the DataFusion website as we are working to <a href="https://github.com/apache/arrow-datafusion/discussions/6475">graduate
+to a top level project</a> (Apache Arrow DataFusion → Apache DataFusion!) which
+will help focus governance and project growth. Also exciting, our <a href="https://github.com/apache/arrow-datafusion/discussions/8522">first
+DataFusion in person meetup</a> is planned for March 2024.</p>
+
+<p>DataFusion is very much a community endeavor. Our core thesis is that as a
+community we can build much more advanced technology than any of us as
+individuals or companies could alone. In the last 6 months between <code class="language-plaintext highlighter-rouge">26.0.0</code> and
+<code class="language-plaintext highlighter-rouge">34.0.0</code>, community growth has been strong. We accepted and reviewed over a
+thousand PRs from 124 different committers, created over 650 issues and closed 517
+of them.
+You can find a list of all changes in the detailed <a href="https://github.com/apache/arrow-datafusion/blob/main/datafusion/CHANGELOG.md">CHANGELOG</a>.</p>
+
+<!--
+$ git log --pretty=oneline 26.0.0..34.0.0 . | wc -l
+ 1009
+
+$ git shortlog -sn 26.0.0..34.0.0 . | wc -l
+ 124
+
+https://crates.io/crates/datafusion/26.0.0
+DataFusion 26 released June 7, 2023
+
+https://crates.io/crates/datafusion/34.0.0
+DataFusion 34 released Dec 17, 2023
+
+Issues created in this time: 214 open, 437 closed
+https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+created%3A2023-06-23..2023-12-17
+
+Issues closes: 517
+https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+closed%3A2023-06-23..2023-12-17+
+
+PRs merged in this time 908
+https://github.com/apache/arrow-datafusion/pulls?q=is%3Apr+merged%3A2023-06-23..2023-12-17
+-->
+
+<h1 id="improved-performance-">Improved Performance 🚀</h1>
+
+<p>Performance is a key feature of DataFusion, DataFusion is
+more than 2x faster on <a href="https://benchmark.clickhouse.com/">ClickBench</a> compared to version <code class="language-plaintext highlighter-rouge">25.0.0</code>, as shown below:</p>
+
+<!--
+ Scripts: https://github.com/alamb/datafusion-duckdb-benchmark/tree/datafusion-25-34
+ Spreadsheet: https://docs.google.com/spreadsheets/d/1FtI3652WIJMC5LmJbLfT3G06w0JQIxEPG4yfMafexh8/edit#gid=1879366976
+ Average runtime on 25.0.0: 7.2s (for the queries that actually ran)
+ Average runtime on 34.0.0: 3.6s (for the same queries that ran in 25.0.0)
+-->
+
+<figure style="text-align: center;">
+ <img src="/blog/img/datafusion-34.0.0/compare-new.png" width="100%" class="img-responsive" alt="Fig 1: Adaptive Arrow schema architecture overview." />
+ <figcaption>
+ <b>Figure 1</b>: Performance improvement between <code>25.0.0</code> and <code>34.0.0</code> on ClickBench.
+ Note that DataFusion <code>25.0.0</code>, could not run several queries due to
+ unsupported SQL (Q9, Q11, Q12, Q14) or memory requirements (Q33).
+ </figcaption>
+</figure>
+
+<figure style="text-align: center;">
+ <img src="/blog/img/datafusion-34.0.0/compare.png" width="100%" class="img-responsive" alt="Fig 1: Adaptive Arrow schema architecture overview." />
+ <figcaption>
+ <b>Figure 2</b>: Total query runtime for DataFusion <code>34.0.0</code> and DataFusion <code>25.0.0</code>.
+ </figcaption>
+</figure>
+
+<p>Here are some specific enhancements we have made to improve performance:</p>
+<ul>
+ <li><a href="https://arrow.apache.org/blog/2023/08/05/datafusion_fast_grouping/">2-3x better aggregation performance with many distinct groups</a></li>
+ <li>Partially ordered grouping / streaming grouping</li>
+ <li><a href="https://github.com/apache/arrow-datafusion/pull/7721">Specialized operator for “TopK” <code class="language-plaintext highlighter-rouge">ORDER BY LIMIT XXX</code></a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/pull/7192">Specialized operator for <code class="language-plaintext highlighter-rouge">min(col) GROUP BY .. ORDER by min(col) LIMIT XXX</code></a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/pull/8126">Improved join performance</a></li>
+ <li>Eliminate redundant sorting with sort order aware optimizers</li>
+</ul>
+
+<h1 id="new-features-">New Features ✨</h1>
+
+<h2 id="dml--insert--creating-files">DML / Insert / Creating Files</h2>
+
+<p>DataFusion now supports writing data in parallel, to individual or multiple
+files, using <code class="language-plaintext highlighter-rouge">Parquet</code>, <code class="language-plaintext highlighter-rouge">CSV</code>, <code class="language-plaintext highlighter-rouge">JSON</code>, <code class="language-plaintext highlighter-rouge">ARROW</code> and user defined formats.
+<a href="https://github.com/apache/arrow-datafusion/pull/7655">Benchmark results</a> show improvements up to 5x in some cases.</p>
+
+<p>Similarly to reading, data can now be written to any <a href="https://docs.rs/object_store/0.9.0/object_store/index.html"><code class="language-plaintext highlighter-rouge">ObjectStore</code></a>
+implementation, including AWS S3, Azure Blob Storage, GCP Cloud Storage, local
+files, and user defined implementations. While reading from <a href="https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html#features">hive style
+partitioned tables</a> has long been supported, it is now possible to write to such
+tables as well.</p>
+
+<p>For example, to write to a local file:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">CREATE</span> <span class="k">EXTERNAL</span> <span class="k">TABLE</span> <span class="n">awesome_table</span><span class="p">(</span><span class="n">x</span> <span class="nb">INT</span><span class="p">)</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">PARQUET</span> <span class="k">LOCATION</span> <span class="s1">'/tmp [...]
+<span class="mi">0</span> <span class="k">rows</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">003</span> <span class="n">seconds</span><span class="p">.</span>
+
+<span class="err">❯</span> <span class="k">INSERT</span> <span class="k">INTO</span> <span class="n">awesome_table</span> <span class="k">SELECT</span> <span class="n">x</span> <span class="o">*</span> <span class="mi">10</span> <span class="k">FROM</span> <span class="n">my_source_table</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="k">count</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="mi">1</span> <span class="k">row</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">024</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<p>You can also write to files with the <a href="https://arrow.apache.org/datafusion/user-guide/sql/dml.html#copy"><code class="language-plaintext highlighter-rouge">COPY</code></a>, similarly to <a href="https://duckdb.org/docs/sql/statements/copy.html">DuckDB’s <code class="language-plaintext highlighter-rouge">COPY</code></a>:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">COPY</span> <span class="p">(</span><span class="k">SELECT</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">FROM</span> <span class="n">my_source_table</span><span class="p">)</span> <span class="k">TO</span> <span class="s1">'/tmp/output.json'</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="k">count</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-------+</span>
+<span class="mi">1</span> <span class="k">row</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">014</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">$ </span><span class="nb">cat</span> /tmp/output.json
+<span class="o">{</span><span class="s2">"x"</span>:1<span class="o">}</span>
+<span class="o">{</span><span class="s2">"x"</span>:2<span class="o">}</span>
+<span class="o">{</span><span class="s2">"x"</span>:3<span class="o">}</span>
+</code></pre></div></div>
+
+<h2 id="improved-struct-and-array-support">Improved <code class="language-plaintext highlighter-rouge">STRUCT</code> and <code class="language-plaintext highlighter-rouge">ARRAY</code> support</h2>
+
+<p>DataFusion <code class="language-plaintext highlighter-rouge">34.0.0</code> has much improved <code class="language-plaintext highlighter-rouge">STRUCT</code> and <code class="language-plaintext highlighter-rouge">ARRAY</code>
+support, including a full range of <a href="https://arrow.apache.org/datafusion/user-guide/sql/scalar_functions.html#struct-functions">struct functions</a> and <a href="https://arrow.apache.org/datafusion/user-guide/sql/scalar_functions.html#array-functions">array functions</a>.</p>
+
+<!--
+❯ create table my_table as values ([1,2,3]), ([2]), ([4,5]);
+-->
+
+<p>For example, you can now use <code class="language-plaintext highlighter-rouge">[]</code> syntax and <code class="language-plaintext highlighter-rouge">array_length</code> to access and inspect arrays:</p>
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">SELECT</span> <span class="n">column1</span><span class="p">,</span>
+ <span class="n">column1</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">AS</span> <span class="n">first_element</span><span class="p">,</span>
+ <span class="n">array_length</span><span class="p">(</span><span class="n">column1</span><span class="p">)</span> <span class="k">AS</span> <span class="n">len</span>
+ <span class="k">FROM</span> <span class="n">my_table</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">-----------+---------------+-----+</span>
+<span class="o">|</span> <span class="n">column1</span> <span class="o">|</span> <span class="n">first_element</span> <span class="o">|</span> <span class="n">len</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-----------+---------------+-----+</span>
+<span class="o">|</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
+<span class="o">|</span> <span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
+<span class="o">|</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">-----------+---------------+-----+</span>
+</code></pre></div></div>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">SELECT</span> <span class="n">column1</span><span class="p">,</span> <span class="n">column1</span><span class="p">[</span><span class="s1">'c0'</span><span class="p">]</span> <span class="k">FROM</span> <span class="n">my_table</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">------------------+----------------------+</span>
+<span class="o">|</span> <span class="n">column1</span> <span class="o">|</span> <span class="n">my_table</span><span class="p">.</span><span class="n">column1</span><span class="p">[</span><span class="n">c0</span><span class="p">]</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">------------------+----------------------+</span>
+<span class="o">|</span> <span class="p">{</span><span class="n">c0</span><span class="p">:</span> <span class="n">foo</span><span class="p">,</span> <span class="n">c1</span><span class="p">:</span> <span class="mi">1</span><span class="p">}</span> <span class="o">|</span> <span class="n">foo</span> <span class="o">|</span>
+<span class="o">|</span> <span class="p">{</span><span class="n">c0</span><span class="p">:</span> <span class="n">bar</span><span class="p">,</span> <span class="n">c1</span><span class="p">:</span> <span class="mi">2</span><span class="p">}</span> <span class="o">|</span> <span class="n">bar</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">------------------+----------------------+</span>
+<span class="mi">2</span> <span class="k">rows</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">002</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<h2 id="other-features">Other Features</h2>
+<p>Other notable features include:</p>
+<ul>
+ <li>Support aggregating datasets that exceed memory size, with <a href="https://github.com/apache/arrow-datafusion/pull/7400">group by spill to disk</a></li>
+ <li>All operators now track and limit their memory consumption, including Joins</li>
+</ul>
+
+<h1 id="building-systems-is-easier-with-datafusion-️">Building Systems is Easier with DataFusion 🛠️</h1>
+
+<h2 id="documentation">Documentation</h2>
+<p>It is easier than ever to get started using DataFusion with the
+new <a href="https://arrow.apache.org/datafusion/library-user-guide/index.html">Library Users Guide</a> as well as significantly improved the <a href="https://docs.rs/datafusion/latest/datafusion/index.html">API documentation</a>.</p>
+
+<h2 id="user-defined-window-and-table-functions">User Defined Window and Table Functions</h2>
+<p>In addition to DataFusion’s <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-a-scalar-udf">User Defined Scalar Functions</a>, and <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-an-aggregate-udf">User Defined Aggregate Functions</a>, DataFusion now supports <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-a-window-udf">User Defined Window Functions</a>
+ and <a href="https://arrow.apache.org/datafusion/library-user-guide/adding-udfs.html#adding-a-user-defined-table-function">User Defined Table Functions</a>.</p>
+
+<p>For example, <a href="https://arrow.apache.org/datafusion/user-guide/cli.html">the <code class="language-plaintext highlighter-rouge">datafusion-cli</code></a> implements a DuckDB style <a href="https://arrow.apache.org/datafusion/user-guide/cli.html#supported-sql"><code class="language-plaintext highlighter-rouge">parquet_metadata</code></a>
+function as a user defined table function (<a href="https://github.com/apache/arrow-datafusion/blob/3f219bc929cfd418b0e3d3501f8eba1d5a2c87ae/datafusion-cli/src/functions.rs#L222-L248">source code here</a>):</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="err">❯</span> <span class="k">SELECT</span>
+ <span class="n">path_in_schema</span><span class="p">,</span> <span class="n">row_group_id</span><span class="p">,</span> <span class="n">row_group_num_rows</span><span class="p">,</span> <span class="n">stats_min</span><span class="p">,</span> <span class="n">stats_max</span><span class="p">,</span> <span class="n">total_compressed_size</span>
+<span class="k">FROM</span>
+ <span class="n">parquet_metadata</span><span class="p">(</span><span class="s1">'hits.parquet'</span><span class="p">)</span>
+<span class="k">WHERE</span> <span class="n">path_in_schema</span> <span class="o">=</span> <span class="s1">'"WatchID"'</span>
+<span class="k">LIMIT</span> <span class="mi">3</span><span class="p">;</span>
+
+<span class="o">+</span><span class="c1">----------------+--------------+--------------------+---------------------+---------------------+-----------------------+</span>
+<span class="o">|</span> <span class="n">path_in_schema</span> <span class="o">|</span> <span class="n">row_group_id</span> <span class="o">|</span> <span class="n">row_group_num_rows</span> <span class="o">|</span> <span class="n">stats_min</span> <span class="o">|</span> <span class="n">stats_max</span> <span class="o">|</span> <span class="n">total_compressed_size</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">----------------+--------------+--------------------+---------------------+---------------------+-----------------------+</span>
+<span class="o">|</span> <span class="nv">"WatchID"</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">450560</span> <span class="o">|</span> <span class="mi">4611687214012840539</span> <span class="o">|</span> <span class="mi">9223369186199968220</span> <span class="o">|</span> <span class="mi">3883759</span> <span class="o">|</span>
+<span class="o">|</span> <span class="nv">"WatchID"</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">612174</span> <span class="o">|</span> <span class="mi">4611689135232456464</span> <span class="o">|</span> <span class="mi">9223371478009085789</span> <span class="o">|</span> <span class="mi">5176803</span> <span class="o">|</span>
+<span class="o">|</span> <span class="nv">"WatchID"</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">344064</span> <span class="o">|</span> <span class="mi">4611692774829951781</span> <span class="o">|</span> <span class="mi">9223363791697310021</span> <span class="o">|</span> <span class="mi">3031680</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">----------------+--------------+--------------------+---------------------+---------------------+-----------------------+</span>
+<span class="mi">3</span> <span class="k">rows</span> <span class="k">in</span> <span class="k">set</span><span class="p">.</span> <span class="n">Query</span> <span class="n">took</span> <span class="mi">0</span><span class="p">.</span><span class="mi">053</span> <span class="n">seconds</span><span class="p">.</span>
+</code></pre></div></div>
+
+<h3 id="growth-of-datafusion-">Growth of DataFusion 📈</h3>
+<p>DataFusion has been appearing more publically in the wild. For example</p>
+<ul>
+ <li>New projects built using DataFusion such as <a href="https://lancedb.com/">lancedb</a>, <a href="https://glaredb.com/">GlareDB</a>, <a href="https://www.arroyo.dev/">Arroyo</a>, and <a href="https://github.com/cmu-db/optd">optd</a>.</li>
+ <li>Public talks such as <a href="https://www.youtube.com/watch?v=AJU9rdRNk9I">Apache Arrow Datafusion: Vectorized
+Execution Framework For Maximum Performance</a> in <a href="https://www.bagevent.com/event/8432178">CommunityOverCode Asia 2023</a></li>
+ <li>Blogs posts such as <a href="https://www.synnada.ai/blog/apache-arrow-arrow-datafusion-ai-native-data-infra-an-interview-with-our-ceo-ozan">Apache Arrow, Arrow/DataFusion, AI-native Data Infra</a>,
+<a href="https://www.influxdata.com/blog/flight-datafusion-arrow-parquet-fdap-architecture-influxdb/">Flight, DataFusion, Arrow, and Parquet: Using the FDAP Architecture to build InfluxDB 3.0</a>, and
+<a href="https://www.linkedin.com/pulse/guide-user-defined-functions-apache-arrow-datafusion-dade-aderemi/">A Guide to User-Defined Functions in Apache Arrow DataFusion</a></li>
+</ul>
+
+<p>We have also <a href="https://github.com/apache/arrow-datafusion/issues/6782">submitted a paper</a> to <a href="https://2024.sigmod.org/">SIGMOD 2024</a>, one of the
+premiere database conferences, describing DataFusion in a technically formal
+style and making the case that it is possible to create a modular and extensive query engine
+without sacrificing performance. We hope this paper helps people
+evaluating DataFusion for their needs understand it better.</p>
+
+<h1 id="datafusion-in-2024-">DataFusion in 2024 🥳</h1>
+
+<p>Some major initiatives from contributors we know of this year are:</p>
+
+<ol>
+ <li>
+ <p><em>Modularity</em>: Make DataFusion even more modular, such as <a href="https://github.com/apache/arrow-datafusion/issues/8045">unifying
+built in and user functions</a>, making it easier to customize
+DataFusion’s behavior.</p>
+ </li>
+ <li>
+ <p><em>Community Growth</em>: Graduate to our own top level Apache project, and
+subsequently add more committers and PMC members to keep pace with project
+growth.</p>
+ </li>
+ <li>
+ <p><em>Use case white papers</em>: Write blog posts and videos explaining
+how to use DataFusion for real-world use cases.</p>
+ </li>
+ <li>
+ <p><em>Testing</em>: Improve CI infrastructure and test coverage, more fuzz
+testing, and better functional and performance regression testing.</p>
+ </li>
+ <li>
+ <p><em>Planning Time</em>: Reduce the time taken to plan queries, both <a href="https://github.com/apache/arrow-datafusion/issues/7698">wide
+tables of 1000s of columns</a>, and in <a href="https://github.com/apache/arrow-datafusion/issues/5637">general</a>.</p>
+ </li>
+ <li>
+ <p><em>Aggregate Performance</em>: Improve the speed of <a href="https://github.com/apache/arrow-datafusion/issues/7000">aggregating “high cardinality”</a> data
+when there are many (e.g. millions) of distinct groups.</p>
+ </li>
+ <li>
+ <p><em>Statistics</em>: <a href="https://github.com/apache/arrow-datafusion/issues/8227">Improved statistics handling</a> with an eye towards more
+sophisticated expression analysis and cost models.</p>
+ </li>
+</ol>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion we would love to have you
+join us. You can try out DataFusion on some of your own data and projects and
+let us know how it goes, contribute suggestions, documentation, bug reports, or
+a PR with documentation, tests or code. A list of open issues
+suitable for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>As the community grows, we are also looking to restart biweekly calls /
+meetings. Timezones are always a challenge for such meetings, but we hope to
+have two calls that can work for most attendees. If you are interested
+in helping, or just want to say hi, please drop us a note via one of
+the methods listed in our <a href="https://arrow.apache.org/datafusion/contributor-guide/communication.html">Communication Doc</a>.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0</title><link href="https://datafusion.apache.org/blog/blog/2023/08/05/datafusion_fast_grouping/" rel="alternate" type="text/html" t [...]
+
+-->
+
+<!-- Converted from Google Docs using https://www.buymeacoffee.com/docstomarkdown -->
+
+<h2 id="aggregating-millions-of-groups-fast-in-apache-arrow-datafusion">Aggregating Millions of Groups Fast in Apache Arrow DataFusion</h2>
+
+<p>Andrew Lamb, Daniël Heres, Raphael Taylor-Davies,</p>
+
+<p><em>Note: this article was originally published on the <a href="https://www.influxdata.com/blog/aggregating-millions-groups-fast-apache-arrow-datafusion">InfluxData Blog</a></em></p>
+
+<h2 id="tldr">TLDR</h2>
+
+<p>Grouped aggregations are a core part of any analytic tool, creating understandable summaries of huge data volumes. <a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a>’s parallel aggregation capability is 2-3x faster in the <a href="https://crates.io/crates/datafusion/28.0.0">newly released version <code class="language-plaintext highlighter-rouge">28.0.0</code></a> for queries with a large number (10,000 or more) of groups.</p>
+
+<p>Improving aggregation performance matters to all users of DataFusion. For example, both InfluxDB, a <a href="https://github.com/influxdata/influxdb">time series data platform</a> and Coralogix, a <a href="https://coralogix.com/?utm_source=InfluxDB&utm_medium=Blog&utm_campaign=organic">full-stack observability</a> platform, aggregate vast amounts of raw data to monitor and create insights for our customers. Improving DataFusion’s performance lets us provide better user experien [...]
+
+<p>With the new optimizations, DataFusion’s grouping speed is now close to DuckDB, a system that regularly reports <a href="https://duckdblabs.github.io/db-benchmark/">great</a> <a href="https://duckdb.org/2022/03/07/aggregate-hashtable.html#experiments">grouping</a> benchmark performance numbers. Figure 1 contains a representative sample of <a href="https://github.com/ClickHouse/ClickBench/tree/main">ClickBench</a> on a single Parquet file, and the full results are at the end of this ar [...]
+
+<p><img src="/blog/assets/datafusion_fast_grouping/summary.png" width="700" /></p>
+
+<p><strong>Figure 1</strong>: Query performance for ClickBench queries on queries 16, 17, 18 and 19 on a single Parquet file for DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code>.</p>
+
+<h2 id="introduction-to-high-cardinality-grouping">Introduction to high cardinality grouping</h2>
+
+<p>Aggregation is a fancy word for computing summary statistics across many rows that have the same value in one or more columns. We call the rows with the same values <em>groups</em> and “high cardinality” means there are a large number of distinct groups in the dataset. At the time of writing, a “large” number of groups in analytic engines is around 10,000.</p>
+
+<p>For example the <a href="https://github.com/ClickHouse/ClickBench">ClickBench</a> <em>hits</em> dataset contains 100 million anonymized user clicks across a set of websites. ClickBench Query 17 is:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="nv">"UserID"</span><span class="p">,</span> <span class="nv">"SearchPhrase"</span><span class="p">,</span> <span class="k">COUNT</span><span class="p">(</span><span class="o">*</span><span class="p">)</span>
+<span class="k">FROM</span> <span class="n">hits</span>
+<span class="k">GROUP</span> <span class="k">BY</span> <span class="nv">"UserID"</span><span class="p">,</span> <span class="nv">"SearchPhrase"</span>
+<span class="k">ORDER</span> <span class="k">BY</span> <span class="k">COUNT</span><span class="p">(</span><span class="o">*</span><span class="p">)</span>
+<span class="k">DESC</span> <span class="k">LIMIT</span> <span class="mi">10</span><span class="p">;</span>
+</code></pre></div></div>
+
+<p>In English, this query finds “the top ten (user, search phrase) combinations, across all clicks” and produces the following results (there are no search phrases for the top ten users):</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+---------------------+--------------+-----------------+
+| UserID | SearchPhrase | COUNT(UInt8(1)) |
++---------------------+--------------+-----------------+
+| 1313338681122956954 | | 29097 |
+| 1907779576417363396 | | 25333 |
+| 2305303682471783379 | | 10597 |
+| 7982623143712728547 | | 6669 |
+| 7280399273658728997 | | 6408 |
+| 1090981537032625727 | | 6196 |
+| 5730251990344211405 | | 6019 |
+| 6018350421959114808 | | 5990 |
+| 835157184735512989 | | 5209 |
+| 770542365400669095 | | 4906 |
++---------------------+--------------+-----------------+
+</code></pre></div></div>
+
+<p>The ClickBench dataset contains</p>
+
+<ul>
+ <li>99,997,497 total rows<sup id="fnref:1" role="doc-noteref"><a href="#fn:1" class="footnote" rel="footnote">1</a></sup></li>
+ <li>17,630,976 different users (distinct UserIDs)<sup id="fnref:2" role="doc-noteref"><a href="#fn:2" class="footnote" rel="footnote">2</a></sup></li>
+ <li>6,019,103 different search phrases<sup id="fnref:3" role="doc-noteref"><a href="#fn:3" class="footnote" rel="footnote">3</a></sup></li>
+ <li>24,070,560 distinct combinations<sup id="fnref:4" role="doc-noteref"><a href="#fn:4" class="footnote" rel="footnote">4</a></sup> of (UserID, SearchPhrase)
+Thus, to answer the query, DataFusion must map each of the 100M different input rows into one of the <strong>24 million different groups</strong>, and keep count of how many such rows there are in each group.</li>
+</ul>
+
+<h2 id="the-solution">The solution</h2>
+
+<p>Like most concepts in databases and other analytic systems, the basic ideas of this algorithm are straightforward and taught in introductory computer science courses. You could compute the query with a program such as this<sup id="fnref:5" role="doc-noteref"><a href="#fn:5" class="footnote" rel="footnote">5</a></sup>:</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kn">import</span> <span class="n">pandas</span> <span class="k">as</span> <span class="n">pd</span>
+<span class="kn">from</span> <span class="n">collections</span> <span class="kn">import</span> <span class="n">defaultdict</span>
+<span class="kn">from</span> <span class="n">operator</span> <span class="kn">import</span> <span class="n">itemgetter</span>
+
+<span class="c1"># read file
+</span><span class="n">hits</span> <span class="o">=</span> <span class="n">pd</span><span class="p">.</span><span class="nf">read_parquet</span><span class="p">(</span><span class="sh">'</span><span class="s">hits.parquet</span><span class="sh">'</span><span class="p">,</span> <span class="n">engine</span><span class="o">=</span><span class="sh">'</span><span class="s">pyarrow</span><span class="sh">'</span><span class="p">)</span>
+
+<span class="c1"># build groups
+</span><span class="n">counts</span> <span class="o">=</span> <span class="nf">defaultdict</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
+<span class="k">for</span> <span class="n">index</span><span class="p">,</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">hits</span><span class="p">.</span><span class="nf">iterrows</span><span class="p">():</span>
+ <span class="n">group</span> <span class="o">=</span> <span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="sh">'</span><span class="s">UserID</span><span class="sh">'</span><span class="p">],</span> <span class="n">row</span><span class="p">[</span><span class="sh">'</span><span class="s">SearchPhrase</span><span class="sh">'</span><span class="p">]);</span>
+ <span class="c1"># update the dict entry for the corresponding key
+</span> <span class="n">counts</span><span class="p">[</span><span class="n">group</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
+
+<span class="c1"># Print the top 10 values
+</span><span class="nf">print </span><span class="p">(</span><span class="nf">dict</span><span class="p">(</span><span class="nf">sorted</span><span class="p">(</span><span class="n">counts</span><span class="p">.</span><span class="nf">items</span><span class="p">(),</span> <span class="n">key</span><span class="o">=</span><span class="nf">itemgetter</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">reverse</span><span class="o">=</span><sp [...]
+</code></pre></div></div>
+
+<p>This approach, while simple, is both slow and very memory inefficient. It requires over 40 seconds to compute the results for less than 1% of the dataset<sup id="fnref:6" role="doc-noteref"><a href="#fn:6" class="footnote" rel="footnote">6</a></sup>. Both DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code> compute results in under 10 seconds for the <em>entire</em> dataset.</p>
+
+<p>To answer this query quickly and efficiently, you have to write your code such that it:</p>
+
+<ol>
+ <li>Keeps all cores busy aggregating via parallelized computation</li>
+ <li>Updates aggregate values quickly, using vectorizable loops that are easy for compilers to translate into the high performance <a href="https://en.wikipedia.org/wiki/Single_instruction,_multiple_data">SIMD</a> instructions available in modern CPUs.</li>
+</ol>
+
+<p>The rest of this article explains how grouping works in DataFusion and the improvements we made in <code class="language-plaintext highlighter-rouge">28.0.0</code>.</p>
+
+<h3 id="two-phase-parallel-partitioned-grouping">Two phase parallel partitioned grouping</h3>
+
+<p>Both DataFusion <code class="language-plaintext highlighter-rouge">27.0.</code> and <code class="language-plaintext highlighter-rouge">28.0.0</code> use state-of-the-art, two phase parallel hash partitioned grouping, similar to other high-performance vectorized engines like <a href="https://duckdb.org/2022/03/07/aggregate-hashtable.html">DuckDB’s Parallel Grouped Aggregates</a>. In pictures this looks like:</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ▲ ▲
+ │ │
+ │ │
+ │ │
+┌───────────────────────┐ ┌───────────────────┐
+│ GroupBy │ │ GroupBy │ Step 4
+│ (Final) │ │ (Final) │
+└───────────────────────┘ └───────────────────┘
+ ▲ ▲
+ │ │
+ └────────────┬───────────┘
+ │
+ │
+ ┌─────────────────────────┐
+ │ Repartition │ Step 3
+ │ HASH(x) │
+ └─────────────────────────┘
+ ▲
+ │
+ ┌────────────┴──────────┐
+ │ │
+ │ │
+ ┌────────────────────┐ ┌─────────────────────┐
+ │ GroupyBy │ │ GroupBy │ Step 2
+ │ (Partial) │ │ (Partial) │
+ └────────────────────┘ └─────────────────────┘
+ ▲ ▲
+ ┌──┘ └─┐
+ │ │
+ .─────────. .─────────.
+ ,─' '─. ,─' '─.
+; Input : ; Input : Step 1
+: Stream 1 ; : Stream 2 ;
+ ╲ ╱ ╲ ╱
+ '─. ,─' '─. ,─'
+ `───────' `───────'
+</code></pre></div></div>
+
+<p><strong>Figure 2</strong>: Two phase repartitioned grouping: data flows from bottom (source) to top (results) in two phases. First (Steps 1 and 2), each core reads the data into a core-specific hash table, computing intermediate aggregates without any cross-core coordination. Then (Steps 3 and 4) DataFusion divides the data (“repartitions”) into distinct subsets by group value, and each subset is sent to a specific core which computes the final aggregate.</p>
+
+<p>The two phases are critical for keeping cores busy in a multi-core system. Both phases use the same hash table approach (explained in the next section), but differ in how the groups are distributed and the partial results emitted from the accumulators. The first phase aggregates data as soon as possible after it is produced. However, as shown in Figure 2, the groups can be anywhere in any input, so the same group is often found on many different cores. The second phase uses a hash fun [...]
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌─────┐ ┌─────┐
+ │ 1 │ │ 3 │
+ │ 2 │ │ 4 │ 2. After Repartitioning: each
+ └─────┘ └─────┘ group key appears in exactly
+ ┌─────┐ ┌─────┐ one partition
+ │ 1 │ │ 3 │
+ │ 2 │ │ 4 │
+ └─────┘ └─────┘
+
+─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
+
+ ┌─────┐ ┌─────┐
+ │ 2 │ │ 2 │
+ │ 1 │ │ 2 │
+ │ 3 │ │ 3 │
+ │ 4 │ │ 1 │
+ └─────┘ └─────┘ 1. Input Stream: groups
+ ... ... values are spread
+ ┌─────┐ ┌─────┐ arbitrarily over each input
+ │ 1 │ │ 4 │
+ │ 4 │ │ 3 │
+ │ 1 │ │ 1 │
+ │ 4 │ │ 3 │
+ │ 3 │ │ 2 │
+ │ 2 │ │ 2 │
+ │ 2 │ └─────┘
+ └─────┘
+
+ Core A Core B
+
+</code></pre></div></div>
+
+<p><strong>Figure 3</strong>: Group value distribution across 2 cores during aggregation phases. In the first phase, every group value <code class="language-plaintext highlighter-rouge">1</code>, <code class="language-plaintext highlighter-rouge">2</code>, <code class="language-plaintext highlighter-rouge">3</code>, <code class="language-plaintext highlighter-rouge">4</code>, is present in the input stream processed by each core. In the second phase, after repartitioning, the group value [...]
+
+<p>There are some additional subtleties in the <a href="https://github.com/apache/arrow-datafusion/blob/main/datafusion/core/src/physical_plan/aggregates/row_hash.rs">DataFusion implementation</a> not mentioned above due to space constraints, such as:</p>
+
+<ol>
+ <li>The policy of when to emit data from the first phase’s hash table (e.g. because the data is partially sorted)</li>
+ <li>Handling specific filters per aggregate (due to the <code class="language-plaintext highlighter-rouge">FILTER</code> SQL clause)</li>
+ <li>Data types of intermediate values (which may not be the same as the final output for some aggregates such as <code class="language-plaintext highlighter-rouge">AVG</code>).</li>
+ <li>Action taken when memory use exceeds its budget.</li>
+</ol>
+
+<h3 id="hash-grouping">Hash grouping</h3>
+
+<p>DataFusion queries can compute many different aggregate functions for each group, both <a href="https://arrow.apache.org/datafusion/user-guide/sql/aggregate_functions.html">built in</a> and/or user defined <a href="https://docs.rs/datafusion/latest/datafusion/logical_expr/struct.AggregateUDF.html"><code class="language-plaintext highlighter-rouge">AggregateUDFs</code></a>. The state for each aggregate function, called an <em>accumulator</em>, is tracked with a hash table (DataFusion u [...]
+
+<h3 id="hash-grouping-in-2700">Hash grouping in <code class="language-plaintext highlighter-rouge">27.0.0</code></h3>
+
+<p>As shown in Figure 3, DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code> stores the data in a <a href="https://github.com/apache/arrow-datafusion/blob/4d93b6a3802151865b68967bdc4c7d7ef425b49a/datafusion/core/src/physical_plan/aggregates/utils.rs#L38-L50"><code class="language-plaintext highlighter-rouge">GroupState</code></a> structure which, unsurprisingly, tracks the state for each group. The state for each group consists of:</p>
+
+<ol>
+ <li>The actual value of the group columns, in <a href="https://docs.rs/arrow-row/latest/arrow_row/index.html">Arrow Row</a> format.</li>
+ <li>In-progress accumulations (e.g. the running counts for the <code class="language-plaintext highlighter-rouge">COUNT</code> aggregate) for each group, in one of two possible formats (<a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/expr/src/accumulator.rs#L24-L49"><code class="language-plaintext highlighter-rouge">Accumulator</code></a> or <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b42 [...]
+ <li>Scratch space for tracking which rows match each aggregate in each batch.</li>
+</ol>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌──────────────────────────────────────┐
+ │ │
+ │ ... │
+ │ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ │
+ │ ┃ ┃ │
+ ┌─────────┐ │ ┃ ┌──────────────────────────────┐ ┃ │
+ │ │ │ ┃ │group values: OwnedRow │ ┃ │
+ │ ┌─────┐ │ │ ┃ └──────────────────────────────┘ ┃ │
+ │ │ 5 │ │ │ ┃ ┌──────────────────────────────┐ ┃ │
+ │ ├─────┤ │ │ ┃ │Row accumulator: │ ┃ │
+ │ │ 9 │─┼────┐ │ ┃ │Vec<u8> │ ┃ │
+ │ ├─────┤ │ │ │ ┃ └──────────────────────────────┘ ┃ │
+ │ │ ... │ │ │ │ ┃ ┌──────────────────────┐ ┃ │
+ │ ├─────┤ │ │ │ ┃ │┌──────────────┐ │ ┃ │
+ │ │ 1 │ │ │ │ ┃ ││Accumulator 1 │ │ ┃ │
+ │ ├─────┤ │ │ │ ┃ │└──────────────┘ │ ┃ │
+ │ │ ... │ │ │ │ ┃ │┌──────────────┐ │ ┃ │
+ │ └─────┘ │ │ │ ┃ ││Accumulator 2 │ │ ┃ │
+ │ │ │ │ ┃ │└──────────────┘ │ ┃ │
+ └─────────┘ │ │ ┃ │ Box<dyn Accumulator> │ ┃ │
+ Hash Table │ │ ┃ └──────────────────────┘ ┃ │
+ │ │ ┃ ┌─────────────────────────┐ ┃ │
+ │ │ ┃ │scratch indices: Vec<u32>│ ┃ │
+ │ │ ┃ └─────────────────────────┘ ┃ │
+ │ │ ┃ GroupState ┃ │
+ └─────▶ │ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ │
+ │ │
+ Hash table tracks an │ ... │
+ index into group_states │ │
+ └──────────────────────────────────────┘
+ group_states: Vec<GroupState>
+
+ There is one GroupState PER GROUP
+
+</code></pre></div></div>
+
+<p><strong>Figure 4</strong>: Hash group operator structure in DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>. A hash table maps each group to a GroupState which contains all the per-group states.</p>
+
+<p>To compute the aggregate, DataFusion performs the following steps for each input batch:</p>
+
+<ol>
+ <li>Calculate hash using <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/physical-expr/src/hash_utils.rs#L264-L307">efficient vectorized code</a>, specialized for each data type.</li>
+ <li>Determine group indexes for each input row using the hash table (creating new entries for newly seen groups).</li>
+ <li><a href="https://github.com/apache/arrow-datafusion/blob/4ab8be57dee3bfa72dd105fbd7b8901b873a4878/datafusion/core/src/physical_plan/aggregates/row_hash.rs#L562-L602">Update Accumulators for each group that had input rows,</a> assembling the rows into a contiguous range for vectorized accumulator if there are a sufficient number of them.</li>
+</ol>
+
+<p>DataFusion also stores the hash values in the table to avoid potentially costly hash recomputation when resizing the hash table.</p>
+
+<p>This scheme works very well for a relatively small number of distinct groups: all accumulators are efficiently updated with large contiguous batches of rows.</p>
+
+<p>However, this scheme is not ideal for high cardinality grouping due to:</p>
+
+<ol>
+ <li><strong>Multiple allocations per group</strong> for the group value row format, as well as for the <code class="language-plaintext highlighter-rouge">RowAccumulator</code>s and each <code class="language-plaintext highlighter-rouge">Accumulator</code>. The <code class="language-plaintext highlighter-rouge">Accumulator</code> may have additional allocations within it as well.</li>
+ <li><strong>Non-vectorized updates:</strong> Accumulator updates often fall back to a slower non-vectorized form because the number of distinct groups is large (and thus number of values per group is small) in each input batch.</li>
+</ol>
+
+<h3 id="hash-grouping-in-2800">Hash grouping in <code class="language-plaintext highlighter-rouge">28.0.0</code></h3>
+
+<p>For <code class="language-plaintext highlighter-rouge">28.0.0</code>, we rewrote the core group by implementation following traditional system optimization principles: fewer allocations, type specialization, and aggressive vectorization.</p>
+
+<p>DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> uses the same RawTable and still stores group indexes. The major differences, as shown in Figure 4, are:</p>
+
+<ol>
+ <li>Group values are stored either
+ <ol>
+ <li>Inline in the <code class="language-plaintext highlighter-rouge">RawTable</code> (for single columns of primitive types), where the conversion to Row format costs more than its benefit</li>
+ <li>In a separate <a href="https://docs.rs/arrow-row/latest/arrow_row/struct.Row.html">Rows</a> structure with a single contiguous allocation for all groups values, rather than an allocation per group. Accumulators manage the state for all the groups internally, so the code to update intermediate values is a tight type specialized loop. The new <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/physical-expr/src/aggregate/gr [...]
+ </ol>
+ </li>
+</ol>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>┌───────────────────────────────────┐ ┌───────────────────────┐
+│ ┌ ─ ─ ─ ─ ─ ┐ ┌─────────────────┐│ │ ┏━━━━━━━━━━━━━━━━━━━┓ │
+│ │ ││ │ ┃ ┌──────────────┐ ┃ │
+│ │ │ │ ┌ ─ ─ ┐┌─────┐ ││ │ ┃ │┌───────────┐ │ ┃ │
+│ │ X │ 5 │ ││ │ ┃ ││ value1 │ │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ │ ┃ │└───────────┘ │ ┃ │
+│ │ Q │ 9 │──┼┼──┐ │ ┃ │ ... │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ └──┼─╋─▶│ │ ┃ │
+│ │ ... │ ... │ ││ │ ┃ │┌───────────┐ │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ │ ┃ ││ valueN │ │ ┃ │
+│ │ H │ 1 │ ││ │ ┃ │└───────────┘ │ ┃ │
+│ │ │ │ ├ ─ ─ ┤├─────┤ ││ │ ┃ │values: Vec<T>│ ┃ │
+│ Rows │ ... │ ... │ ││ │ ┃ └──────────────┘ ┃ │
+│ │ │ │ └ ─ ─ ┘└─────┘ ││ │ ┃ ┃ │
+│ ─ ─ ─ ─ ─ ─ │ ││ │ ┃ GroupsAccumulator ┃ │
+│ └─────────────────┘│ │ ┗━━━━━━━━━━━━━━━━━━━┛ │
+│ Hash Table │ │ │
+│ │ │ ... │
+└───────────────────────────────────┘ └───────────────────────┘
+ GroupState Accumulators
+
+
+Hash table value stores group_indexes One GroupsAccumulator
+and group values. per aggregate. Each
+ stores the state for
+Group values are stored either inline *ALL* groups, typically
+in the hash table or in a single using a native Vec<T>
+allocation using the arrow Row format
+</code></pre></div></div>
+
+<p><strong>Figure 5</strong>: Hash group operator structure in DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code>. Group values are stored either directly in the hash table, or in a single allocation using the arrow Row format. The hash table contains group indexes. A single <code class="language-plaintext highlighter-rouge">GroupsAccumulator</code> stores the per-aggregate state for <em>all</em> groups.</p>
+
+<p>This new structure improves performance significantly for high cardinality groups due to:</p>
+
+<ol>
+ <li><strong>Reduced allocations</strong>: There are no longer any individual allocations per group.</li>
+ <li><strong>Contiguous native accumulator states</strong>: Type-specialized accumulators store the values for all groups in a single contiguous allocation using a <a href="https://doc.rust-lang.org/std/vec/struct.Vec.html">Rust Vec<T></a> of some native type.</li>
+ <li><strong>Vectorized state update</strong>: The inner aggregate update loops, which are type-specialized and in terms of native <code class="language-plaintext highlighter-rouge">Vec</code>s, are well-vectorized by the Rust compiler (thanks <a href="https://llvm.org/">LLVM</a>!).</li>
+</ol>
+
+<h3 id="notes">Notes</h3>
+
+<p>Some vectorized grouping implementations store the accumulator state row-wise directly in the hash table, which often uses modern CPU caches efficiently. Managing accumulator state in columnar fashion may sacrifice some cache locality, however it ensures the size of the hash table remains small, even when there are large numbers of groups and aggregates, making it easier for the compiler to vectorize the accumulator update.</p>
+
+<p>Depending on the cost of recomputing hash values, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> may or may not store the hash values in the table. This optimizes the tradeoff between the cost of computing the hash value (which is expensive for strings, for example) vs. the cost of storing it in the hash table.</p>
+
+<p>One subtlety that arises from pushing state updates into GroupsAccumulators is that each accumulator must handle similar variations with/without filtering and with/without nulls in the input. DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> uses a templated <a href="https://github.com/apache/arrow-datafusion/blob/a6dcd943051a083693c352c6b4279156548490a0/datafusion/physical-expr/src/aggregate/groups_accumulator/accumulate.rs#L28-L54"><code class="language-pla [...]
+
+<p>The code structure is heavily influenced by the fact DataFusion is implemented using <a href="https://www.rust-lang.org/">Rust</a>, a new(ish) systems programming language focused on speed and safety. Rust heavily discourages many of the traditional pointer casting “tricks” used in C/C++ hash grouping implementations. The DataFusion aggregation code is almost entirely <a href="https://doc.rust-lang.org/nomicon/meet-safe-and-unsafe.html#:~:text=Safe%20Rust%20is%20the%20true,Undefined%2 [...]
+
+<h2 id="clickbench-results">ClickBench results</h2>
+
+<p>The full results of running the <a href="https://github.com/ClickHouse/ClickBench/tree/main">ClickBench</a> queries against the single Parquet file with DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code>, and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code> are below. These numbers were run on a GCP <code class="language-plaintext highlighter-rouge">e2-standard-8 [...]
+
+<p>As the industry moves towards data systems assembled from components, it is increasingly important that they exchange data using open standards such as <a href="https://arrow.apache.org/">Apache Arrow</a> and <a href="https://parquet.apache.org/">Parquet</a> rather than custom storage and in-memory formats. Thus, this benchmark uses a single input Parquet file representative of many DataFusion users and aligned with the current trend in analytics of avoiding a costly load/transformati [...]
+
+<p>DataFusion now reaches near-DuckDB-speeds querying Parquet data. While we don’t plan to engage in a benchmarking shootout with a team that literally wrote <a href="https://dl.acm.org/doi/abs/10.1145/3209950.3209955">Fair Benchmarking Considered Difficult</a>, hopefully everyone can agree that DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> is a significant improvement.</p>
+
+<p><img src="/blog/assets/datafusion_fast_grouping/full.png" width="700" /></p>
+
+<p><strong>Figure 6</strong>: Performance of DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code>, DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code>, and DuckDB <code class="language-plaintext highlighter-rouge">0.8.1</code> on all 43 ClickBench queries against a single <code class="language-plaintext highlighter-rouge">hits.parquet</code> file. Lower is better.</p>
+
+<h3 id="notes-1">Notes</h3>
+
+<p>DataFusion <code class="language-plaintext highlighter-rouge">27.0.0</code> was not able to run several queries due to either planner bugs (Q9, Q11, Q12, 14) or running out of memory (Q33). DataFusion <code class="language-plaintext highlighter-rouge">28.0.0</code> solves those issues.</p>
+
+<p>DataFusion is faster than DuckDB for query 21 and 22, likely due to optimized implementations of string pattern matching.</p>
+
+<h2 id="conclusion-performance-matters">Conclusion: performance matters</h2>
+
+<p>Improving aggregation performance by more than a factor of two allows developers building products and projects with DataFusion to spend more time on value-added domain specific features. We believe building systems with DataFusion is much faster than trying to build something similar from scratch. DataFusion increases productivity because it eliminates the need to rebuild well-understood, but costly to implement, analytic database technology. While we’re pleased with the improvements [...]
+
+<h2 id="acknowledgments">Acknowledgments</h2>
+
+<p>DataFusion is a <a href="https://arrow.apache.org/datafusion/contributor-guide/communication.html">community effort</a> and this work was not possible without contributions from many in the community. A special shout out to <a href="https://github.com/sunchao">sunchao</a>, <a href="https://github.com/jyshen">yjshen</a>, <a href="https://github.com/yahoNanJing">yahoNanJing</a>, <a href="https://github.com/mingmwang">mingmwang</a>, <a href="https://github.com/ozankabak">ozankabak</a>, < [...]
+
+<h2 id="about-datafusion">About DataFusion</h2>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> is an extensible query engine and database toolkit, written in <a href="https://www.rust-lang.org/">Rust</a>, that uses <a href="https://arrow.apache.org/">Apache Arrow</a> as its in-memory format. DataFusion, along with <a href="https://calcite.apache.org/">Apache Calcite</a>, Facebook’s <a href="https://github.com/facebookincubator/velox">Velox</a>, and similar technology are part of the next generation “<a h [...]
+
+<!-- Footnotes themselves at the bottom. -->
+<h2 id="notes-2">Notes</h2>
+
+<div class="footnotes" role="doc-endnotes">
+ <ol>
+ <li id="fn:1" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(*) FROM 'hits.parquet';</code> <a href="#fnref:1" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:2" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(DISTINCT "UserID") as num_users FROM 'hits.parquet';</code> <a href="#fnref:2" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:3" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(DISTINCT "SearchPhrase") as num_phrases FROM 'hits.parquet';</code> <a href="#fnref:3" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:4" role="doc-endnote">
+ <p><code class="language-plaintext highlighter-rouge">SELECT COUNT(*) FROM (SELECT DISTINCT "UserID", "SearchPhrase" FROM 'hits.parquet')</code> <a href="#fnref:4" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:5" role="doc-endnote">
+ <p>Full script at <a href="https://github.com/alamb/datafusion-duckdb-benchmark/blob/main/hash.py">hash.py</a> <a href="#fnref:5" class="reversefootnote" role="doc-backlink">↩</a></p>
+ </li>
+ <li id="fn:6" role="doc-endnote">
+ <p><a href="https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_%7B%7D.parquet">hits_0.parquet</a>, one of the files from the partitioned ClickBench dataset, which has <code class="language-plaintext highlighter-rouge">100,000</code> rows and is 117 MB in size. The entire dataset has <code class="language-plaintext highlighter-rouge">100,000,000</code> rows in a single 14 GB Parquet file. The script did not complete on the entire dataset after 40 minutes, and us [...]
+ </li>
+ </ol>
+</div>]]></content><author><name>alamb, Dandandan, tustvold</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Apache Arrow DataFusion 26.0.0</title><link href="https://datafusion.apache.org/blog/blog/2023/06/24/datafusion-25.0.0/" rel="alternate" type="text/html" title="Apache Arrow DataFusion 26.0.0" /><published>2023-06-24T00:00:00-06:00</published><updated>2023-06-24T00:00:00-06:00</updated><id>https://datafusi [...]
+
+-->
+
+<p>It has been a whirlwind 6 months of DataFusion development since <a href="https://arrow.apache.org/blog/2023/01/19/datafusion-16.0.0">our
+last update</a>: the community has grown, many features have been added,
+performance improved and we are <a href="https://github.com/apache/arrow-datafusion/discussions/6475">discussing</a> branching out to our own
+top level Apache Project.</p>
+
+<h2 id="background">Background</h2>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> is an extensible query engine and database
+toolkit, written in <a href="https://www.rust-lang.org/">Rust</a>, that uses <a href="https://arrow.apache.org">Apache Arrow</a> as its in-memory
+format.</p>
+
+<p>DataFusion, along with <a href="https://calcite.apache.org">Apache Calcite</a>, Facebook’s <a href="https://github.com/facebookincubator/velox">Velox</a> and
+similar technology are part of the next generation “<a href="https://www.usenix.org/publications/login/winter2018/khurana">Deconstructed
+Database</a>” architectures, where new systems are built on a foundation
+of fast, modular components, rather as a single tightly integrated
+system.</p>
+
+<p>While single tightly integrated systems such as <a href="https://spark.apache.org/">Spark</a>, <a href="https://duckdb.org">DuckDB</a> and
+<a href="https://www.pola.rs/">Pola.rs</a> are great pieces of technology, our community believes that
+anyone developing new data heavy application, such as those common in
+machine learning in the next 5 years, will <strong>require</strong> a high
+performance, vectorized, query engine to remain relevant. The only
+practical way to gain access to such technology without investing many
+millions of dollars to build a new tightly integrated engine, is
+though open source projects like DataFusion and similar enabling
+technologies such as <a href="https://arrow.apache.org">Apache Arrow</a> and <a href="https://www.rust-lang.org/">Rust</a>.</p>
+
+<p>DataFusion is targeted primarily at developers creating other data
+intensive analytics, and offers:</p>
+
+<ul>
+ <li>High performance, native, parallel streaming execution engine</li>
+ <li>Mature <a href="https://arrow.apache.org/datafusion/user-guide/sql/index.html">SQL support</a>, featuring subqueries, window functions, grouping sets, and more</li>
+ <li>Built in support for Parquet, Avro, CSV, JSON and Arrow formats and easy extension for others</li>
+ <li>Native DataFrame API and <a href="https://arrow.apache.org/datafusion-python/">python bindings</a></li>
+ <li><a href="https://docs.rs/datafusion/latest/datafusion/index.html">Well documented</a> source code and architecture, designed to be customized to suit downstream project needs</li>
+ <li>High quality, easy to use code <a href="https://crates.io/crates/datafusion/versions">released every 2 weeks to crates.io</a></li>
+ <li>Welcoming, open community, governed by the highly regarded and well understood <a href="https://www.apache.org/">Apache Software Foundation</a></li>
+</ul>
+
+<p>The rest of this post highlights some of the improvements we have made
+to DataFusion over the last 6 months and a preview of where we are
+heading. You can see a list of all changes in the detailed
+<a href="https://github.com/apache/arrow-datafusion/blob/main/datafusion/CHANGELOG.md">CHANGELOG</a>.</p>
+
+<h2 id="even-better-performance">(Even) Better Performance</h2>
+
+<p><a href="https://voltrondata.com/resources/speeds-and-feeds-hardware-and-software-matter">Various</a> benchmarks show DataFusion to be quite close or <a href="https://github.com/tustvold/access-log-bench">even
+faster</a> to the state of the art in analytic performance (at the moment
+this seems to be DuckDB). We continually work on improving performance
+(see <a href="https://github.com/apache/arrow-datafusion/issues/5546">#5546</a> for a list) and would love additional help in this area.</p>
+
+<p>DataFusion now reads single large Parquet files significantly faster by
+<a href="https://github.com/apache/arrow-datafusion/pull/5057">parallelizing across multiple cores</a>. Native speeds for reading JSON
+and CSV files are also up to 2.5x faster thanks to improvements
+upstream in arrow-rs <a href="https://github.com/apache/arrow-rs/pull/3479#issuecomment-1384353159">JSON reader</a> and <a href="https://github.com/apache/arrow-rs/pull/3365">CSV reader</a>.</p>
+
+<p>Also, we have integrated the <a href="https://arrow.apache.org/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">arrow-rs Row Format</a> into DataFusion resulting in up to <a href="https://github.com/apache/arrow-datafusion/pull/6163">2-3x faster sorting and merging</a>.</p>
+
+<h2 id="improved-documentation-and-website">Improved Documentation and Website</h2>
+
+<p>Part of growing the DataFusion community is ensuring that DataFusion’s
+features are understood and that it is easy to contribute and
+participate. To that end the <a href="https://arrow.apache.org/datafusion/">website</a> has been cleaned up, <a href="https://docs.rs/datafusion/latest/datafusion/index.html#architecture">the
+architecture guide</a> expanded, the <a href="https://arrow.apache.org/datafusion/contributor-guide/roadmap.html">roadmap</a> updated, and several
+overview talks created:</p>
+
+<ul>
+ <li>Apr 2023 <em>Query Engine</em>: <a href="https://youtu.be/NVKujPxwSBA">recording</a> and <a href="https://docs.google.com/presentation/d/1D3GDVas-8y0sA4c8EOgdCvEjVND4s2E7I6zfs67Y4j8/edit#slide=id.p">slides</a></li>
+ <li>April 2023 <em>Logical Plan and Expressions</em>: <a href="https://youtu.be/EzZTLiSJnhY">recording</a> and <a href="https://docs.google.com/presentation/d/1ypylM3-w60kVDW7Q6S99AHzvlBgciTdjsAfqNP85K30">slides</a></li>
+ <li>April 2023 <em>Physical Plan and Execution</em>: <a href="https://youtu.be/2jkWU3_w6z0">recording</a> and <a href="https://docs.google.com/presentation/d/1cA2WQJ2qg6tx6y4Wf8FH2WVSm9JQ5UgmBWATHdik0hg">slides</a></li>
+</ul>
+
+<h2 id="new-features">New Features</h2>
+
+<h3 id="more-streaming-less-memory">More Streaming, Less Memory</h3>
+
+<p>We have made significant progress on the <a href="https://github.com/apache/arrow-datafusion/issues/4285">streaming execution roadmap</a>
+such as <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/trait.ExecutionPlan.html#method.unbounded_output">unbounded datasources</a>, <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/aggregates/enum.GroupByOrderMode.html">streaming group by</a>, sophisticated
+<a href="https://docs.rs/datafusion/latest/datafusion/physical_optimizer/global_sort_selection/index.html">sort</a> and <a href="https://docs.rs/datafusion/latest/datafusion/physical_optimizer/repartition/index.html">repartitioning</a> improvements in the optimizer, and support
+for <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/joins/struct.SymmetricHashJoinExec.html">symmetric hash join</a> (read more about that in the great <a href="https://www.synnada.ai/blog/general-purpose-stream-joins-via-pruning-symmetric-hash-joins">Synnada
+Blog Post</a> on the topic). Together, these features both 1) make it
+easier to build streaming systems using DataFusion that can
+incrementally generate output before (or ever) seeing the end of the
+input and 2) allow general queries to use less memory and generate their
+results faster.</p>
+
+<p>We have also improved the runtime <a href="https://docs.rs/datafusion/latest/datafusion/execution/memory_pool/index.html">memory management</a> system so that
+DataFusion now stays within its declared memory budget <a href="https://github.com/apache/arrow-datafusion/issues/3941">generate
+runtime errors</a>.</p>
+
+<h3 id="dml-support-insert-delete-update-etc">DML Support (<code class="language-plaintext highlighter-rouge">INSERT</code>, <code class="language-plaintext highlighter-rouge">DELETE</code>, <code class="language-plaintext highlighter-rouge">UPDATE</code>, etc)</h3>
+
+<p>Part of building high performance data systems includes writing data,
+and DataFusion supports several features for creating new files:</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">INSERT INTO</code> and <code class="language-plaintext highlighter-rouge">SELECT ... INTO </code> support for memory backed and CSV tables</li>
+ <li>New <a href="https://docs.rs/datafusion/latest/datafusion/physical_plan/insert/trait.DataSink.html">API for writing data into TableProviders</a></li>
+</ul>
+
+<p>We are working on easier to use <a href="https://github.com/apache/arrow-datafusion/issues/5654">COPY INTO</a> syntax, better support
+for writing parquet, JSON, and AVRO, and more – see our <a href="https://github.com/apache/arrow-datafusion/issues/6569">tracking epic</a>
+for more details.</p>
+
+<h3 id="timestamp-and-intervals">Timestamp and Intervals</h3>
+
+<p>One mark of the maturity of a SQL engine is how it handles the tricky
+world of timestamp, date, times and interval arithmetic. DataFusion is
+feature complete in this area and behaves as you would expect,
+supporting queries such as</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="n">now</span><span class="p">()</span> <span class="o">+</span> <span class="s1">'1 month'</span> <span class="k">FROM</span> <span class="n">my_table</span><span class="p">;</span>
+</code></pre></div></div>
+
+<p>We still have a long tail of <a href="https://github.com/apache/arrow-datafusion/issues/3148">date and time improvements</a>, which we are working on as well.</p>
+
+<h3 id="querying-structured-types-list-and-structs">Querying Structured Types (<code class="language-plaintext highlighter-rouge">List</code> and <code class="language-plaintext highlighter-rouge">Struct</code>s)</h3>
+
+<p>Arrow and Parquet <a href="https://arrow.apache.org/blog/2022/10/08/arrow-parquet-encoding-part-2/">support nested data</a> well and DataFusion lets you
+easily query such <code class="language-plaintext highlighter-rouge">Struct</code> and <code class="language-plaintext highlighter-rouge">List</code>. For example, you can use
+DataFusion to read and query the <a href="https://data.mendeley.com/datasets/ct8f9skv97">JSON Datasets for Exploratory OLAP -
+Mendeley Data</a> like this:</p>
+
+<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c1">----------</span>
+<span class="c1">-- Explore structured data using SQL</span>
+<span class="c1">----------</span>
+<span class="k">SELECT</span> <span class="k">delete</span> <span class="k">FROM</span> <span class="s1">'twitter-sample-head-100000.parquet'</span> <span class="k">WHERE</span> <span class="k">delete</span> <span class="k">IS</span> <span class="k">NOT</span> <span class="k">NULL</span> <span class="k">limit</span> <span class="mi">10</span><span class="p">;</span>
+<span class="o">+</span><span class="c1">---------------------------------------------------------------------------------------------------------------------------+</span>
+<span class="o">|</span> <span class="k">delete</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">---------------------------------------------------------------------------------------------------------------------------+</span>
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">135037425050320896</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">135037425050320896</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134703982051463168</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134703982051463168</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134773741740765184</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134773741740765184</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">132543659655704576</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">132543659655704576</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">133786431926697984</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">133786431926697984</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134619093570560002</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134619093570560002</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134019857527214080</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134019857527214080</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">133931546469076993</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">133931546469076993</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">134397743350296576</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">134397743350296576</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">|</span> <span class="p">{</span><span class="n">status</span><span class="p">:</span> <span class="p">{</span><span class="n">id</span><span class="p">:</span> <span class="p">{</span><span class="err">$</span><span class="n">numberLong</span><span class="p">:</span> <span class="mi">127833661767823360</span><span class="p">},</span> <span class="n">id_str</span><span class="p">:</span> <span class="mi">127833661767823360</span><span class="p">,</span> <span class="n">us [...]
+<span class="o">+</span><span class="c1">---------------------------------------------------------------------------------------------------------------------------+</span>
+
+<span class="c1">----------</span>
+<span class="c1">-- Select some deeply nested fields</span>
+<span class="c1">----------</span>
+<span class="k">SELECT</span>
+ <span class="k">delete</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'id'</span><span class="p">][</span><span class="s1">'$numberLong'</span><span class="p">]</span> <span class="k">as</span> <span class="n">delete_id</span><span class="p">,</span>
+ <span class="k">delete</span><span class="p">[</span><span class="s1">'status'</span><span class="p">][</span><span class="s1">'user_id'</span><span class="p">]</span> <span class="k">as</span> <span class="n">delete_user_id</span>
+<span class="k">FROM</span> <span class="s1">'twitter-sample-head-100000.parquet'</span> <span class="k">WHERE</span> <span class="k">delete</span> <span class="k">IS</span> <span class="k">NOT</span> <span class="k">NULL</span> <span class="k">LIMIT</span> <span class="mi">10</span><span class="p">;</span>
+
+<span class="o">+</span><span class="c1">--------------------+----------------+</span>
+<span class="o">|</span> <span class="n">delete_id</span> <span class="o">|</span> <span class="n">delete_user_id</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">--------------------+----------------+</span>
+<span class="o">|</span> <span class="mi">135037425050320896</span> <span class="o">|</span> <span class="mi">334902461</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134703982051463168</span> <span class="o">|</span> <span class="mi">405383453</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134773741740765184</span> <span class="o">|</span> <span class="mi">64823441</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">132543659655704576</span> <span class="o">|</span> <span class="mi">45917834</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">133786431926697984</span> <span class="o">|</span> <span class="mi">67229952</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134619093570560002</span> <span class="o">|</span> <span class="mi">182430773</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134019857527214080</span> <span class="o">|</span> <span class="mi">257396311</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">133931546469076993</span> <span class="o">|</span> <span class="mi">124539548</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">134397743350296576</span> <span class="o">|</span> <span class="mi">139836391</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">127833661767823360</span> <span class="o">|</span> <span class="mi">244442687</span> <span class="o">|</span>
+<span class="o">+</span><span class="c1">--------------------+----------------+</span>
+</code></pre></div></div>
+
+<h3 id="subqueries-all-the-way-down">Subqueries All the Way Down</h3>
+
+<p>DataFusion can run many different subqueries by rewriting them to
+joins. It has been able to run the full suite of TPC-H queries for at
+least the last year, but recently we have implemented significant
+improvements to this logic, sufficient to run almost all queries in
+the TPC-DS benchmark as well.</p>
+
+<h2 id="community-and-project-growth">Community and Project Growth</h2>
+
+<p>The six months since <a href="https://arrow.apache.org/blog/2023/01/19/datafusion-16.0.0">our last update</a> saw significant growth in
+the DataFusion community. Between versions <code class="language-plaintext highlighter-rouge">17.0.0</code> and <code class="language-plaintext highlighter-rouge">26.0.0</code>,
+DataFusion merged 711 PRs from 107 distinct contributors, not
+including all the work that goes into our core dependencies such as
+<a href="https://crates.io/crates/arrow">arrow</a>,
+<a href="https://crates.io/crates/parquet">parquet</a>, and
+<a href="https://crates.io/crates/object_store">object_store</a>, that much of
+the same community helps support.</p>
+
+<p>In addition, we have added 7 new committers and 1 new PMC member to
+the Apache Arrow project, largely focused on DataFusion, and we
+learned about some of the cool <a href="https://arrow.apache.org/datafusion/user-guide/introduction.html#known-users">new systems</a> which are using
+DataFusion. Given the growth of the community and interest in the
+project, we also clarified the <a href="https://github.com/apache/arrow-datafusion/discussions/6441">mission statement</a> and are
+<a href="https://github.com/apache/arrow-datafusion/discussions/6475">discussing</a> “graduate”ing DataFusion to a new top level
+Apache Software Foundation project.</p>
+
+<!--
+$ git log --pretty=oneline 17.0.0..26.0.0 . | wc -l
+ 711
+
+$ git shortlog -sn 17.0.0..26.0.0 . | wc -l
+ 107
+-->
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>Kudos to everyone in the community who has contributed ideas,
+discussions, bug reports, documentation and code. It is exciting to be
+innovating on the next generation of database architectures together!</p>
+
+<p>If you are interested in contributing to DataFusion, we would love to
+have you join us. You can try out DataFusion on some of your own
+data and projects and let us know how it goes or contribute a PR with
+documentation, tests or code. A list of open issues suitable for
+beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>Check out our <a href="https://arrow.apache.org/datafusion/contributor-guide/communication.html">Communication Doc</a> for more ways to engage with the
+community.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Apache Arrow DataFusion 16.0.0 Project Update</title><link href="https://datafusion.apache.org/blog/blog/2023/01/19/datafusion-16.0.0/" rel="alternate" type="text/html" title="Apache Arrow DataFusion 16.0.0 Project Update" /><published>2023-01-19T00:00:00-07:00</published><updated>2023-01-19T00:00:00-07:00</updated><id>h [...]
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible
+query execution framework, written in <a href="https://www.rust-lang.org/">Rust</a>,
+that uses <a href="https://arrow.apache.org">Apache Arrow</a> as its
+in-memory format. It is targeted primarily at developers creating data
+intensive analytics, and offers mature
+<a href="https://arrow.apache.org/datafusion/user-guide/sql/index.html">SQL support</a>,
+a DataFrame API, and many extension points.</p>
+
+<p>Systems based on DataFusion perform very well in benchmarks,
+especially considering they operate directly on parquet files rather
+than first loading into a specialized format. Some recent highlights
+include <a href="https://benchmark.clickhouse.com/">clickbench</a> and the
+<a href="https://www.cloudfuse.io/dashboards/standalone-engines">Cloudfuse.io standalone query
+engines</a> page.</p>
+
+<p>DataFusion is also part of a longer term trend, articulated clearly by
+<a href="http://www.cs.cmu.edu/~pavlo/">Andy Pavlo</a> in his <a href="https://ottertune.com/blog/2022-databases-retrospective/">2022 Databases
+Retrospective</a>.
+Database frameworks are proliferating and it is likely that all OLAP
+DBMSs and other data heavy applications, such as machine learning,
+will <strong>require</strong> a vectorized, highly performant query engine in the next
+5 years to remain relevant. The only practical way to make such
+technology so widely available without many millions of dollars of
+investment is though open source engine such as DataFusion or
+<a href="https://github.com/facebookincubator/velox">Velox</a>.</p>
+
+<p>The rest of this post describes the improvements made to DataFusion
+over the last three months and some hints of where we are heading.</p>
+
+<h2 id="community-growth">Community Growth</h2>
+
+<p>We again saw significant growth in the DataFusion community since <a href="https://arrow.apache.org/blog/2022/10/25/datafusion-13.0.0/">our last update</a>. There are some interesting metrics on <a href="https://ossrank.com/p/1573-apache-arrow-datafusion">OSSRank</a>.</p>
+
+<p>The DataFusion 16.0.0 release consists of 543 PRs from 73 distinct contributors, not including all the work that goes into dependencies such as <a href="https://crates.io/crates/arrow">arrow</a>, <a href="https://crates.io/crates/parquet">parquet</a>, and <a href="https://crates.io/crates/object_store">object_store</a>, that much of the same community helps support. Thank you all for your help</p>
+
+<!--
+$ git log --pretty=oneline 13.0.0..16.0.0 . | wc -l
+ 543
+
+$ git shortlog -sn 13.0.0..16.0.0 . | wc -l
+ 73
+-->
+<p>Several <a href="https://github.com/apache/arrow-datafusion#known-uses">new systems based on DataFusion</a> were recently added:</p>
+
+<ul>
+ <li><a href="https://github.com/GreptimeTeam/greptimedb">Greptime DB</a></li>
+ <li><a href="https://synnada.ai/">Synnada</a></li>
+ <li><a href="https://github.com/PRQL/prql-query">PRQL</a></li>
+ <li><a href="https://github.com/parseablehq/parseable">Parseable</a></li>
+ <li><a href="https://github.com/splitgraph/seafowl">SeaFowl</a></li>
+</ul>
+
+<h2 id="performance-">Performance 🚀</h2>
+
+<p>Performance and efficiency are core values for
+DataFusion. While there is still a gap between DataFusion and the best of
+breed, tightly integrated systems such as <a href="https://duckdb.org">DuckDB</a>
+and <a href="https://www.pola.rs/">Polars</a>, DataFusion is
+closing the gap quickly. Performance highlights from the last three
+months:</p>
+
+<ul>
+ <li>Up to 30% Faster Sorting and Merging using the new <a href="https://arrow.apache.org/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">Row Format</a></li>
+ <li><a href="https://arrow.apache.org/blog/2022/12/26/querying-parquet-with-millisecond-latency/">Advanced predicate pushdown</a>, directly on parquet, directly from object storage, enabling sub millisecond filtering. <!-- Andrew nots: we should really get this turned on by default --></li>
+ <li><code class="language-plaintext highlighter-rouge">70%</code> faster <code class="language-plaintext highlighter-rouge">IN</code> expressions evaluation (<a href="https://github.com/apache/arrow-datafusion/issues/4057">#4057</a>)</li>
+ <li>Sort and partition aware optimizations (<a href="https://github.com/apache/arrow-datafusion/issues/3969">#3969</a> and <a href="https://github.com/apache/arrow-datafusion/issues/4691">#4691</a>)</li>
+ <li>Filter selectivity analysis (<a href="https://github.com/apache/arrow-datafusion/issues/3868">#3868</a>)</li>
+</ul>
+
+<h2 id="runtime-resource-limits">Runtime Resource Limits</h2>
+
+<p>Previously, DataFusion could potentially use unbounded amounts of memory for certain queries that included Sorts, Grouping or Joins.</p>
+
+<p>In version 16.0.0, it is possible to limit DataFusion’s memory usage for Sorting and Grouping. We are looking for help adding similar limiting for Joins as well as expanding our algorithms to optionally spill to secondary storage. See <a href="https://github.com/apache/arrow-datafusion/issues/3941">#3941</a> for more detail.</p>
+
+<h2 id="sql-window-functions">SQL Window Functions</h2>
+
+<p><a href="https://en.wikipedia.org/wiki/Window_function_(SQL)">SQL Window Functions</a> are useful for a variety of analysis and DataFusion’s implementation support expanded significantly:</p>
+
+<ul>
+ <li>Custom window frames such as <code class="language-plaintext highlighter-rouge">... OVER (ORDER BY ... RANGE BETWEEN 0.2 PRECEDING AND 0.2 FOLLOWING)</code></li>
+ <li>Unbounded window frames such as <code class="language-plaintext highlighter-rouge">... OVER (ORDER BY ... RANGE UNBOUNDED ROWS PRECEDING)</code></li>
+ <li>Support for the <code class="language-plaintext highlighter-rouge">NTILE</code> window function (<a href="https://github.com/apache/arrow-datafusion/issues/4676">#4676</a>)</li>
+ <li>Support for <code class="language-plaintext highlighter-rouge">GROUPS</code> mode (<a href="https://github.com/apache/arrow-datafusion/issues/4155">#4155</a>)</li>
+</ul>
+
+<h1 id="improved-joins">Improved Joins</h1>
+
+<p>Joins are often the most complicated operations to handle well in
+analytics systems and DataFusion 16.0.0 offers significant improvements
+such as</p>
+
+<ul>
+ <li>Cost based optimizer (CBO) automatically reorders join evaluations, selects algorithms (Merge / Hash), and pick build side based on available statistics and join type (<code class="language-plaintext highlighter-rouge">INNER</code>, <code class="language-plaintext highlighter-rouge">LEFT</code>, etc) (<a href="https://github.com/apache/arrow-datafusion/issues/4219">#4219</a>)</li>
+ <li>Fast non <code class="language-plaintext highlighter-rouge">column=column</code> equijoins such as <code class="language-plaintext highlighter-rouge">JOIN ON a.x + 5 = b.y</code></li>
+ <li>Better performance on non-equijoins (<a href="https://github.com/apache/arrow-datafusion/issues/4562">#4562</a>) <!-- TODO is this a good thing to mention as any time this is usd the query is going to go slow or the data size is small --></li>
+</ul>
+
+<h1 id="streaming-execution">Streaming Execution</h1>
+
+<p>One emerging use case for Datafusion is as a foundation for
+streaming-first data platforms. An important prerequisite
+is support for incremental execution for queries that can be computed
+incrementally.</p>
+
+<p>With this release, DataFusion now supports the following streaming features:</p>
+
+<ul>
+ <li>Data ingestion from infinite files such as FIFOs (<a href="https://github.com/apache/arrow-datafusion/issues/4694">#4694</a>),</li>
+ <li>Detection of pipeline-breaking queries in streaming use cases (<a href="https://github.com/apache/arrow-datafusion/issues/4694">#4694</a>),</li>
+ <li>Automatic input swapping for joins so probe side is a data stream (<a href="https://github.com/apache/arrow-datafusion/issues/4694">#4694</a>),</li>
+ <li>Intelligent elision of pipeline-breaking sort operations whenever possible (<a href="https://github.com/apache/arrow-datafusion/issues/4691">#4691</a>),</li>
+ <li>Incremental execution for more types of queries; e.g. queries involving finite window frames (<a href="https://github.com/apache/arrow-datafusion/issues/4777">#4777</a>).</li>
+</ul>
+
+<p>These are a major steps forward, and we plan even more improvements over the next few releases.</p>
+
+<h1 id="better-support-for-distributed-catalogs">Better Support for Distributed Catalogs</h1>
+
+<p>16.0.0 has been enhanced support for asynchronous catalogs (<a href="https://github.com/apache/arrow-datafusion/issues/4607">#4607</a>)
+to better support distributed metadata stores such as
+<a href="https://delta.io/">Delta.io</a> and <a href="https://iceberg.apache.org/">Apache
+Iceberg</a> which require asynchronous I/O
+during planning to access remote catalogs. Previously, DataFusion
+required synchronous access to all relevant catalog information.</p>
+
+<h1 id="additional-sql-support">Additional SQL Support</h1>
+<p>SQL support continues to improve, including some of these highlights:</p>
+
+<ul>
+ <li>Add TPC-DS query planning regression tests <a href="https://github.com/apache/arrow-datafusion/issues/4719">#4719</a></li>
+ <li>Support for <code class="language-plaintext highlighter-rouge">PREPARE</code> statement <a href="https://github.com/apache/arrow-datafusion/issues/4490">#4490</a></li>
+ <li>Automatic coercions ast between Date and Timestamp <a href="https://github.com/apache/arrow-datafusion/issues/4726">#4726</a></li>
+ <li>Support type coercion for timestamp and utf8 <a href="https://github.com/apache/arrow-datafusion/issues/4312">#4312</a></li>
+ <li>Full support for time32 and time64 literal values (<code class="language-plaintext highlighter-rouge">ScalarValue</code>) <a href="https://github.com/apache/arrow-datafusion/issues/4156">#4156</a></li>
+ <li>New functions, incuding <code class="language-plaintext highlighter-rouge">uuid()</code> <a href="https://github.com/apache/arrow-datafusion/issues/4041">#4041</a>, <code class="language-plaintext highlighter-rouge">current_time</code> <a href="https://github.com/apache/arrow-datafusion/issues/4054">#4054</a>, <code class="language-plaintext highlighter-rouge">current_date</code> <a href="https://github.com/apache/arrow-datafusion/issues/4022">#4022</a></li>
+ <li>Compressed CSV/JSON support <a href="https://github.com/apache/arrow-datafusion/issues/3642">#3642</a></li>
+</ul>
+
+<p>The community has also invested in new <a href="https://github.com/apache/arrow-datafusion/blob/master/datafusion/core/tests/sqllogictests/README.md">sqllogic based</a> tests to keep improving DataFusion’s quality with less effort.</p>
+
+<h1 id="plan-serialization-and-substrait">Plan Serialization and Substrait</h1>
+
+<p>DataFusion now supports serialization of physical plans, with a custom protocol buffers format. In addition, we are adding initial support for <a href="https://substrait.io/">Substrait</a>, a Cross-Language Serialization for Relational Algebra</p>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>Kudos to everyone in the community who contributed ideas, discussions, bug reports, documentation and code. It is exciting to be building something so cool together!</p>
+
+<p>If you are interested in contributing to DataFusion, we would love to
+have you join us. You can try out DataFusion on some of your own
+data and projects and let us know how it goes or contribute a PR with
+documentation, tests or code. A list of open issues suitable for
+beginners is
+<a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>Check out our <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+<h2 id="appendix-contributor-shoutout">Appendix: Contributor Shoutout</h2>
+
+<p>Here is a list of people who have contributed PRs to this project over the last three releases, derived from <code class="language-plaintext highlighter-rouge">git shortlog -sn 13.0.0..16.0.0 .</code> Thank you all!</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 113 Andrew Lamb
+ 58 jakevin
+ 46 Raphael Taylor-Davies
+ 30 Andy Grove
+ 19 Batuhan Taskaya
+ 19 Remzi Yang
+ 17 ygf11
+ 16 Burak
+ 16 Jeffrey
+ 16 Marco Neumann
+ 14 Kun Liu
+ 12 Yang Jiang
+ 10 mingmwang
+ 9 Daniël Heres
+ 9 Mustafa akur
+ 9 comphead
+ 9 mvanschellebeeck
+ 9 xudong.w
+ 7 dependabot[bot]
+ 7 yahoNanJing
+ 6 Brent Gardner
+ 5 AssHero
+ 4 Jiayu Liu
+ 4 Wei-Ting Kuo
+ 4 askoa
+ 3 André Calado Coroado
+ 3 Jie Han
+ 3 Jon Mease
+ 3 Metehan Yıldırım
+ 3 Nga Tran
+ 3 Ruihang Xia
+ 3 baishen
+ 2 Berkay Şahin
+ 2 Dan Harris
+ 2 Dongyan Zhou
+ 2 Eduard Karacharov
+ 2 Kikkon
+ 2 Liang-Chi Hsieh
+ 2 Marko Milenković
+ 2 Martin Grigorov
+ 2 Roman Nozdrin
+ 2 Tim Van Wassenhove
+ 2 r.4ntix
+ 2 unconsolable
+ 2 unvalley
+ 1 Ajaya Agrawal
+ 1 Alexander Spies
+ 1 ArkashaJavelin
+ 1 Artjoms Iskovs
+ 1 BoredPerson
+ 1 Christian Salvati
+ 1 Creampanda
+ 1 Data Psycho
+ 1 Francis Du
+ 1 Francis Le Roy
+ 1 LFC
+ 1 Marko Grujic
+ 1 Matt Willian
+ 1 Matthijs Brobbel
+ 1 Max Burke
+ 1 Mehmet Ozan Kabak
+ 1 Rito Takeuchi
+ 1 Roman Zeyde
+ 1 Vrishabh
+ 1 Zhang Li
+ 1 ZuoTiJia
+ 1 byteink
+ 1 cfraz89
+ 1 nbr
+ 1 xxchan
+ 1 yujie.zhang
+ 1 zembunia
+ 1 哇呜哇呜呀咦耶
+</code></pre></div></div>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Apache Arrow Ballista 0.9.0 Release</title><link href="https://datafusion.apache.org/blog/blog/2022/10/28/ballista-0.9.0/" rel="alternate" type="text/html" title="Apache Arrow Ballista 0.9.0 Release" /><published>2022-10-28T00:00:00-06:00</published><updated>2022-10-28T00:00:00-06:00</updated><id>https://dataf [...]
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://github.com/apache/arrow-ballista">Ballista</a> is an Arrow-native distributed SQL query engine implemented in Rust.</p>
+
+<p>Ballista 0.9.0 is now available and is the most significant release since the project was <a href="http://arrow.apache.org/blog/2021/04/12/ballista-donation/">donated</a> to Apache
+Arrow in 2021.</p>
+
+<p>This release represents 4 weeks of work, with 66 commits from 14 contributors:</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 22 Andy Grove
+ 12 yahoNanJing
+ 6 Daniël Heres
+ 4 Brent Gardner
+ 4 dependabot[bot]
+ 4 r.4ntix
+ 3 Stefan Stanciulescu
+ 3 mingmwang
+ 2 Ken Suenobu
+ 2 Yang Jiang
+ 1 Metehan Yıldırım
+ 1 Trent Feda
+ 1 askoa
+ 1 yangzhong
+</code></pre></div></div>
+
+<h2 id="release-highlights">Release Highlights</h2>
+
+<p>The release notes below are not exhaustive and only expose selected highlights of the release. Many other bug fixes
+and improvements have been made: we refer you to the <a href="https://github.com/apache/arrow-ballista/blob/0.9.0-rc2/ballista/CHANGELOG.md">complete changelog</a>.</p>
+
+<h3 id="support-for-cloud-object-stores-and-distributed-file-systems">Support for Cloud Object Stores and Distributed File Systems</h3>
+
+<p>This is the first release of Ballista to have documented support for querying data from distributed file systems and
+object stores. Currently, S3 and HDFS are supported. Support for Google Cloud Storage and Azure Blob Storage is planned
+for the next release.</p>
+
+<h3 id="flight-sql--jdbc-support">Flight SQL & JDBC support</h3>
+
+<p>The Ballista scheduler now implements the <a href="https://arrow.apache.org/blog/2022/02/16/introducing-arrow-flight-sql/">Flight SQL protocol</a>, enabling any compliant Flight SQL client
+to connect to and run queries against a Ballista cluster.</p>
+
+<p>The Apache Arrow Flight SQL JDBC driver can be used to connect Business Intelligence tools to a Ballista cluster.</p>
+
+<h3 id="python-bindings">Python Bindings</h3>
+
+<p>It is now possible to connect to a Ballista cluster from Python and execute queries using both the DataFrame and SQL
+interfaces.</p>
+
+<h3 id="scheduler-web-user-interface-and-rest-api">Scheduler Web User Interface and REST API</h3>
+
+<p>The scheduler now has a web user interface for monitoring queries. It is also possible to view graphical query plans
+that show how the query was executed, along with metrics.</p>
+
+<p><img src="/blog/img/2022-10-28-ballista-web-ui.png" width="800" /></p>
+
+<p>The REST API that powers the user interface can also be accessed directly.</p>
+
+<h3 id="simplified-kubernetes-deployment">Simplified Kubernetes Deployment</h3>
+
+<p>Ballista now provides a <a href="https://github.com/apache/arrow-ballista/tree/master/helm">Helm chart</a> for simplified Kubernetes deployment.</p>
+
+<h3 id="user-guide">User Guide</h3>
+
+<p>The user guide is published at <a href="https://arrow.apache.org/ballista/">https://arrow.apache.org/ballista/</a> and provides
+deployment instructions for Docker, Docker Compose, and Kubernetes, as well as references for configuring and
+tuning Ballista.</p>
+
+<h2 id="roadmap">Roadmap</h2>
+
+<p>The Ballista community is currently focused on the following tasks for the next release:</p>
+
+<ul>
+ <li>Support for Azure Blob Storage and Google Cloud Storage</li>
+ <li>Improve benchmark performance by implementing more query optimizations</li>
+ <li>Improve scheduler web user interface</li>
+ <li>Publish Docker images to GitHub Container Registry</li>
+</ul>
+
+<p>The detailed list of issues planned for the 0.10.0 release can be found in the <a href="https://github.com/apache/arrow-ballista/issues/361">tracking issue</a>.</p>
+
+<h2 id="getting-involved">Getting Involved</h2>
+
+<p>Ballista has a friendly community and we welcome contributions. A good place to start is to following the instructions
+in the <a href="https://arrow.apache.org/ballista/">user guide</a> and try using Ballista with your own SQL queries and ETL pipelines, and file issues
+for any bugs or feature suggestions.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Apache Arrow DataFusion 13.0.0 Project Update</title><link href="https://datafusion.apache.org/blog/blog/2022/10/25/datafusion-13.0.0/" rel="alternate" type="text/html" title="Apache Arrow DataFusion 13.0.0 Project Update" /><published>2022-10-25T00:00:00-06:00</published><updated>2022-10-25T00: [...]
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">Apache Arrow DataFusion</a> <a href="https://crates.io/crates/datafusion"><code class="language-plaintext highlighter-rouge">13.0.0</code></a> is released, and this blog contains an update on the project for the 5 months since our <a href="https://arrow.apache.org/blog/2022/05/16/datafusion-8.0.0/">last update in May 2022</a>.</p>
+
+<p>DataFusion is an extensible and embeddable query engine, written in Rust used to create modern, fast and efficient data pipelines, ETL processes, and database systems. You may want to check out DataFusion to extend your Rust project to:</p>
+
+<ul>
+ <li>Support <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a></li>
+ <li>Support <a href="https://docs.rs/datafusion/13.0.0/datafusion/dataframe/struct.DataFrame.html">DataFrame API</a></li>
+ <li>Support a Domain Specific Query Language</li>
+ <li>Easily and quickly read and process Parquet, JSON, Avro or CSV data.</li>
+ <li>Read from remote object stores such as AWS S3, Azure Blob Storage, GCP.</li>
+</ul>
+
+<p>Even though DataFusion is 4 years “young,” it has seen significant community growth in the last few months and the momentum continues to accelerate.</p>
+
+<h1 id="background">Background</h1>
+
+<p>DataFusion is used as the engine in <a href="https://github.com/apache/arrow-datafusion#known-uses">many open source and commercial projects</a> and was one of the early open source projects to provide this capability. 2022 has validated our belief in the need for such a <a href="https://docs.google.com/presentation/d/1iNX_35sWUakee2q3zMFPyHE4IV2nC3lkCK_H6Y2qK84/edit#slide=id.p">“LLVM for database and AI systems”</a><a href="https://www.slideshare.net/AndrewLamb32/20220623-apache-arro [...]
+
+<p>While Velox and Acero focus on execution engines, DataFusion provides the entire suite of components needed to build most analytic systems, including a SQL frontend, a dataframe API, and extension points for just about everything. Some <a href="https://github.com/apache/arrow-datafusion#known-uses">DataFusion users</a> use a subset of the features such as the frontend (e.g. <a href="https://dask-sql.readthedocs.io/en/latest/">dask-sql</a>) or the execution engine, (e.g. <a href="htt [...]
+
+<p>One of DataFusion’s advantages is its implementation in <a href="https://www.rust-lang.org/">Rust</a> and thus its easy integration with the broader Rust ecosystem. Rust continues to be a major source of benefit, from the <a href="https://www.influxdata.com/blog/using-rustlangs-async-tokio-runtime-for-cpu-bound-tasks/">ease of parallelization with the high quality and standardized <code class="language-plaintext highlighter-rouge">async</code> ecosystem</a> , as well as its modern dep [...]
+<!--While we haven’t invested in the benchmarking ratings game datafusion continues to be quite speedy (todo quantity this, with some evidence) – maybe clickbench?--></p>
+
+<!--
+Maybe we can do this un a future post
+# DataFusion in Action
+
+While DataFusion really shines as an embeddable query engine, if you want to try it out and get a feel for its power, you can use the basic[`datafusion-cli`](https://docs.rs/datafusion-cli/13.0.0/datafusion_cli/) tool to get a sense for what is possible to add in your application
+
+(TODO example here of using datafusion-cli to query from local parquet files on disk)
+
+TODO: also mention you can use the same thing to query data from S3
+-->
+
+<h1 id="summary">Summary</h1>
+
+<p>We have increased the frequency of DataFusion releases to monthly instead of quarterly. This
+makes it easier for the increasing number of projects that now depend on DataFusion.</p>
+
+<p>We have also completed the “graduation” of <a href="https://github.com/apache/arrow-ballista">Ballista to its own top-level arrow-ballista repository</a>
+which decouples the two projects and allows each project to move even faster.</p>
+
+<p>Along with numerous other bug fixes and smaller improvements, here are some of the major advances:</p>
+
+<h1 id="improved-support-for-cloud-object-stores">Improved Support for Cloud Object Stores</h1>
+
+<p>DataFusion now supports many major cloud object stores (Amazon S3, Azure Blob Storage, and Google Cloud Storage) “out of the box” via the <a href="https://crates.io/crates/object_store">object_store</a> crate. Using this integration, DataFusion optimizes reading parquet files by reading only the parts of the files that are needed.</p>
+
+<h2 id="advanced-sql">Advanced SQL</h2>
+
+<p>DataFusion now supports correlated subqueries, by rewriting them as joins. See the <a href="https://arrow.apache.org/datafusion/user-guide/sql/subqueries.html">Subquery</a> page in the User Guide for more information.</p>
+
+<p>In addition to numerous other small improvements, the following SQL features are now supported:</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">ROWS</code>, <code class="language-plaintext highlighter-rouge">RANGE</code>, <code class="language-plaintext highlighter-rouge">PRECEDING</code> and <code class="language-plaintext highlighter-rouge">FOLLOWING</code> in <code class="language-plaintext highlighter-rouge">OVER</code> clauses <a href="https://github.com/apache/arrow-datafusion/issues/3570">#3570</a></li>
+ <li><code class="language-plaintext highlighter-rouge">ROLLUP</code> and <code class="language-plaintext highlighter-rouge">CUBE</code> grouping set expressions <a href="https://github.com/apache/arrow-datafusion/issues/2446">#2446</a></li>
+ <li><code class="language-plaintext highlighter-rouge">SUM DISTINCT</code> aggregate support <a href="https://github.com/apache/arrow-datafusion/issues/2405">#2405</a></li>
+ <li><code class="language-plaintext highlighter-rouge">IN</code> and <code class="language-plaintext highlighter-rouge">NOT IN</code> Subqueries by rewriting them to <code class="language-plaintext highlighter-rouge">SEMI</code> / <code class="language-plaintext highlighter-rouge">ANTI</code> <a href="https://github.com/apache/arrow-datafusion/issues/2885">#2421</a></li>
+ <li>Non equality predicates in <code class="language-plaintext highlighter-rouge">ON</code> clause of <code class="language-plaintext highlighter-rouge">LEFT</code>, <code class="language-plaintext highlighter-rouge">RIGHT, </code>and <code class="language-plaintext highlighter-rouge">FULL</code> joins <a href="https://github.com/apache/arrow-datafusion/issues/2591">#2591</a></li>
+ <li>Exact <code class="language-plaintext highlighter-rouge">MEDIAN</code> <a href="https://github.com/apache/arrow-datafusion/issues/3009">#3009</a></li>
+ <li><code class="language-plaintext highlighter-rouge">GROUPING SETS</code>/<code class="language-plaintext highlighter-rouge">CUBE</code>/<code class="language-plaintext highlighter-rouge">ROLLUP</code> <a href="https://github.com/apache/arrow-datafusion/issues/2716">#2716</a></li>
+</ul>
+
+<h1 id="more-ddl-support">More DDL Support</h1>
+
+<p>Just as it is important to query, it is also important to give users the ability to define their data sources. We have added:</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">CREATE VIEW</code> <a href="https://github.com/apache/arrow-datafusion/issues/2279">#2279</a></li>
+ <li><code class="language-plaintext highlighter-rouge">DESCRIBE <table></code> <a href="https://github.com/apache/arrow-datafusion/issues/2642">#2642</a></li>
+ <li>Custom / Dynamic table provider factories <a href="https://github.com/apache/arrow-datafusion/issues/3311">#3311</a></li>
+ <li><code class="language-plaintext highlighter-rouge">SHOW CREATE TABLE</code> for support for views <a href="https://github.com/apache/arrow-datafusion/issues/2830">#2830</a></li>
+</ul>
+
+<h1 id="faster-execution">Faster Execution</h1>
+<p>Performance is always an important goal for DataFusion, and there are a number of significant new optimizations such as</p>
+
+<ul>
+ <li>Optimizations of TopK (queries with a <code class="language-plaintext highlighter-rouge">LIMIT</code> or <code class="language-plaintext highlighter-rouge">OFFSET</code> clause): <a href="https://github.com/apache/arrow-datafusion/issues/3527">#3527</a>, <a href="https://github.com/apache/arrow-datafusion/issues/2521">#2521</a></li>
+ <li>Reduce <code class="language-plaintext highlighter-rouge">left</code>/<code class="language-plaintext highlighter-rouge">right</code>/<code class="language-plaintext highlighter-rouge">full</code> joins to <code class="language-plaintext highlighter-rouge">inner</code> join <a href="https://github.com/apache/arrow-datafusion/issues/2750">#2750</a></li>
+ <li>Convert cross joins to inner joins when possible <a href="https://github.com/apache/arrow-datafusion/issues/3482">#3482</a></li>
+ <li>Sort preserving <code class="language-plaintext highlighter-rouge">SortMergeJoin</code> <a href="https://github.com/apache/arrow-datafusion/issues/2699">#2699</a></li>
+ <li>Improvements in group by and sort performance <a href="https://github.com/apache/arrow-datafusion/issues/2375">#2375</a></li>
+ <li>Adaptive <code class="language-plaintext highlighter-rouge">regex_replace</code> implementation <a href="https://github.com/apache/arrow-datafusion/issues/3518">#3518</a></li>
+</ul>
+
+<h1 id="optimizer-enhancements">Optimizer Enhancements</h1>
+<p>Internally the optimizer has been significantly enhanced as well.</p>
+
+<ul>
+ <li>Casting / coercion now happens during logical planning <a href="https://github.com/apache/arrow-datafusion/issues/3396">#3185</a> <a href="https://github.com/apache/arrow-datafusion/issues/3636">#3636</a></li>
+ <li>More sophisticated expression analysis and simplification is available</li>
+</ul>
+
+<h1 id="parquet">Parquet</h1>
+<ul>
+ <li>The parquet reader can now read directly from parquet files on remote object storage <a href="https://github.com/apache/arrow-datafusion/issues/2677">#2489</a> <a href="https://github.com/apache/arrow-datafusion/issues/3051">#3051</a></li>
+ <li>Experimental support for “predicate pushdown” with late materialization after filtering during the scan (another blog post on this topic is coming soon).</li>
+ <li>Support reading directly from AWS S3 and other object stores via <code class="language-plaintext highlighter-rouge">datafusion-cli </code> <a href="https://github.com/apache/arrow-datafusion/issues/3631">#3631</a></li>
+</ul>
+
+<h1 id="datatype-support">DataType Support</h1>
+<ul>
+ <li>Support for <code class="language-plaintext highlighter-rouge">TimestampTz</code> <a href="https://github.com/apache/arrow-datafusion/issues/3660">#3660</a></li>
+ <li>Expanded support for the <code class="language-plaintext highlighter-rouge">Decimal</code> type, including <code class="language-plaintext highlighter-rouge">IN</code> list and better built in coercion.</li>
+ <li>Expanded support for date/time manipulation such as <code class="language-plaintext highlighter-rouge">date_bin</code> built-in function , timestamp <code class="language-plaintext highlighter-rouge">+/-</code> interval, <code class="language-plaintext highlighter-rouge">TIME</code> literal values <a href="https://github.com/apache/arrow-datafusion/issues/3010">#3010</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3110">#3110</a>, <a href="https://github.com/apache [...]
+ <li>Binary operations (<code class="language-plaintext highlighter-rouge">AND</code>, <code class="language-plaintext highlighter-rouge">XOR</code>, etc): <a href="https://github.com/apache/arrow-datafusion/issues/1619">#3037</a> <a href="https://github.com/apache/arrow-datafusion/issues/3430">#3420</a></li>
+ <li><code class="language-plaintext highlighter-rouge">IS TRUE/FALSE</code> and <code class="language-plaintext highlighter-rouge">IS [NOT] UNKNOWN</code> <a href="https://github.com/apache/arrow-datafusion/issues/3235">#3235</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3246">#3246</a></li>
+</ul>
+
+<h2 id="upcoming-work">Upcoming Work</h2>
+<p>With the community growing and code accelerating, there is so much great stuff on the horizon. Some features we expect to land in the next few months:</p>
+
+<ul>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/3462">Complete Parquet Pushdown</a></li>
+ <li><a href="https://github.com/apache/arrow-datafusion/issues/3148">Additional date/time support</a></li>
+ <li>Cost models, Nested Join Optimizations, analysis framework <a href="https://github.com/apache/arrow-datafusion/issues/128">#128</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3843">#3843</a>, <a href="https://github.com/apache/arrow-datafusion/issues/3845">#3845</a></li>
+</ul>
+
+<h1 id="community-growth">Community Growth</h1>
+
+<p>The DataFusion 9.0.0 and 13.0.0 releases consists of 433 PRs from 64 distinct contributors. This does not count all the work that goes into our dependencies such as <a href="https://crates.io/crates/arrow">arrow</a>, <a href="https://crates.io/crates/parquet">parquet</a>, and <a href="https://crates.io/crates/object_store">object_store</a>, that much of the same community helps nurture.</p>
+
+<!--
+$ git log --pretty=oneline 9.0.0..13.0.0 . | wc -l
+433
+
+$ git shortlog -sn 9.0.0..13.0.0 . | wc -l
+65
+-->
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>Kudos to everyone in the community who contributed ideas, discussions, bug reports, documentation and code. It is exciting to be building something so cool together!</p>
+
+<p>If you are interested in contributing to DataFusion, we would love to
+have you join us on our journey to create the most advanced open
+source query engine. You can try out DataFusion on some of your own
+data and projects and let us know how it goes or contribute a PR with
+documentation, tests or code. A list of open issues suitable for
+beginners is
+<a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>.</p>
+
+<p>Check out our <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>
+
+<h2 id="appendix-contributor-shoutout">Appendix: Contributor Shoutout</h2>
+
+<p>To give a sense of the number of people who contribute to this project regularly, we present for your consideration the following list derived from <code class="language-plaintext highlighter-rouge">git shortlog -sn 9.0.0..13.0.0 .</code> Thank you all again!</p>
+
+<!-- Note: combined kmitchener and Kirk Mitchener -->
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 87 Andy Grove
+ 71 Andrew Lamb
+ 29 Kun Liu
+ 29 Kirk Mitchener
+ 17 Wei-Ting Kuo
+ 14 Yang Jiang
+ 12 Raphael Taylor-Davies
+ 11 Batuhan Taskaya
+ 10 Brent Gardner
+ 10 Remzi Yang
+ 10 comphead
+ 10 xudong.w
+ 8 AssHero
+ 7 Ruihang Xia
+ 6 Dan Harris
+ 6 Daniël Heres
+ 6 Ian Alexander Joiner
+ 6 Mike Roberts
+ 6 askoa
+ 4 BaymaxHWY
+ 4 gorkem
+ 4 jakevin
+ 3 George Andronchik
+ 3 Sarah Yurick
+ 3 Stuart Carnie
+ 2 Dalton Modlin
+ 2 Dmitry Patsura
+ 2 JasonLi
+ 2 Jon Mease
+ 2 Marco Neumann
+ 2 yahoNanJing
+ 1 Adilet Sarsembayev
+ 1 Ayush Dattagupta
+ 1 Dezhi Wu
+ 1 Dhamotharan Sritharan
+ 1 Eduard Karacharov
+ 1 Francis Du
+ 1 Harbour Zheng
+ 1 Ismaël Mejía
+ 1 Jack Klamer
+ 1 Jeremy Dyer
+ 1 Jiayu Liu
+ 1 Kamil Konior
+ 1 Liang-Chi Hsieh
+ 1 Martin Grigorov
+ 1 Matthijs Brobbel
+ 1 Mehmet Ozan Kabak
+ 1 Metehan Yıldırım
+ 1 Morgan Cassels
+ 1 Nitish Tiwari
+ 1 Renjie Liu
+ 1 Rito Takeuchi
+ 1 Robert Pack
+ 1 Thomas Cameron
+ 1 Vrishabh
+ 1 Xin Hao
+ 1 Yijie Shen
+ 1 byteink
+ 1 kamille
+ 1 mateuszkj
+ 1 nvartolomei
+ 1 yourenawo
+ 1 Özgür Akkurt
+</code></pre></div></div>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Apache Arrow DataFusion 8.0.0 Release</title><link href="https://datafusion.apache.org/blog/blog/2022/05/16/datafusion-8.0.0/" rel="alternate" type="text/html" title="Apache Arrow DataFusion 8.0.0 Release" /><published>2022-05-16T00:00:00-06:00</published><updated>2022-05-16T00:00:00-06:00</updated><id>https:/ [...]
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible query execution framework, written in Rust, that
+uses Apache Arrow as its in-memory format.</p>
+
+<p>When you want to extend your Rust project with <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a>,
+a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth
+checking out.</p>
+
+<p>DataFusion’s SQL, <code class="language-plaintext highlighter-rouge">DataFrame</code>, and manual <code class="language-plaintext highlighter-rouge">PlanBuilder</code> API let users access a sophisticated query optimizer and
+execution engine capable of fast, resource efficient, and parallel execution that takes optimal advantage of
+today’s multicore hardware. Being written in Rust means DataFusion can offer <em>both</em> the safety of a dynamic language and
+the resource efficiency of a compiled language.</p>
+
+<p>The Apache Arrow team is pleased to announce the DataFusion 8.0.0 release (and also the release of version 0.7.0 of
+the Ballista subproject). This covers 3 months of development work and includes 279 commits from the following 49
+distinct contributors.</p>
+
+<!--
+$ git log --pretty=oneline 7.0.0..8.0.0 datafusion datafusion-cli datafusion-examples ballista ballista-cli ballista-examples | wc -l
+279
+
+$ git shortlog -sn 7.0.0..8.0.0 datafusion datafusion-cli datafusion-examples ballista ballista-cli ballista-examples | wc -l
+49
+
+(feynman han, feynman.h, Feynman Han were assumed to be the same person)
+-->
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 39 Andy Grove
+ 33 Andrew Lamb
+ 21 DuRipeng
+ 20 Yijie Shen
+ 19 Yang Jiang
+ 17 Raphael Taylor-Davies
+ 11 Dan Harris
+ 11 Matthew Turner
+ 11 yahoNanJing
+ 9 dependabot[bot]
+ 8 jakevin
+ 6 Kun Liu
+ 5 Jiayu Liu
+ 4 Daniël Heres
+ 4 mingmwang
+ 4 xudong.w
+ 3 Carol (Nichols || Goulding)
+ 3 Dmitry Patsura
+ 3 Eduard Karacharov
+ 3 Jeremy Dyer
+ 3 Kaushik
+ 3 Rich
+ 3 comphead
+ 3 gaojun2048
+ 3 Feynman Han
+ 2 Jie Han
+ 2 Jon Mease
+ 2 Tim Van Wassenhove
+ 2 Yt
+ 2 Zhang Li
+ 2 silence-coding
+ 1 Alexander Spies
+ 1 George Andronchik
+ 1 Guillaume Balaine
+ 1 Hao Xin
+ 1 Jiacai Liu
+ 1 Jörn Horstmann
+ 1 Liang-Chi Hsieh
+ 1 Max Burke
+ 1 NaincyKumariKnoldus
+ 1 Nga Tran
+ 1 Patrick More
+ 1 Pierre Zemb
+ 1 Remzi Yang
+ 1 Sergey Melnychuk
+ 1 Stephen Carman
+ 1 doki
+</code></pre></div></div>
+
+<p>The following sections highlight some of the changes in this release. Of course, many other bug fixes and
+improvements have been made and we encourage you to check out the
+<a href="https://github.com/apache/arrow-datafusion/blob/8.0.0/datafusion/CHANGELOG.md">changelog</a> for full details.</p>
+
+<h1 id="summary">Summary</h1>
+
+<h2 id="ddl-support">DDL Support</h2>
+
+<p>DDL support has been expanded to include the following commands for creating databases, schemas, and views. This
+allows DataFusion to be used more effectively from the CLI.</p>
+
+<ul>
+ <li><code class="language-plaintext highlighter-rouge">CREATE DATABASE</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE VIEW</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE SCHEMA</code></li>
+ <li><code class="language-plaintext highlighter-rouge">CREATE EXTERNAL TABLE</code> now supports JSON files, <code class="language-plaintext highlighter-rouge">IF NOT EXISTS</code>, and partition columns</li>
+</ul>
+
+<h2 id="sql-support">SQL Support</h2>
+
+<p>The SQL query planner now supports a number of new SQL features, including:</p>
+
+<ul>
+ <li><em>Subqueries</em>: when used via <code class="language-plaintext highlighter-rouge">IN</code>, <code class="language-plaintext highlighter-rouge">EXISTS</code>, and as scalars</li>
+ <li><em>Grouping Sets</em>: <code class="language-plaintext highlighter-rouge">CUBE</code> and <code class="language-plaintext highlighter-rouge">ROLLUP</code> grouping sets.</li>
+ <li><em>Aggregate functions</em>: <code class="language-plaintext highlighter-rouge">approx_percentile</code>, <code class="language-plaintext highlighter-rouge">approx_percentile_cont</code>, <code class="language-plaintext highlighter-rouge">approx_percentile_cont_with_weight</code>, <code class="language-plaintext highlighter-rouge">approx_distinct</code>, <code class="language-plaintext highlighter-rouge">approx_median</code> and <code class="language-plaintext highlighter-rouge">a [...]
+ <li><em><code class="language-plaintext highlighter-rouge">null</code> literals</em></li>
+ <li><em>bitwise operations</em>: for example ‘<code class="language-plaintext highlighter-rouge">|</code>’</li>
+</ul>
+
+<p>There are also many bug fixes and improvements around normalizing identifiers consistently.</p>
+
+<p>We continue our tradition of incrementally releasing support for new
+features as they are developed. Thus, while the physical plan may not yet
+support all new features, it gets more complete each release. These
+changes also make DataFusion an increasingly compelling choice for
+projects looking for a SQL parser and query planner that can produce
+optimized logical plans that can be translated to
+their own execution engine.</p>
+
+<h2 id="query-execution--internals">Query Execution & Internals</h2>
+
+<p>There are several notable improvements and new features in the query execution engine:</p>
+
+<ul>
+ <li>The <code class="language-plaintext highlighter-rouge">ExecutionContext</code> has been renamed to <code class="language-plaintext highlighter-rouge">SessionContext</code> and now supports multi-tenancy</li>
+ <li>The <code class="language-plaintext highlighter-rouge">ExecutionPlan</code> trait is no longer <code class="language-plaintext highlighter-rouge">async</code></li>
+ <li>A new serialization API for serializing plans to bytes (based on protobuf)</li>
+</ul>
+
+<p>In addition, we have added several foundational features to drive even
+more advanced query processing into DataFusion, focusing on running
+arbitrary queries larger than available memory, and pushing the
+envelope for performance of sorting, grouping, and joining even
+further:</p>
+
+<ul>
+ <li>Morsel-Driven Scheduler based on <a href="https://15721.courses.cs.cmu.edu/spring2016/papers/p743-leis.pdf">“Morsel-Driven Parallelism: A NUMA-Aware Query
+Evaluation Framework for the Many-Core Age”</a></li>
+ <li>Consolidated object store implementation and integration with parquet decoding</li>
+ <li>Memory Limited Spilling sort operator</li>
+ <li>Memory Limited Sort-Merge join operator</li>
+ <li>High performance JIT code generation for tuple comparisons</li>
+ <li>Memory efficient Row Format</li>
+</ul>
+
+<h2 id="improved-file-support">Improved file support</h2>
+
+<p>DataFusion now supports JSON, both for reading and writing. There are also new DataFrame methods for writing query
+results to files in CSV, Parquet, and JSON format.</p>
+
+<h2 id="ballista">Ballista</h2>
+
+<p>Ballista continues to mature and now supports a wider range of operators and expressions. There are also improvements
+to the scheduler to support UDFs, and there are some robustness improvements, such as cleaning up work directories
+and persisting session configs to allow schedulers to restart and continue processing in-flight jobs.</p>
+
+<h2 id="upcoming-work">Upcoming Work</h2>
+
+<p>Here are some of the initiatives that the community plans on working on prior to the next release.</p>
+
+<ul>
+ <li>There is a <a href="https://docs.google.com/document/d/1jNRbadyStSrV5kifwn0khufAwq6OnzGczG4z8oTQJP4/edit?usp=sharing">proposal to move Ballista to its own top-level arrow-ballista repository</a>
+ to decouple DataFusion and Ballista releases and to allow each project to have documentation better targeted at
+its particular audience.</li>
+ <li>We plan on increasing the frequency of DataFusion releases, with monthly releases now instead of quarterly. This
+is driven by requests from the increasing number of projects that now depend on DataFusion.</li>
+ <li>There is ongoing work to implement new optimizer rules to rewrite queries containing subquery expressions as
+joins, to support a wider range of queries.</li>
+ <li>The new scheduler based on morsel-driven execution will continue to evolve in this next release, with work to
+refine IO abstractions to improve performance and integration with the new scheduler.</li>
+ <li>Improved performance for Sort, Grouping and Joins</li>
+</ul>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, and learning about state-of-the-art query processing, we would
+love to have you join us on the journey! You can help by trying out DataFusion on some of your own data and projects
+and let us know how it goes or contribute a PR with documentation, tests or code. A list of open issues suitable
+for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a></p>
+
+<p>Check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
+ways to engage with the community.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry><entry><title type="html">Introducing Apache Arrow DataFusion Contrib</title><link href="https://datafusion.apache.org/blog/blog/2022/03/21/datafusion-contrib/" rel="alternate" type="text/html" title="Introducing Apache Arrow DataFusion Contrib" /><published>2022-03-21T00:00:00-06:00</published><updated>2022-03-21T00:00:00 [...]
+
+-->
+
+<h1 id="introduction">Introduction</h1>
+
+<p>Apache Arrow <a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible query execution framework, written in Rust, that uses <a href="https://arrow.apache.org">Apache Arrow</a> as its in-memory format.</p>
+
+<p>When you want to extend your Rust project with <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a>, a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth checking out. DataFusion’s pluggable design makes creating extensions at various points particular easy to build.</p>
+
+<p>DataFusion’s SQL, <code class="language-plaintext highlighter-rouge">DataFrame</code>, and manual <code class="language-plaintext highlighter-rouge">PlanBuilder</code> API let users access a sophisticated query optimizer and execution engine capable of fast, resource efficient, and parallel execution that takes optimal advantage of todays multicore hardware. Being written in Rust means DataFusion can offer <em>both</em> the safety of dynamic languages as well as the resource efficien [...]
+
+<p>The DataFusion team is pleased to announce the creation of the <a href="https://github.com/datafusion-contrib">DataFusion-Contrib</a> GitHub organization to support and accelerate other projects. While the core DataFusion library remains under Apache governance, the contrib organization provides a more flexible testing ground for new DataFusion features and a home for DataFusion extensions. With this announcement, we are pleased to introduce the following inaugural DataFusion-Contri [...]
+
+<h2 id="datafusion-python">DataFusion-Python</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-python">project</a> provides Python bindings to the core Rust implementation of DataFusion, which allows users to:</p>
+
+<ul>
+ <li>Work with familiar SQL or DataFrame APIs to run queries in a safe, multi-threaded environment, returning results in Python</li>
+ <li>Create User Defined Functions and User Defined Aggregate Functions for complex operations</li>
+ <li>Pay no overhead to copy between Python and underlying Rust execution engine (by way of Apache Arrow arrays)</li>
+</ul>
+
+<h3 id="upcoming-enhancements">Upcoming enhancements</h3>
+
+<p>The team is focusing on exposing more features from the underlying Rust implementation of DataFusion and improving documentation.</p>
+
+<h3 id="how-to-install">How to install</h3>
+
+<p>From <code class="language-plaintext highlighter-rouge">pip</code></p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>pip <span class="nb">install </span>datafusion
+</code></pre></div></div>
+
+<p>Or</p>
+
+<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>python <span class="nt">-m</span> pip <span class="nb">install </span>datafusion
+</code></pre></div></div>
+
+<h2 id="datafusion-objectstore-s3">DataFusion-ObjectStore-S3</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-objectstore-s3">crate</a> provides an <code class="language-plaintext highlighter-rouge">ObjectStore</code> implementation for querying data stored in S3 or S3 compatible storage. This makes it almost as easy to query data that lives on S3 as lives in local files</p>
+
+<ul>
+ <li>Ability to create <code class="language-plaintext highlighter-rouge">S3FileSystem</code> to register as part of DataFusion <code class="language-plaintext highlighter-rouge">ExecutionContext</code></li>
+ <li>Register files or directories stored on S3 with <code class="language-plaintext highlighter-rouge">ctx.register_listing_table</code></li>
+</ul>
+
+<h3 id="upcoming-enhancements-1">Upcoming enhancements</h3>
+
+<p>The current priority is adding python bindings for <code class="language-plaintext highlighter-rouge">S3FileSystem</code>. After that there will be async improvements as DataFusion adopts more of that functionality and we are looking into S3 Select functionality.</p>
+
+<h3 id="how-to-install-1">How to Install</h3>
+
+<p>Add the below to your <code class="language-plaintext highlighter-rouge">Cargo.toml</code> in your Rust Project with DataFusion.</p>
+
+<div class="language-toml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="py">datafusion-objectstore-s3</span> <span class="p">=</span> <span class="s">"0.1.0"</span>
+</code></pre></div></div>
+
+<h2 id="datafusion-substrait">DataFusion-Substrait</h2>
+
+<p><a href="https://substrait.io/">Substrait</a> is an emerging standard that provides a cross-language serialization format for relational algebra (e.g. expressions and query plans).</p>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-substrait">crate</a> provides a Substrait producer and consumer for DataFusion. A producer converts a DataFusion logical plan into a Substrait protobuf and a consumer does the reverse.</p>
+
+<p>Examples of how to use this crate can be found <a href="https://github.com/datafusion-contrib/datafusion-substrait/blob/main/src/lib.rs">here</a>.</p>
+
+<h3 id="potential-use-cases">Potential Use Cases</h3>
+
+<ul>
+ <li>Replace custom DataFusion protobuf serialization.</li>
+ <li>Make it easier to pass query plans over FFI boundaries, such as from Python to Rust</li>
+ <li>Allow Apache Calcite query plans to be executed in DataFusion</li>
+</ul>
+
+<h2 id="datafusion-bigtable">DataFusion-BigTable</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-bigtable">crate</a> implements <a href="https://cloud.google.com/bigtable">Bigtable</a> as a data source and physical executor for DataFusion queries. It currently supports both UTF-8 string and 64-bit big-endian signed integers in Bigtable. From a SQL perspective it supports both simple and composite row keys with <code class="language-plaintext highlighter-rouge">=</code>, <code class="language-plaintext highlighter-ro [...]
+
+<h3 id="upcoming-enhancements-2">Upcoming Enhancements</h3>
+
+<ul>
+ <li>Predicate pushdown
+ <ul>
+ <li>Value range</li>
+ <li>Value Regex</li>
+ <li>Timestamp range</li>
+ </ul>
+ </li>
+ <li>Multithreaded</li>
+ <li>Partition aware execution</li>
+ <li>Production ready</li>
+</ul>
+
+<h3 id="how-to-install-2">How to Install</h3>
+
+<p>Add the below to your <code class="language-plaintext highlighter-rouge">Cargo.toml</code> in your Rust Project with DataFusion.</p>
+
+<div class="language-toml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="py">datafusion-bigtable</span> <span class="p">=</span> <span class="s">"0.1.0"</span>
+</code></pre></div></div>
+
+<h2 id="datafusion-hdfs">DataFusion-HDFS</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-objectstore-hdfs">crate</a> introduces <code class="language-plaintext highlighter-rouge">HadoopFileSystem</code> as a remote <code class="language-plaintext highlighter-rouge">ObjectStore</code> which provides the ability to query HDFS files. For HDFS access the <a href="https://github.com/yahoNanJing/fs-hdfs">fs-hdfs</a> library is used.</p>
+
+<h2 id="datafusion-tokomak">DataFusion-Tokomak</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-tokomak">crate</a> provides an e-graph based DataFusion optimization framework based on the Rust <a href="https://egraphs-good.github.io">egg</a> library. An e-graph is a data structure that powers the equality saturation optimization technique.</p>
+
+<p>As context, the optimizer framework within DataFusion is currently <a href="https://github.com/apache/arrow-datafusion/issues/1972">under review</a> with the objective of implementing a more strategic long term solution that is more efficient and simpler to develop.</p>
+
+<p>Some of the benefits of using <code class="language-plaintext highlighter-rouge">egg</code> within DataFusion are:</p>
+
+<ul>
+ <li>Implements optimized algorithms that are hard to match with manually written optimization passes</li>
+ <li>Makes it easy and less verbose to add optimization rules</li>
+ <li>Plugin framework to add more complex optimizations</li>
+ <li>Egg does not depend on rule order and can lead to a higher level of optimization by being able to apply multiple rules at the same time until it converges</li>
+ <li>Allows for cost-based optimizations</li>
+</ul>
+
+<p>This is an exciting new area for DataFusion with lots of opportunity for community involvement!</p>
+
+<h2 id="datafusion-tui">DataFusion-Tui</h2>
+
+<p><a href="https://github.com/datafusion-contrib/datafusion-tui">DataFusion-tui</a> aka <code class="language-plaintext highlighter-rouge">dft</code> provides a feature rich terminal application for using DataFusion. It has drawn inspiration and several features from <code class="language-plaintext highlighter-rouge">datafusion-cli</code>. In contrast to <code class="language-plaintext highlighter-rouge">datafusion-cli</code> the objective of this tool is to provide a light SQL IDE ex [...]
+
+<ul>
+ <li>Tab Management to provide clean and structured organization of DataFusion queries, results, <code class="language-plaintext highlighter-rouge">ExecutionContext</code> information, and logs
+ <ul>
+ <li>SQL Editor
+ <ul>
+ <li>Text editor for writing SQL queries</li>
+ </ul>
+ </li>
+ <li>Query History
+ <ul>
+ <li>History of executed queries, their execution time, and the number of returned rows</li>
+ </ul>
+ </li>
+ <li><code class="language-plaintext highlighter-rouge">ExecutionContext</code> information
+ <ul>
+ <li>Expose information on which physical optimizers are used and which <code class="language-plaintext highlighter-rouge">ExecutionConfig</code> settings are set</li>
+ </ul>
+ </li>
+ <li>Logs
+ <ul>
+ <li>Logs from <code class="language-plaintext highlighter-rouge">dft</code>, DataFusion, and any dependent libraries</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li>Support for custom <code class="language-plaintext highlighter-rouge">ObjectStore</code>s
+ <ul>
+ <li>S3</li>
+ </ul>
+ </li>
+ <li>Preload DDL from <code class="language-plaintext highlighter-rouge">~/.datafusionrc</code> to enable having local “database” available at startup</li>
+</ul>
+
+<h3 id="upcoming-enhancements-3">Upcoming Enhancements</h3>
+
+<ul>
+ <li>SQL Editor
+ <ul>
+ <li>Command to write query results to file</li>
+ <li>Multiple SQL editor tabs</li>
+ </ul>
+ </li>
+ <li>Expose more information from <code class="language-plaintext highlighter-rouge">ExecutionContext</code></li>
+ <li>A help tab that provides information on functions</li>
+ <li>Query custom <code class="language-plaintext highlighter-rouge">TableProvider</code>s such as <a href="https://github.com/delta-io/delta-rs">DeltaTable</a> or <a href="https://github.com/datafusion-contrib/datafusion-bigtable">BigTable</a></li>
+</ul>
+
+<h2 id="datafusion-streams">DataFusion-Streams</h2>
+
+<p><a href="https://github.com/datafusion-contrib/datafusion-streams">DataFusion-Stream</a> is a new testing ground for creating a <code class="language-plaintext highlighter-rouge">StreamProvider</code> in DataFusion that will enable querying streaming data sources such as Apache Kafka. The implementation for this feature is currently being designed and is under active review. Once the design is finalized the trait and attendant data structures will be added back to the core DataFusio [...]
+
+<h2 id="datafusion-java">DataFusion-Java</h2>
+
+<p>This <a href="https://github.com/datafusion-contrib/datafusion-java">project</a> created an initial set of Java bindings to DataFusion. The project is currently in maintenance mode and is looking for maintainers to drive future development.</p>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to DataFusion, and learning about state of
+the art query processing, we would love to have you join us on the journey! You
+can help by trying out DataFusion on some of your own data and projects and let us know how it goes or contribute a PR with documentation, tests or code. A list of open issues suitable for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a></p>
+
+<p>The best way to find out about creating new extensions within DataFusion-Contrib is reaching out on the <code class="language-plaintext highlighter-rouge">#arrow-rust</code> channel of the Apache Software Foundation <a href="https://join.slack.com/t/the-asf/shared_invite/zt-vlfbf7ch-HkbNHiU_uDlcH_RvaHv9gQ">Slack</a> workspace.</p>
+
+<p>You can also check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more ways to engage with the community.</p>
+
+<p>Links for each DataFusion-Contrib repository are provided above if you would like to contribute to those.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[<!–]]></summary></entry></feed>
\ No newline at end of file
diff --git a/img/2021-08-18-datafusion500perf.png b/img/2021-08-18-datafusion500perf.png
new file mode 100644
index 0000000..4406473
Binary files /dev/null and b/img/2021-08-18-datafusion500perf.png differ
diff --git a/img/2x_bgwhite_original.png b/img/2x_bgwhite_original.png
new file mode 100644
index 0000000..abb5fca
Binary files /dev/null and b/img/2x_bgwhite_original.png differ
diff --git a/img/datafusion-34.0.0/compare-new.png b/img/datafusion-34.0.0/compare-new.png
new file mode 100644
index 0000000..9e2ae22
Binary files /dev/null and b/img/datafusion-34.0.0/compare-new.png differ
diff --git a/img/datafusion-34.0.0/compare.png b/img/datafusion-34.0.0/compare.png
new file mode 100644
index 0000000..8360737
Binary files /dev/null and b/img/datafusion-34.0.0/compare.png differ
diff --git a/img/datafusion-comet/comet-architecture.png b/img/datafusion-comet/comet-architecture.png
new file mode 100644
index 0000000..e7c9075
Binary files /dev/null and b/img/datafusion-comet/comet-architecture.png differ
diff --git a/index.html b/index.html
new file mode 100644
index 0000000..36a3b80
--- /dev/null
+++ b/index.html
@@ -0,0 +1,152 @@
+<!DOCTYPE html>
+<html lang="en"><head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
+<title>Apache DataFusion Project News & Blog | Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</title>
+<meta name="generator" content="Jekyll v4.3.3" />
+<meta property="og:title" content="Apache DataFusion Project News & Blog" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format." />
+<meta property="og:description" content="Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format." />
+<link rel="canonical" href="https://datafusion.apache.org/blog/" />
+<meta property="og:url" content="https://datafusion.apache.org/blog/" />
+<meta property="og:site_name" content="Apache DataFusion Project News & Blog" />
+<meta property="og:type" content="website" />
+<meta name="twitter:card" content="summary" />
+<meta property="twitter:title" content="Apache DataFusion Project News & Blog" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"WebSite","description":"Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.","headline":"Apache DataFusion Project News & Blog","name":"Apache DataFusion Project News & Blog","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://datafusion.apache.org/blog/img/2x_bgwhite_original.png"}},"url":"https://datafusion [...]
+<!-- End Jekyll SEO tag -->
+<link rel="stylesheet" href="/blog/assets/main.css"><link type="application/atom+xml" rel="alternate" href="https://datafusion.apache.org/blog/feed.xml" title="Apache DataFusion Project News & Blog" /></head>
+<body><header class="site-header" role="banner">
+
+ <div class="wrapper"><a class="site-title" rel="author" href="/blog/">Apache DataFusion Project News & Blog</a><nav class="site-nav">
+ <input type="checkbox" id="nav-trigger" class="nav-trigger" />
+ <label for="nav-trigger">
+ <span class="menu-icon">
+ <svg viewBox="0 0 18 15" width="18px" height="15px">
+ <path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
+ </svg>
+ </span>
+ </label>
+
+ <div class="trigger"><a class="page-link" href="/blog/about/">About</a></div>
+ </nav></div>
+</header>
+<main class="page-content" aria-label="Content">
+ <div class="wrapper">
+ <div class="home">
+<h2 class="post-list-heading">Posts</h2>
+ <ul class="post-list"><li><span class="post-meta">May 7, 2024</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2024/05/07/datafusion-tlp/">
+ Announcing Apache Arrow DataFusion is now Apache DataFusion
+ </a>
+ </h3></li><li><span class="post-meta">Mar 6, 2024</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2024/03/06/comet-donation/">
+ Announcing Apache Arrow DataFusion Comet
+ </a>
+ </h3></li><li><span class="post-meta">Jan 19, 2024</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2024/01/19/datafusion-34.0.0/">
+ Apache Arrow DataFusion 34.0.0 Released, Looking Forward to 2024
+ </a>
+ </h3></li><li><span class="post-meta">Aug 5, 2023</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2023/08/05/datafusion_fast_grouping/">
+ Aggregating Millions of Groups Fast in Apache Arrow DataFusion 28.0.0
+ </a>
+ </h3></li><li><span class="post-meta">Jun 24, 2023</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2023/06/24/datafusion-25.0.0/">
+ Apache Arrow DataFusion 26.0.0
+ </a>
+ </h3></li><li><span class="post-meta">Jan 19, 2023</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2023/01/19/datafusion-16.0.0/">
+ Apache Arrow DataFusion 16.0.0 Project Update
+ </a>
+ </h3></li><li><span class="post-meta">Oct 28, 2022</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2022/10/28/ballista-0.9.0/">
+ Apache Arrow Ballista 0.9.0 Release
+ </a>
+ </h3></li><li><span class="post-meta">Oct 25, 2022</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2022/10/25/datafusion-13.0.0/">
+ Apache Arrow DataFusion 13.0.0 Project Update
+ </a>
+ </h3></li><li><span class="post-meta">May 16, 2022</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2022/05/16/datafusion-8.0.0/">
+ Apache Arrow DataFusion 8.0.0 Release
+ </a>
+ </h3></li><li><span class="post-meta">Mar 21, 2022</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2022/03/21/datafusion-contrib/">
+ Introducing Apache Arrow DataFusion Contrib
+ </a>
+ </h3></li><li><span class="post-meta">Feb 28, 2022</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2022/02/28/datafusion-7.0.0/">
+ Apache Arrow DataFusion 7.0.0 Release
+ </a>
+ </h3></li><li><span class="post-meta">Nov 19, 2021</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2021/11/19/datafusion-6.0.0/">
+ Apache Arrow DataFusion 6.0.0 Release
+ </a>
+ </h3></li><li><span class="post-meta">Aug 18, 2021</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2021/08/18/datafusion-5.0.0/">
+ Apache Arrow DataFusion 5.0.0 Release
+ </a>
+ </h3></li><li><span class="post-meta">Aug 18, 2021</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2021/08/18/ballista-0.5.0/">
+ Apache Arrow Ballista 0.5.0 Release
+ </a>
+ </h3></li><li><span class="post-meta">Apr 12, 2021</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2021/04/12/ballista-donation/">
+ Ballista: A Distributed Scheduler for Apache Arrow
+ </a>
+ </h3></li><li><span class="post-meta">Feb 3, 2019</span>
+ <h3>
+ <a class="post-link" href="/blog/blog/2019/02/03/datafusion-donation/">
+ DataFusion: A Rust-native Query Engine for Apache Arrow
+ </a>
+ </h3></li></ul>
+
+ <p class="rss-subscribe">subscribe <a href="/blog/feed.xml">via RSS</a></p></div>
+
+ </div>
+ </main><footer class="site-footer h-card">
+ <data class="u-url" href="/blog/"></data>
+
+ <div class="wrapper">
+
+ <h2 class="footer-heading">Apache DataFusion Project News & Blog</h2>
+
+ <div class="footer-col-wrapper">
+ <div class="footer-col footer-col-1">
+ <ul class="contact-list">
+ <li class="p-name">Apache DataFusion Project News & Blog</li><li><a class="u-email" href="mailto:dev@datafusion.apache.org">dev@datafusion.apache.org</a></li></ul>
+ </div>
+
+ <div class="footer-col footer-col-2"><ul class="social-media-list"><li><a href="https://github.com/apache"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#github"></use></svg> <span class="username">apache</span></a></li><li><a href="https://www.twitter.com/ApacheDataFusio"><svg class="svg-icon"><use xlink:href="/blog/assets/minima-social-icons.svg#twitter"></use></svg> <span class="username">ApacheDataFusio</span></a></li></ul>
+</div>
+
+ <div class="footer-col footer-col-3">
+ <p>Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in Rust, using the Apache Arrow in-memory format.</p>
+ </div>
+ </div>
+
+ </div>
+
+</footer>
+</body>
+
+</html>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org