You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by gi...@apache.org on 2022/11/07 12:21:56 UTC
[arrow-site] branch asf-site updated: Updating built site (build 4920b06c289f4c48f7f522f42cbbd6479daf08e9)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/arrow-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new a2a647a7f9 Updating built site (build 4920b06c289f4c48f7f522f42cbbd6479daf08e9)
a2a647a7f9 is described below
commit a2a647a7f9076c423bb3e5a70447193a641c8def
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Mon Nov 7 12:21:50 2022 +0000
Updating built site (build 4920b06c289f4c48f7f522f42cbbd6479daf08e9)
---
.../index.html | 492 ++++++++++++++
.../index.html | 486 ++++++++++++++
blog/index.html | 30 +
docs/c_glib/index.html | 4 +-
feed.xml | 738 ++++++++++++---------
release/0.1.0.html | 4 +-
release/0.10.0.html | 4 +-
release/0.11.0.html | 4 +-
release/0.11.1.html | 4 +-
release/0.12.0.html | 4 +-
release/0.13.0.html | 4 +-
release/0.14.0.html | 4 +-
release/0.14.1.html | 4 +-
release/0.15.0.html | 4 +-
release/0.15.1.html | 4 +-
release/0.16.0.html | 4 +-
release/0.17.0.html | 4 +-
release/0.17.1.html | 4 +-
release/0.2.0.html | 4 +-
release/0.3.0.html | 4 +-
release/0.4.0.html | 4 +-
release/0.4.1.html | 4 +-
release/0.5.0.html | 4 +-
release/0.6.0.html | 4 +-
release/0.7.0.html | 4 +-
release/0.7.1.html | 4 +-
release/0.8.0.html | 4 +-
release/0.9.0.html | 4 +-
release/1.0.0.html | 4 +-
release/1.0.1.html | 4 +-
release/10.0.0.html | 4 +-
release/2.0.0.html | 4 +-
release/3.0.0.html | 4 +-
release/4.0.0.html | 4 +-
release/4.0.1.html | 4 +-
release/5.0.0.html | 4 +-
release/6.0.0.html | 4 +-
release/6.0.1.html | 4 +-
release/7.0.0.html | 4 +-
release/8.0.0.html | 4 +-
release/9.0.0.html | 4 +-
release/index.html | 4 +-
42 files changed, 1509 insertions(+), 389 deletions(-)
diff --git a/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/index.html b/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/index.html
new file mode 100644
index 0000000000..780f41d183
--- /dev/null
+++ b/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/index.html
@@ -0,0 +1,492 @@
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above meta tags *must* come first in the head; any other head content must come *after* these tags -->
+
+ <title>Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 2 | Apache Arrow</title>
+
+
+ <!-- Begin Jekyll SEO tag v2.8.0 -->
+<meta name="generator" content="Jekyll v4.2.0" />
+<meta property="og:title" content="Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 2" />
+<meta name="author" content="tustvold and alamb" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Introduction In Part 1 of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new row format in the Rust implementation of Apache Arrow works and is constructed. Row Format The row format is a variable length byte sequence created by concatenating the encoded form of each column. The encoding for each column depends on its datatype (and sort options). ┌─────┐ ┌── [...]
+<meta property="og:description" content="Introduction In Part 1 of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new row format in the Rust implementation of Apache Arrow works and is constructed. Row Format The row format is a variable length byte sequence created by concatenating the encoded form of each column. The encoding for each column depends on its datatype (and sort options). ┌─── [...]
+<link rel="canonical" href="https://arrow.apache.org/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/" />
+<meta property="og:url" content="https://arrow.apache.org/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/" />
+<meta property="og:site_name" content="Apache Arrow" />
+<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-10-30T00:00:00-04:00" />
+<meta name="twitter:card" content="summary_large_image" />
+<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
+<meta property="twitter:title" content="Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 2" />
+<meta name="twitter:site" content="@ApacheArrow" />
+<meta name="twitter:creator" content="@tustvold and alamb" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"tustvold and alamb"},"dateModified":"2022-10-30T00:00:00-04:00","datePublished":"2022-10-30T00:00:00-04:00","description":"Introduction In Part 1 of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new row format in the Rust implementation of Apache Arrow works and is constructed. Row Format The row forma [...]
+<!-- End Jekyll SEO tag -->
+
+
+ <!-- favicons -->
+ <link rel="icon" type="image/png" sizes="16x16" href="/img/favicon-16x16.png" id="light1">
+ <link rel="icon" type="image/png" sizes="32x32" href="/img/favicon-32x32.png" id="light2">
+ <link rel="apple-touch-icon" type="image/png" sizes="180x180" href="/img/apple-touch-icon.png" id="light3">
+ <link rel="apple-touch-icon" type="image/png" sizes="120x120" href="/img/apple-touch-icon-120x120.png" id="light4">
+ <link rel="apple-touch-icon" type="image/png" sizes="76x76" href="/img/apple-touch-icon-76x76.png" id="light5">
+ <link rel="apple-touch-icon" type="image/png" sizes="60x60" href="/img/apple-touch-icon-60x60.png" id="light6">
+ <!-- dark mode favicons -->
+ <link rel="icon" type="image/png" sizes="16x16" href="/img/favicon-16x16-dark.png" id="dark1">
+ <link rel="icon" type="image/png" sizes="32x32" href="/img/favicon-32x32-dark.png" id="dark2">
+ <link rel="apple-touch-icon" type="image/png" sizes="180x180" href="/img/apple-touch-icon-dark.png" id="dark3">
+ <link rel="apple-touch-icon" type="image/png" sizes="120x120" href="/img/apple-touch-icon-120x120-dark.png" id="dark4">
+ <link rel="apple-touch-icon" type="image/png" sizes="76x76" href="/img/apple-touch-icon-76x76-dark.png" id="dark5">
+ <link rel="apple-touch-icon" type="image/png" sizes="60x60" href="/img/apple-touch-icon-60x60-dark.png" id="dark6">
+
+ <script>
+ // Switch to the dark-mode favicons if prefers-color-scheme: dark
+ function onUpdate() {
+ light1 = document.querySelector('link#light1');
+ light2 = document.querySelector('link#light2');
+ light3 = document.querySelector('link#light3');
+ light4 = document.querySelector('link#light4');
+ light5 = document.querySelector('link#light5');
+ light6 = document.querySelector('link#light6');
+
+ dark1 = document.querySelector('link#dark1');
+ dark2 = document.querySelector('link#dark2');
+ dark3 = document.querySelector('link#dark3');
+ dark4 = document.querySelector('link#dark4');
+ dark5 = document.querySelector('link#dark5');
+ dark6 = document.querySelector('link#dark6');
+
+ if (matcher.matches) {
+ light1.remove();
+ light2.remove();
+ light3.remove();
+ light4.remove();
+ light5.remove();
+ light6.remove();
+ document.head.append(dark1);
+ document.head.append(dark2);
+ document.head.append(dark3);
+ document.head.append(dark4);
+ document.head.append(dark5);
+ document.head.append(dark6);
+ } else {
+ dark1.remove();
+ dark2.remove();
+ dark3.remove();
+ dark4.remove();
+ dark5.remove();
+ dark6.remove();
+ document.head.append(light1);
+ document.head.append(light2);
+ document.head.append(light3);
+ document.head.append(light4);
+ document.head.append(light5);
+ document.head.append(light6);
+ }
+ }
+ matcher = window.matchMedia('(prefers-color-scheme: dark)');
+ matcher.addListener(onUpdate);
+ onUpdate();
+ </script>
+
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+ <link href="/css/main.css" rel="stylesheet">
+ <link href="/css/syntax.css" rel="stylesheet">
+ <script src="/javascript/main.js"></script>
+
+ <!-- Matomo -->
+<script>
+ var _paq = window._paq = window._paq || [];
+ /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+ /* We explicitly disable cookie tracking to avoid privacy issues */
+ _paq.push(['disableCookies']);
+ _paq.push(['trackPageView']);
+ _paq.push(['enableLinkTracking']);
+ (function() {
+ var u="https://analytics.apache.org/";
+ _paq.push(['setTrackerUrl', u+'matomo.php']);
+ _paq.push(['setSiteId', '20']);
+ var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+ g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
+ })();
+</script>
+<!-- End Matomo Code -->
+
+
+ </head>
+
+
+<body class="wrap">
+ <header>
+ <nav class="navbar navbar-expand-md navbar-dark bg-dark">
+
+ <a class="navbar-brand no-padding" href="/"><img src="/img/arrow-inverse-300px.png" height="40px"/></a>
+
+ <button class="navbar-toggler ml-auto" type="button" data-toggle="collapse" data-target="#arrow-navbar" aria-controls="arrow-navbar" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse justify-content-end" id="arrow-navbar">
+ <ul class="nav navbar-nav">
+ <li class="nav-item"><a class="nav-link" href="/overview/" role="button" aria-haspopup="true" aria-expanded="false">Overview</a></li>
+ <li class="nav-item"><a class="nav-link" href="/faq/" role="button" aria-haspopup="true" aria-expanded="false">FAQ</a></li>
+ <li class="nav-item"><a class="nav-link" href="/blog" role="button" aria-haspopup="true" aria-expanded="false">Blog</a></li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownGetArrow" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Get Arrow
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownGetArrow">
+ <a class="dropdown-item" href="/install/">Install</a>
+ <a class="dropdown-item" href="/release/">Releases</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow">Source Code</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownDocumentation" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Documentation
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownDocumentation">
+ <a class="dropdown-item" href="/docs">Project Docs</a>
+ <a class="dropdown-item" href="/docs/format/Columnar.html">Format</a>
+ <hr/>
+ <a class="dropdown-item" href="/docs/c_glib">C GLib</a>
+ <a class="dropdown-item" href="/docs/cpp">C++</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow/blob/master/csharp/README.md">C#</a>
+ <a class="dropdown-item" href="https://godoc.org/github.com/apache/arrow/go/arrow">Go</a>
+ <a class="dropdown-item" href="/docs/java">Java</a>
+ <a class="dropdown-item" href="/docs/js">JavaScript</a>
+ <a class="dropdown-item" href="https://arrow.juliadata.org/stable/">Julia</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow/blob/master/matlab/README.md">MATLAB</a>
+ <a class="dropdown-item" href="/docs/python">Python</a>
+ <a class="dropdown-item" href="/docs/r">R</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow/blob/master/ruby/README.md">Ruby</a>
+ <a class="dropdown-item" href="https://docs.rs/crate/arrow/">Rust</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownSubprojects" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Subprojects
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownSubprojects">
+ <a class="dropdown-item" href="/docs/format/Flight.html">Arrow Flight</a>
+ <a class="dropdown-item" href="/docs/format/FlightSql.html">Arrow Flight SQL</a>
+ <a class="dropdown-item" href="/datafusion">DataFusion</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownCommunity" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Community
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownCommunity">
+ <a class="dropdown-item" href="/community/">Communication</a>
+ <a class="dropdown-item" href="/docs/developers/contributing.html">Contributing</a>
+ <a class="dropdown-item" href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a>
+ <a class="dropdown-item" href="/committers/">Governance</a>
+ <a class="dropdown-item" href="/use_cases/">Use Cases</a>
+ <a class="dropdown-item" href="/powered_by/">Powered By</a>
+ <a class="dropdown-item" href="/visual_identity/">Visual Identity</a>
+ <a class="dropdown-item" href="/security/">Security</a>
+ <a class="dropdown-item" href="https://www.apache.org/foundation/policies/conduct.html">Code of Conduct</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownASF" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ ASF Links
+ </a>
+ <div class="dropdown-menu dropdown-menu-right" aria-labelledby="navbarDropdownASF">
+ <a class="dropdown-item" href="https://www.apache.org/">ASF Website</a>
+ <a class="dropdown-item" href="https://www.apache.org/licenses/">License</a>
+ <a class="dropdown-item" href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
+ <a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a>
+ <a class="dropdown-item" href="https://www.apache.org/security/">Security</a>
+ </div>
+ </li>
+ </ul>
+ </div><!-- /.navbar-collapse -->
+ </nav>
+
+ </header>
+
+ <div class="container p-4 pt-5">
+ <div class="col-md-8 mx-auto">
+ <main role="main" class="pb-5">
+
+<h1>
+ Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 2
+</h1>
+<hr class="mt-4 mb-3">
+
+
+
+<p class="mb-4 pb-1">
+ <span class="badge badge-secondary">Published</span>
+ <span class="published mr-3">
+ 30 Oct 2022
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+
+ tustvold and alamb
+
+
+
+</p>
+
+
+ <!--
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>In <a href="/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">Part 1</a> of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new <a href="https://docs.rs/arrow/25.0.0/arrow/row/index.html">row format</a> in the <a href="https://github.com/apache/arrow-rs">Rust implementation</a> of <a href="https://arrow.apache.org/">Apache Arrow</a> works and is constructed.</p>
+
+<h2 id="row-format">Row Format</h2>
+
+<p>The row format is a variable length byte sequence created by concatenating the encoded form of each column. The encoding for each column depends on its datatype (and sort options).</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌─────┐ ┌─────┐ ┌─────┐
+ │ │ │ │ │ │
+ ├─────┤ ┌ ┼─────┼ ─ ┼─────┼ ┐ ┏━━━━━━━━━━━━━┓
+ │ │ │ │ │ │ ─────────────▶┃ ┃
+ ├─────┤ └ ┼─────┼ ─ ┼─────┼ ┘ ┗━━━━━━━━━━━━━┛
+ │ │ │ │ │ │
+ └─────┘ └─────┘ └─────┘
+ ...
+ ┌─────┐ ┌ ┬─────┬ ─ ┬─────┬ ┐ ┏━━━━━━━━┓
+ │ │ │ │ │ │ ─────────────▶┃ ┃
+ └─────┘ └ ┴─────┴ ─ ┴─────┴ ┘ ┗━━━━━━━━┛
+ Customer State Orders
+ UInt64 Utf8 F64
+
+ Input Arrays Row Format
+ (Columns)
+</code></pre></div></div>
+
+<p>The encoding is carefully designed in such a way that escaping is unnecessary: it is never ambiguous as to whether a byte is part of a sentinel (e.g. null) or a value.</p>
+
+<h3 id="unsigned-integers">Unsigned Integers</h3>
+
+<p>To encode a non-null unsigned integer, the byte <code class="language-plaintext highlighter-rouge">0x01</code> is written, followed by the integer’s bytes starting with the most significant, i.e. big endian. A null is encoded as a <code class="language-plaintext highlighter-rouge">0x00</code> byte, followed by the encoded bytes of the integer’s zero value</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 3 │03│00│00│00│ │01│00│00│00│03│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 258 │02│01│00│00│ │01│00│00│01│02│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 23423 │7F│5B│00│00│ │01│00│00│5B│7F│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ NULL │??│??│??│??│ │00│00│00│00│00│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+
+ 32-bit (4 bytes) Row Format
+ Value Little Endian
+</code></pre></div></div>
+
+<h3 id="signed-integers">Signed Integers</h3>
+
+<p>In Rust and most modern computer architectures, signed integers are encoded using <a href="https://en.wikipedia.org/wiki/Two%27s_complement">two’s complement</a>, where a number is negated by flipping all the bits, and adding 1. Therefore, flipping the top-most bit and treating the result as an unsigned integer preserves the order. This unsigned integer can then be encoded using the same encoding for unsigned integers described in the previous section. For example</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 5 │05│00│00│00│ │05│00│00│80│ │01│80│00│00│05│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ -5 │FB│FF│FF│FF│ │FB│FF│FF│7F│ │01│7F│FF│FF│FB│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+
+ Value 32-bit (4 bytes) High bit flipped Row Format
+ Little Endian
+</code></pre></div></div>
+
+<h3 id="floating-point">Floating Point</h3>
+
+<p>Floating point values can be ordered according to the <a href="https://en.wikipedia.org/wiki/IEEE_754#Total-ordering_predicate">IEEE 754 totalOrder predicate</a> (implemented in Rust by <a href="https://doc.rust-lang.org/std/primitive.f32.html#method.total_cmp">f32::total_cmp</a>). This ordering interprets the bytes of the floating point value as the correspondingly sized, signed, little-endian integer, flipping all the bits except the sign bit in the case of negatives.</p>
+
+<p>Floating point values are therefore encoded to row format by converting them to the appropriate sized signed integer representation, and then using the same encoding for signed integers described in the previous section.</p>
+
+<h3 id="byte-arrays-including-strings">Byte Arrays (Including Strings)</h3>
+
+<p>Unlike primitive types above, byte arrays are variable length. For short strings, such as <code class="language-plaintext highlighter-rouge">state</code> in our example above, it is possible to pad all values to the length of the longest one with some fixed value such as <code class="language-plaintext highlighter-rouge">0x00</code> and produce a fixed length row. This is the approach described in the DuckDB blog for encoding <code class="language-plaintext highlighter-rouge">c_birth_ [...]
+
+<p>However, often values in string columns differ substantially in length or the maximum length is not known at the start of execution, making it inadvisable and/or impractical to pad the strings to a fixed length. The Rust Arrow row format therefore uses a variable length encoding.</p>
+
+<p>We need an encoding that unambiguously terminates the end of the byte array. This not only permits recovering the original value from the row format, but ensures that bytes of a longer byte array are not compared against bytes from a different column when compared against a row containing a shorter byte array.</p>
+
+<p>A null byte array is encoded as a single <code class="language-plaintext highlighter-rouge">0x00</code> byte. Similarly, an empty byte array is encoded as a single <code class="language-plaintext highlighter-rouge">0x01</code> byte.</p>
+
+<p>To encode a non-null, non-empty array, first a single <code class="language-plaintext highlighter-rouge">0x02</code> byte is written. Then the array is written in 32-byte blocks, with each complete block followed by a <code class="language-plaintext highlighter-rouge">0xFF</code> byte as a continuation token. The final block is padded to 32-bytes with <code class="language-plaintext highlighter-rouge">0x00</code>, and is then followed by the unpadded length of this final block as a s [...]
+
+<p>Note the following example encodings use a block size of 4 bytes, as opposed to 32 bytes for brevity</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌───┬───┬───┬───┬───┬───┐
+ "MEEP" │02 │'M'│'E'│'E'│'P'│04 │
+ └───┴───┴───┴───┴───┴───┘
+
+ ┌───┐
+ "" │01 |
+ └───┘
+
+ NULL ┌───┐
+ │00 │
+ └───┘
+
+"Defenestration" ┌───┬───┬───┬───┬───┬───┐
+ │02 │'D'│'e'│'f'│'e'│FF │
+ └───┼───┼───┼───┼───┼───┤
+ │'n'│'e'│'s'│'t'│FF │
+ ├───┼───┼───┼───┼───┤
+ │'r'│'a'│'t'│'r'│FF │
+ ├───┼───┼───┼───┼───┤
+ │'a'│'t'│'i'│'o'│FF │
+ ├───┼───┼───┼───┼───┤
+ │'n'│00 │00 │00 │17 │
+ └───┴───┴───┴───┴───┘
+</code></pre></div></div>
+
+<p>This approach is loosely inspired by <a href="https://en.wikipedia.org/wiki/Consistent_Overhead_Byte_Stuffing">COBS encoding</a>, and chosen over more traditional <a href="https://en.wikipedia.org/wiki/High-Level_Data_Link_Control#Asynchronous_framing">byte stuffing</a> as it is more amenable to vectorization, in particular hardware with AVX-256 can copy a 32-byte block in a single instruction.</p>
+
+<h3 id="dictionary-arrays">Dictionary Arrays</h3>
+<p>Dictionary Encoded Data (called <a href="https://pandas.pydata.org/docs/user_guide/categorical.html">categorical</a> in pandas) is increasingly important because they can store and process low cardinality data very efficiently.</p>
+
+<p>A simple approach to encoding dictionary arrays would be to encode the logical values directly using the encodings for primitive values described previously. However, this would lose the benefits of dictionary encoding to reduce memory and CPU consumption.</p>
+
+<p>To further complicate matters, the <a href="https://arrow.apache.org/docs/format/Columnar.html#dictionary-encoded-layout">Arrow implementation of Dictionary encoding</a> is quite general, and we can make no assumptions about the contents of the dictionaries. In particular, we cannot assume that the dictionary values are sorted, nor that the same dictionary is used for all arrays within a column</p>
+
+<p>The following example shows how a string column might be encoded in two arrays using two different dictionaries. The dictionary keys <code class="language-plaintext highlighter-rouge">0</code>, <code class="language-plaintext highlighter-rouge">1</code>, and <code class="language-plaintext highlighter-rouge">2</code> in the first batch correspond to different values than the same keys in the second dictionary.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
+ ┌───────────┐ ┌─────┐ │
+│ │"Fabulous" │ │ 0 │
+ ├───────────┤ ├─────┤ │
+│ │ "Bar" │ │ 2 │
+ ├───────────┤ ├─────┤ │ ┌───────────┐
+│ │ "Soup" │ │ 2 │ │"Fabulous" │
+ └───────────┘ ├─────┤ │ ├───────────┤
+│ │ 0 │ │ "Soup" │
+ ├─────┤ │ ├───────────┤
+│ │ 1 │ │ "Soup" │
+ └─────┘ │ ├───────────┤
+│ │"Fabulous" │
+ Values │ ├───────────┤
+│ Dictionary (indexes in │ "Bar" │
+ dictionary) │ ├───────────┤
+│ │ "ZZ" │
+ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ├───────────┤
+┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ "Bar" │
+ │ ├───────────┤
+│ ┌───────────┐ ┌─────┐ │ "ZZ" │
+ │"Fabulous" │ │ 1 │ │ ├───────────┤
+│ ├───────────┤ ├─────┤ │"Fabulous" │
+ │ "ZZ" │ │ 2 │ │ └───────────┘
+│ ├───────────┤ ├─────┤
+ │ "Bar" │ │ 1 │ │
+│ └───────────┘ ├─────┤
+ │ 0 │ │ Logical column
+│ └─────┘ values
+ Values │
+│ Dictionary (indexes in
+ dictionary) │
+ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘
+</code></pre></div></div>
+
+<p>The key observation which allows us to efficiently create a row format for this kind of data is that given a byte array, a new byte array can always be created which comes before or after it in the sort order by adding an additional byte.</p>
+
+<p>Therefore we can incrementally build an order-preserving mapping from dictionary values to variable length byte arrays, without needing to know all possible dictionary values beforehand, instead introducing mappings for new dictionary values as we encounter them.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>┌──────────┐ ┌─────┐
+│ "Bar" │ ───────────────▶│ 01 │
+└──────────┘ └─────┘
+┌──────────┐ ┌─────┬─────┐
+│"Fabulous"│ ───────────────▶│ 01 │ 02 │
+└──────────┘ └─────┴─────┘
+┌──────────┐ ┌─────┐
+│ "Soup" │ ───────────────▶│ 05 │
+└──────────┘ └─────┘
+┌──────────┐ ┌─────┐
+│ "ZZ" │ ───────────────▶│ 07 │
+└──────────┘ └─────┘
+
+ Example Order Preserving Mapping
+</code></pre></div></div>
+
+<p>The details of the data structure used to generate this mapping are beyond the scope of this blog post, but may be the topic of a future post. You can find <a href="https://github.com/apache/arrow-rs/blob/07024f6a16b870fda81cba5779b8817b20386ebf/arrow/src/row/interner.rs">the code here</a>.</p>
+
+<p>The data structure also ensures that no values contain <code class="language-plaintext highlighter-rouge">0x00</code> and therefore we can encode the arrays directly using <code class="language-plaintext highlighter-rouge">0x00</code> as an end-delimiter.</p>
+
+<p>A null value is encoded as a single <code class="language-plaintext highlighter-rouge">0x00</code> byte, and a non-null value encoded as a single <code class="language-plaintext highlighter-rouge">0x01</code> byte, followed by the <code class="language-plaintext highlighter-rouge">0x00</code> terminated byte array determined by the order preserving mapping</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌─────┬─────┬─────┬─────┐
+ "Fabulous" │ 01 │ 03 │ 05 │ 00 │
+ └─────┴─────┴─────┴─────┘
+
+ ┌─────┬─────┬─────┐
+ "ZZ" │ 01 │ 07 │ 00 │
+ └─────┴─────┴─────┘
+
+ ┌─────┐
+ NULL │ 00 │
+ └─────┘
+
+ Input Row Format
+</code></pre></div></div>
+
+<h3 id="sort-options">Sort Options</h3>
+
+<p>One detail we have so far ignored over is how to support ascending and descending sorts (e.g. <code class="language-plaintext highlighter-rouge">ASC</code> or <code class="language-plaintext highlighter-rouge">DESC</code> in SQL). The Arrow Rust row format supports these options by simply inverting the bytes of the encoded representation, except the initial byte used for nullability encoding, on a per column basis.</p>
+
+<p>Similarly, supporting SQL compatible sorting also requires a format that can specify the order of <code class="language-plaintext highlighter-rouge">NULL</code>s (before or after all non <code class="language-plaintext highlighter-rouge">NULL</code> values). The row format supports this option by optionally encoding nulls as <code class="language-plaintext highlighter-rouge">0xFF</code> instead of <code class="language-plaintext highlighter-rouge">0x00</code> on a per column basis.</p>
+
+<h2 id="conclusion">Conclusion</h2>
+
+<p>Hopefully these two articles have given you a flavor of what is possible with a comparable row format and how it works. Feel free to check out the <a href="https://docs.rs/arrow/latest/arrow/row/index.html">docs</a> for instructions on getting started, and report any issues on our <a href="https://github.com/apache/arrow-rs/issues">bugtracker</a>.</p>
+
+<p>Using this format for lexicographic sorting is more than <a href="https://github.com/apache/arrow-rs/pull/2929">3x</a> faster than the comparator based approach, with the benefits especially pronounced for strings, dictionaries and sorts with large numbers of columns.</p>
+
+<p>We have also already used it to more than <a href="https://github.com/apache/arrow-datafusion/pull/3386">double</a> the performance of sort preserving merge in the <a href="https://arrow.apache.org/datafusion/">DataFusion project</a>, and expect similar or greater performance uplift as we apply it to sort, grouping, join, and window function operators as well.</p>
+
+<p>As always, the <a href="https://github.com/apache/arrow-rs#arrow-rust-community">Arrow community</a> very much looks forward to seeing what you build with it!</p>
+
+ </main>
+ </div>
+
+ <hr/>
+<footer class="footer">
+ <div class="row">
+ <div class="col-md-9">
+ <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ <p>© 2016-2022 The Apache Software Foundation</p>
+ </div>
+ <div class="col-md-3">
+ <a class="d-sm-none d-md-inline pr-2" href="https://www.apache.org/events/current-event.html">
+ <img src="https://www.apache.org/events/current-event-234x60.png"/>
+ </a>
+ </div>
+ </div>
+</footer>
+
+ </div>
+</body>
+</html>
diff --git a/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/index.html b/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/index.html
new file mode 100644
index 0000000000..6844a83a4c
--- /dev/null
+++ b/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/index.html
@@ -0,0 +1,486 @@
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above meta tags *must* come first in the head; any other head content must come *after* these tags -->
+
+ <title>Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 1 | Apache Arrow</title>
+
+
+ <!-- Begin Jekyll SEO tag v2.8.0 -->
+<meta name="generator" content="Jekyll v4.2.0" />
+<meta property="og:title" content="Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 1" />
+<meta name="author" content="tustvold and alamb" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Introduction Sorting is one of the most fundamental operations in modern databases and other analytic systems, underpinning important operators such as aggregates, joins, window functions, merge, and more. By some estimates, more than half of the execution time in data processing systems is spent sorting. Optimizing sorts is therefore vital to improving query performance and overall system efficiency. Sorting is also one of the most well studied topics i [...]
+<meta property="og:description" content="Introduction Sorting is one of the most fundamental operations in modern databases and other analytic systems, underpinning important operators such as aggregates, joins, window functions, merge, and more. By some estimates, more than half of the execution time in data processing systems is spent sorting. Optimizing sorts is therefore vital to improving query performance and overall system efficiency. Sorting is also one of the most well studied t [...]
+<link rel="canonical" href="https://arrow.apache.org/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/" />
+<meta property="og:url" content="https://arrow.apache.org/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/" />
+<meta property="og:site_name" content="Apache Arrow" />
+<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2022-11-07T00:00:00-05:00" />
+<meta name="twitter:card" content="summary_large_image" />
+<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
+<meta property="twitter:title" content="Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 1" />
+<meta name="twitter:site" content="@ApacheArrow" />
+<meta name="twitter:creator" content="@tustvold and alamb" />
+<script type="application/ld+json">
+{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"tustvold and alamb"},"dateModified":"2022-11-07T00:00:00-05:00","datePublished":"2022-11-07T00:00:00-05:00","description":"Introduction Sorting is one of the most fundamental operations in modern databases and other analytic systems, underpinning important operators such as aggregates, joins, window functions, merge, and more. By some estimates, more than half of the execution time in data processin [...]
+<!-- End Jekyll SEO tag -->
+
+
+ <!-- favicons -->
+ <link rel="icon" type="image/png" sizes="16x16" href="/img/favicon-16x16.png" id="light1">
+ <link rel="icon" type="image/png" sizes="32x32" href="/img/favicon-32x32.png" id="light2">
+ <link rel="apple-touch-icon" type="image/png" sizes="180x180" href="/img/apple-touch-icon.png" id="light3">
+ <link rel="apple-touch-icon" type="image/png" sizes="120x120" href="/img/apple-touch-icon-120x120.png" id="light4">
+ <link rel="apple-touch-icon" type="image/png" sizes="76x76" href="/img/apple-touch-icon-76x76.png" id="light5">
+ <link rel="apple-touch-icon" type="image/png" sizes="60x60" href="/img/apple-touch-icon-60x60.png" id="light6">
+ <!-- dark mode favicons -->
+ <link rel="icon" type="image/png" sizes="16x16" href="/img/favicon-16x16-dark.png" id="dark1">
+ <link rel="icon" type="image/png" sizes="32x32" href="/img/favicon-32x32-dark.png" id="dark2">
+ <link rel="apple-touch-icon" type="image/png" sizes="180x180" href="/img/apple-touch-icon-dark.png" id="dark3">
+ <link rel="apple-touch-icon" type="image/png" sizes="120x120" href="/img/apple-touch-icon-120x120-dark.png" id="dark4">
+ <link rel="apple-touch-icon" type="image/png" sizes="76x76" href="/img/apple-touch-icon-76x76-dark.png" id="dark5">
+ <link rel="apple-touch-icon" type="image/png" sizes="60x60" href="/img/apple-touch-icon-60x60-dark.png" id="dark6">
+
+ <script>
+ // Switch to the dark-mode favicons if prefers-color-scheme: dark
+ function onUpdate() {
+ light1 = document.querySelector('link#light1');
+ light2 = document.querySelector('link#light2');
+ light3 = document.querySelector('link#light3');
+ light4 = document.querySelector('link#light4');
+ light5 = document.querySelector('link#light5');
+ light6 = document.querySelector('link#light6');
+
+ dark1 = document.querySelector('link#dark1');
+ dark2 = document.querySelector('link#dark2');
+ dark3 = document.querySelector('link#dark3');
+ dark4 = document.querySelector('link#dark4');
+ dark5 = document.querySelector('link#dark5');
+ dark6 = document.querySelector('link#dark6');
+
+ if (matcher.matches) {
+ light1.remove();
+ light2.remove();
+ light3.remove();
+ light4.remove();
+ light5.remove();
+ light6.remove();
+ document.head.append(dark1);
+ document.head.append(dark2);
+ document.head.append(dark3);
+ document.head.append(dark4);
+ document.head.append(dark5);
+ document.head.append(dark6);
+ } else {
+ dark1.remove();
+ dark2.remove();
+ dark3.remove();
+ dark4.remove();
+ dark5.remove();
+ dark6.remove();
+ document.head.append(light1);
+ document.head.append(light2);
+ document.head.append(light3);
+ document.head.append(light4);
+ document.head.append(light5);
+ document.head.append(light6);
+ }
+ }
+ matcher = window.matchMedia('(prefers-color-scheme: dark)');
+ matcher.addListener(onUpdate);
+ onUpdate();
+ </script>
+
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+ <link href="/css/main.css" rel="stylesheet">
+ <link href="/css/syntax.css" rel="stylesheet">
+ <script src="/javascript/main.js"></script>
+
+ <!-- Matomo -->
+<script>
+ var _paq = window._paq = window._paq || [];
+ /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+ /* We explicitly disable cookie tracking to avoid privacy issues */
+ _paq.push(['disableCookies']);
+ _paq.push(['trackPageView']);
+ _paq.push(['enableLinkTracking']);
+ (function() {
+ var u="https://analytics.apache.org/";
+ _paq.push(['setTrackerUrl', u+'matomo.php']);
+ _paq.push(['setSiteId', '20']);
+ var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+ g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
+ })();
+</script>
+<!-- End Matomo Code -->
+
+
+ </head>
+
+
+<body class="wrap">
+ <header>
+ <nav class="navbar navbar-expand-md navbar-dark bg-dark">
+
+ <a class="navbar-brand no-padding" href="/"><img src="/img/arrow-inverse-300px.png" height="40px"/></a>
+
+ <button class="navbar-toggler ml-auto" type="button" data-toggle="collapse" data-target="#arrow-navbar" aria-controls="arrow-navbar" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse justify-content-end" id="arrow-navbar">
+ <ul class="nav navbar-nav">
+ <li class="nav-item"><a class="nav-link" href="/overview/" role="button" aria-haspopup="true" aria-expanded="false">Overview</a></li>
+ <li class="nav-item"><a class="nav-link" href="/faq/" role="button" aria-haspopup="true" aria-expanded="false">FAQ</a></li>
+ <li class="nav-item"><a class="nav-link" href="/blog" role="button" aria-haspopup="true" aria-expanded="false">Blog</a></li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownGetArrow" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Get Arrow
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownGetArrow">
+ <a class="dropdown-item" href="/install/">Install</a>
+ <a class="dropdown-item" href="/release/">Releases</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow">Source Code</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownDocumentation" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Documentation
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownDocumentation">
+ <a class="dropdown-item" href="/docs">Project Docs</a>
+ <a class="dropdown-item" href="/docs/format/Columnar.html">Format</a>
+ <hr/>
+ <a class="dropdown-item" href="/docs/c_glib">C GLib</a>
+ <a class="dropdown-item" href="/docs/cpp">C++</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow/blob/master/csharp/README.md">C#</a>
+ <a class="dropdown-item" href="https://godoc.org/github.com/apache/arrow/go/arrow">Go</a>
+ <a class="dropdown-item" href="/docs/java">Java</a>
+ <a class="dropdown-item" href="/docs/js">JavaScript</a>
+ <a class="dropdown-item" href="https://arrow.juliadata.org/stable/">Julia</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow/blob/master/matlab/README.md">MATLAB</a>
+ <a class="dropdown-item" href="/docs/python">Python</a>
+ <a class="dropdown-item" href="/docs/r">R</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow/blob/master/ruby/README.md">Ruby</a>
+ <a class="dropdown-item" href="https://docs.rs/crate/arrow/">Rust</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownSubprojects" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Subprojects
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownSubprojects">
+ <a class="dropdown-item" href="/docs/format/Flight.html">Arrow Flight</a>
+ <a class="dropdown-item" href="/docs/format/FlightSql.html">Arrow Flight SQL</a>
+ <a class="dropdown-item" href="/datafusion">DataFusion</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownCommunity" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Community
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownCommunity">
+ <a class="dropdown-item" href="/community/">Communication</a>
+ <a class="dropdown-item" href="/docs/developers/contributing.html">Contributing</a>
+ <a class="dropdown-item" href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a>
+ <a class="dropdown-item" href="/committers/">Governance</a>
+ <a class="dropdown-item" href="/use_cases/">Use Cases</a>
+ <a class="dropdown-item" href="/powered_by/">Powered By</a>
+ <a class="dropdown-item" href="/visual_identity/">Visual Identity</a>
+ <a class="dropdown-item" href="/security/">Security</a>
+ <a class="dropdown-item" href="https://www.apache.org/foundation/policies/conduct.html">Code of Conduct</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownASF" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ ASF Links
+ </a>
+ <div class="dropdown-menu dropdown-menu-right" aria-labelledby="navbarDropdownASF">
+ <a class="dropdown-item" href="https://www.apache.org/">ASF Website</a>
+ <a class="dropdown-item" href="https://www.apache.org/licenses/">License</a>
+ <a class="dropdown-item" href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
+ <a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a>
+ <a class="dropdown-item" href="https://www.apache.org/security/">Security</a>
+ </div>
+ </li>
+ </ul>
+ </div><!-- /.navbar-collapse -->
+ </nav>
+
+ </header>
+
+ <div class="container p-4 pt-5">
+ <div class="col-md-8 mx-auto">
+ <main role="main" class="pb-5">
+
+<h1>
+ Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 1
+</h1>
+<hr class="mt-4 mb-3">
+
+
+
+<p class="mb-4 pb-1">
+ <span class="badge badge-secondary">Published</span>
+ <span class="published mr-3">
+ 07 Nov 2022
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+
+ tustvold and alamb
+
+
+
+</p>
+
+
+ <!--
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>Sorting is one of the most fundamental operations in modern databases and other analytic systems, underpinning important operators such as aggregates, joins, window functions, merge, and more. By some estimates, more than half of the execution time in data processing systems is spent sorting. Optimizing sorts is therefore vital to improving query performance and overall system efficiency.</p>
+
+<p>Sorting is also one of the most well studied topics in computer science. The classic survey paper for databases is <a href="https://dl.acm.org/doi/10.1145/1132960.1132964">Implementing Sorting in Database Systems</a> by Goetz Graefe which provides a thorough academic treatment and is still very applicable today. However, it may not be obvious how to apply the wisdom and advanced techniques described in that paper to modern systems. In addition, the excellent <a href="https://duckdb.or [...]
+
+<p>In this series we explain in detail the new <a href="https://docs.rs/arrow/25.0.0/arrow/row/index.html">row format</a> in the <a href="https://github.com/apache/arrow-rs">Rust implementation</a> of <a href="https://arrow.apache.org/">Apache Arrow</a>, and how we used to make sorting more than <a href="https://github.com/apache/arrow-rs/pull/2929">3x</a> faster than an alternate comparator based approach. The benefits are especially pronounced for strings, dictionary encoded data, and [...]
+
+<h2 id="multicolumn--lexicographical-sort-problem">Multicolumn / Lexicographical Sort Problem</h2>
+
+<p>Most languages have native, optimized operations to sort a single column (array) of data, which are specialized based on the type of data being sorted. The reason that sorting is typically more challenging in analytic systems is that it must:</p>
+
+<ol>
+ <li>They must support multiple columns of data</li>
+ <li>The column types are not knowable at compile time, and thus the compiler can not typically generate optimized code.</li>
+</ol>
+
+<p>Multicolumn sorting is also referred to as lexicographical sorting in some libraries.</p>
+
+<p>For example, given sales data for various customers and their state of residence, a user might want to find the lowest 10 orders for each state.</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Customer | State | Orders
+—--------+-------+-------
+12345 | MA | 10.12
+532432 | MA | 8.44
+12345 | CA | 3.25
+56232 | WA | 6.00
+23442 | WA | 132.50
+7844 | CA | 9.33
+852353 | MA | 1.30
+</code></pre></div></div>
+
+<p>One way to do so is to order the data first by <code class="language-plaintext highlighter-rouge">State</code> and then by <code class="language-plaintext highlighter-rouge">Orders</code>:</p>
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Customer | State | Orders
+—--------+-------+-------
+12345 | CA | 3.25
+7844 | CA | 9.33
+852353 | MA | 1.30
+532432 | MA | 8.44
+12345 | MA | 10.12
+56232 | WA | 6.00
+23442 | WA | 132.50
+</code></pre></div></div>
+
+<p>(Note: While there are specialized ways for computing this particular query other than fully sorting the entire input (e.g. “TopK”), they typically need the same multi-column comparison operation described below. Thus while we will use the simplified example in this series, it applies much more broadly)</p>
+
+<h2 id="basic-implementation">Basic Implementation</h2>
+
+<p>Let us take the example of a basic sort kernel which takes a set of columns as input, and returns a list of indices identifying a sorted order.</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="o">></span> <span class="n">lexsort_to_indices</span><span class="p">([</span>
+ <span class="p">[</span><span class="s">"MA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">]</span>
+ <span class="p">])</span>
+
+<span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
+
+<span class="o">></span> <span class="n">lexsort_to_indices</span><span class="p">([</span>
+ <span class="p">[</span><span class="s">"MA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">],</span>
+ <span class="p">[</span><span class="mf">10.10</span><span class="p">,</span> <span class="mf">8.44</span><span class="p">,</span> <span class="mf">3.25</span><span class="p">,</span> <span class="mf">6.00</span><span class="p">,</span> <span class="mf">132.50</span><span class="p">,</span> <span class="mf">9.33</span><span class="p">,</span> <span class="mf">1.30</span><span class="p">]</span>
+ <span class="p">])</span>
+
+<span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
+</code></pre></div></div>
+
+<p>This function returns a list of indices instead of sorting the columns directly because it:</p>
+<ol>
+ <li>Avoids expensive copying data during the sorting process</li>
+ <li>Allows deferring copying of values until the latest possible moment</li>
+ <li>Can be used to reorder additional columns that weren’t part of the sort key</li>
+</ol>
+
+<p>A straightforward implementation of lexsort_to_indices uses a comparator function,</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code> row
+ index
+ ┌─────┐ ┌─────┐ ┌─────┐ compare(left_index, right_index)
+ 0 │ │ │ │ │ │
+ ┌├─────┤─ ─├─────┤─ ─├─────┤┐ │ │
+ │ │ │ │ │ │ ◀──────────────────┘ │
+ └├─────┤─ ─├─────┤─ ─├─────┤┘ │
+ │ │ │ │ │ │Comparator function compares one │
+ ├─────┤ ├─────┤ ├─────┤ multi-column row with another. │
+ │ │ │ │ │ │ │
+ ├─────┤ ├─────┤ ├─────┤ The data types of the columns │
+ │ │ │ │ │ │ and the sort options are not │
+ └─────┘ └─────┘ └─────┘ known at compile time, only │
+ ... runtime │
+ │
+ ┌┌─────┐─ ─┌─────┐─ ─┌─────┐┐ │
+ │ │ │ │ │ │ ◀────────────────────────────────┘
+ └├─────┤─ ─├─────┤─ ─├─────┤┘
+ │ │ │ │ │ │
+ ├─────┤ ├─────┤ ├─────┤
+ N-1 │ │ │ │ │ │
+ └─────┘ └─────┘ └─────┘
+ Customer State Orders
+ UInt64 Utf8 F64
+</code></pre></div></div>
+
+<p>The comparator function compares each row a column at a time, based on the column types</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌────────────────────────────────┐
+ │ │
+ ▼ │
+ ┌ ─ ─ ─ ┐ ┌ ─ ─ ─ ┐ │
+ │
+ ┌─────┐ │┌─────┐│ │┌─────┐│ │
+left_index │ │ │ │ │ │ │
+ └─────┘ │└─────┘│ │└─────┘│ Step 1: Compare State
+ (UInt64)
+ │ │ │ │
+
+ │ │ │ │
+ ┌─────┐ ┌─────┐ ┌─────┐
+ right_index│ │ ││ ││ ││ ││
+ └─────┘ └─────┘ └─────┘ Step 2: If State values equal
+ │ │ │ │ compare Orders (F64)
+ Customer State Orders │
+ UInt64 │ Utf8 │ │ F64 │ │
+ ─ ─ ─ ─ ─ ─ ─ ─ │
+ ▲ │
+ │ │
+ └───────────────────────┘
+</code></pre></div></div>
+
+<p>Pseudocode for this operation might look something like</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c1"># Takes a list of columns and returns the lexicographically
+# sorted order as a list of indices
+</span><span class="k">def</span> <span class="nf">lexsort_to_indices</span><span class="p">(</span><span class="n">columns</span><span class="p">):</span>
+ <span class="n">comparator</span> <span class="o">=</span> <span class="n">build_comparator</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
+
+ <span class="c1"># Construct a list of integers from 0 to the number of rows
+</span> <span class="c1"># and sort it according to the comparator
+</span> <span class="p">[</span><span class="mf">0.</span><span class="p">.</span><span class="n">columns</span><span class="p">.</span><span class="n">num_rows</span><span class="p">()].</span><span class="n">sort_by</span><span class="p">(</span><span class="n">comparator</span><span class="p">)</span>
+
+<span class="c1"># Build a function that given indexes (left_idx, right_idx)
+# returns the comparison of the sort keys at the left
+# and right indices respectively
+</span><span class="k">def</span> <span class="nf">build_comparator</span><span class="p">(</span><span class="n">columns</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">comparator</span><span class="p">(</span><span class="n">left_idx</span><span class="p">,</span> <span class="n">right_idx</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">:</span>
+ <span class="c1"># call a compare function which performs
+</span> <span class="c1"># dynamic dispatch on type of left and right columns
+</span> <span class="n">ordering</span> <span class="o">=</span> <span class="n">compare</span><span class="p">(</span><span class="n">column</span><span class="p">,</span> <span class="n">left_idx</span><span class="p">,</span><span class="n">right_idx</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">ordering</span> <span class="o">!=</span> <span class="n">Equal</span> <span class="p">{</span>
+ <span class="k">return</span> <span class="n">ordering</span>
+ <span class="p">}</span>
+ <span class="c1"># All values equal
+</span> <span class="n">Equal</span>
+ <span class="c1"># Return comparator function
+</span> <span class="n">comparator</span>
+
+ <span class="c1"># compares the values in a single column at left_idx and right_idx
+</span> <span class="k">def</span> <span class="nf">compare</span><span class="p">(</span><span class="n">column</span><span class="p">,</span> <span class="n">left_idx</span><span class="p">,</span> <span class="n">right_idx</span><span class="p">):</span>
+ <span class="c1"># Choose comparison based on type of column ("dynamic dispatch")
+</span> <span class="k">if</span> <span class="n">column</span><span class="p">.</span><span class="nb">type</span> <span class="o">==</span> <span class="n">Int</span><span class="p">:</span>
+ <span class="nb">cmp</span><span class="p">(</span><span class="n">column</span><span class="p">[</span><span class="n">left_idx</span><span class="p">].</span><span class="n">as_int</span><span class="p">(),</span> <span class="n">column</span><span class="p">[</span><span class="n">right_idx</span><span class="p">].</span><span class="n">as_int</span><span class="p">())</span>
+ <span class="k">elif</span> <span class="n">column</span><span class="p">.</span><span class="nb">type</span> <span class="o">==</span> <span class="n">Float</span><span class="p">:</span>
+ <span class="nb">cmp</span><span class="p">(</span><span class="n">column</span><span class="p">[</span><span class="n">left_idx</span><span class="p">].</span><span class="n">as_float</span><span class="p">(),</span> <span class="n">column</span><span class="p">[</span><span class="n">right_idx</span><span class="p">].</span><span class="n">as_float</span><span class="p">())</span>
+ <span class="p">...</span>
+</code></pre></div></div>
+
+<p>Greater detail is beyond the scope of this post, but in general the more predictable the behavior of a block of code, the better its performance will be. In the case of this pseudocode, there is clear room for improvement:</p>
+
+<ol>
+ <li><code class="language-plaintext highlighter-rouge">comparator</code> performs a large number of unpredictable conditional branches, where the path execution takes depends on the data values</li>
+ <li><code class="language-plaintext highlighter-rouge">comparator</code> and <code class="language-plaintext highlighter-rouge">compare</code> use dynamic dispatch, which not only adds further conditional branches, but also function call overhead</li>
+ <li><code class="language-plaintext highlighter-rouge">comparator</code> performs a large number of reads of memory at unpredictable locations</li>
+</ol>
+
+<p>You can find the complete implementation of multi-column comparator construction in arrow-rs in <a href="https://github.com/apache/arrow-rs/blob/f629a2ebe08033e7b78585d82e98c50a4439e7a2/arrow/src/compute/kernels/sort.rs#L905-L1036">sort.rs</a> and <a href="https://github.com/apache/arrow-rs/blob/f629a2e/arrow/src/array/ord.rs#L178-L313">ord.rs</a>.</p>
+
+<h1 id="normalized-keys--byte-array-comparisons">Normalized Keys / Byte Array Comparisons</h1>
+
+<p>Now imagine we had a way to represent each logical row of data as a sequence of bytes, and that byte-wise comparison of that sequence yielded the same result as comparing the actual column values using the code above. Such a representation would require no switching on column types, and the kernel would become</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">def</span> <span class="nf">lexsort_to_indices</span><span class="p">(</span><span class="n">columns</span><span class="p">):</span>
+ <span class="n">rows</span> <span class="o">=</span> <span class="n">convert_to_rows</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
+ <span class="p">[</span><span class="mf">0.</span><span class="p">.</span><span class="n">columns</span><span class="p">.</span><span class="n">num_rows</span><span class="p">()].</span><span class="n">sort_by</span><span class="p">(</span><span class="k">lambda</span> <span class="n">l</span><span class="p">,</span> <span class="n">r</span><span class="p">:</span> <span class="nb">cmp</span><span class="p">(</span><span class="n">rows</span><span class="p">[</span><span class="n">l</s [...]
+</code></pre></div></div>
+
+<p>While this approach does require converting to/from the byte array representation, it has some major advantages:</p>
+
+<ul>
+ <li>Rows can be compared by comparing bytes in memory, which modern computer hardware excels at with the extremely well optimized <a href="https://www.man7.org/linux/man-pages/man3/memcmp.3.html">memcmp</a></li>
+ <li>Memory accesses are largely predictable</li>
+ <li>There is no dynamic dispatch overhead</li>
+ <li>Extends straightforwardly to more sophisticated sorting strategies such as
+ <ul>
+ <li>Distribution-based sorting techniques such as radix sort</li>
+ <li>Parallel merge sort</li>
+ <li>External sort</li>
+ <li>…</li>
+ </ul>
+ </li>
+</ul>
+
+<p>You can find more information on how to leverage such representation in the “Binary String Comparison” section of the <a href="https://duckdb.org/2021/08/27/external-sorting.html">DuckDB blog post</a> on the topic as well as <a href="https://dl.acm.org/doi/10.1145/1132960.1132964">Graefe’s paper</a>. However, we found it wasn’t immediately obvious how to apply this technique to variable length string or dictionary encoded data, which we will explain in the next post in this series.</p>
+
+<h2 id="next-up-row-format">Next up: Row Format</h2>
+
+<p>This post has introduced the concept and challenges of multi column sorting, and shown why a comparable byte array representation, such as the <a href="https://docs.rs/arrow/25.0.0/arrow/row/index.html">row format</a> introduced to the <a href="https://github.com/apache/arrow-rs">Rust implementation</a> of <a href="https://arrow.apache.org/">Apache Arrow</a>, is such a compelling primitive.</p>
+
+<p>In <a href="/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/">the next post</a> we explain how this encoding works, but if you just want to use it, check out the <a href="https://docs.rs/arrow/latest/arrow/row/index.html">docs</a> for getting started, and report any issues on our <a href="https://github.com/apache/arrow-rs/issues">bugtracker</a>. As always, the <a href="https://github.com/apache/arrow-rs#arrow-rust-community">Arrow community</a> very much looks forward to see [...]
+
+ </main>
+ </div>
+
+ <hr/>
+<footer class="footer">
+ <div class="row">
+ <div class="col-md-9">
+ <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ <p>© 2016-2022 The Apache Software Foundation</p>
+ </div>
+ <div class="col-md-3">
+ <a class="d-sm-none d-md-inline pr-2" href="https://www.apache.org/events/current-event.html">
+ <img src="https://www.apache.org/events/current-event-234x60.png"/>
+ </a>
+ </div>
+ </div>
+</footer>
+
+ </div>
+</body>
+</html>
diff --git a/blog/index.html b/blog/index.html
index 1bdddb366c..1856c58a20 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -235,6 +235,21 @@
+ <p>
+ <h3>
+ <a href="/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 1</a>
+ </h3>
+
+ <p>
+ <span class="blog-list-date">
+ 7 November 2022
+ </span>
+ </p>
+ Introduction Sorting is one of the most fundamental operations in modern databases and other analytic systems, underpinning important operators such as aggregates, joins, window functions, merge, and more. By some estimates, more than half of the execution time in data processing systems is spent sorting. Optimizing sorts is therefore vital...
+ </p>
+
+
+
<p>
<h3>
<a href="/blog/2022/11/01/arrow-flight-sql-jdbc/">Expanding Arrow's Reach with a JDBC Driver for Arrow Flight SQL</a>
@@ -265,6 +280,21 @@
+ <p>
+ <h3>
+ <a href="/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/">Fast and Memory Efficient Multi-Column Sorts in Apache Arrow Rust, Part 2</a>
+ </h3>
+
+ <p>
+ <span class="blog-list-date">
+ 30 October 2022
+ </span>
+ </p>
+ Introduction In Part 1 of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new row format in the Rust implementation of Apache Arrow works and is constructed. Row Format The row format is a variable length...
+ </p>
+
+
+
<p>
<h3>
<a href="/blog/2022/10/28/ballista-0.9.0/">Apache Arrow Ballista 0.9.0 Release</a>
diff --git a/docs/c_glib/index.html b/docs/c_glib/index.html
index 360d6fc765..aaaea6dfd2 100644
--- a/docs/c_glib/index.html
+++ b/docs/c_glib/index.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow GLib (C)" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow GLib (C) Apache Arrow GLib is a wrapper library for Apache Arrow C++. Apache Arrow GLib provides C API. Apache Arrow GLib supports GObject Introspection. It means that you can create language bindings at runtime or compile time automatically. API reference manuals Apache Arrow GLib Apache Parquet GLib Gandiva GLib Plasma [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow GLib (C) Apache Arrow GLib is a wrapper library for Apache Arrow C++. Apache Arrow GLib provides C API. Apache Arrow GLib supports GObject Introspection. It means that you can create language bindings at runtime or compile time automatically. API reference manuals Apache Arrow GLib Apache Parquet GLib Gandiva GLib Plasma [...]
<!-- End Jekyll SEO tag -->
diff --git a/feed.xml b/feed.xml
index a464120b73..c949952791 100644
--- a/feed.xml
+++ b/feed.xml
@@ -1,4 +1,212 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="4.2.0">Jekyll</generator><link href="https://arrow.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://arrow.apache.org/" rel="alternate" type="text/html" /><updated>2022-11-05T05:58:12-04:00</updated><id>https://arrow.apache.org/feed.xml</id><title type="html">Apache Arrow</title><subtitle>Apache Arrow is a cross-language developm [...]
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="4.2.0">Jekyll</generator><link href="https://arrow.apache.org/feed.xml" rel="self" type="application/atom+xml" /><link href="https://arrow.apache.org/" rel="alternate" type="text/html" /><updated>2022-11-07T07:19:02-05:00</updated><id>https://arrow.apache.org/feed.xml</id><title type="html">Apache Arrow</title><subtitle>Apache Arrow is a cross-language developm [...]
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>Sorting is one of the most fundamental operations in modern databases and other analytic systems, underpinning important operators such as aggregates, joins, window functions, merge, and more. By some estimates, more than half of the execution time in data processing systems is spent sorting. Optimizing sorts is therefore vital to improving query performance and overall system efficiency.</p>
+
+<p>Sorting is also one of the most well studied topics in computer science. The classic survey paper for databases is <a href="https://dl.acm.org/doi/10.1145/1132960.1132964">Implementing Sorting in Database Systems</a> by Goetz Graefe which provides a thorough academic treatment and is still very applicable today. However, it may not be obvious how to apply the wisdom and advanced techniques described in that paper to modern systems. In addition, the excellent <a href="https://duckdb.or [...]
+
+<p>In this series we explain in detail the new <a href="https://docs.rs/arrow/25.0.0/arrow/row/index.html">row format</a> in the <a href="https://github.com/apache/arrow-rs">Rust implementation</a> of <a href="https://arrow.apache.org/">Apache Arrow</a>, and how we used to make sorting more than <a href="https://github.com/apache/arrow-rs/pull/2929">3x</a> faster than an alternate comparator based approach. The benefits are especially pronounced for strings, dictionary encoded data, and [...]
+
+<h2 id="multicolumn--lexicographical-sort-problem">Multicolumn / Lexicographical Sort Problem</h2>
+
+<p>Most languages have native, optimized operations to sort a single column (array) of data, which are specialized based on the type of data being sorted. The reason that sorting is typically more challenging in analytic systems is that it must:</p>
+
+<ol>
+ <li>They must support multiple columns of data</li>
+ <li>The column types are not knowable at compile time, and thus the compiler can not typically generate optimized code.</li>
+</ol>
+
+<p>Multicolumn sorting is also referred to as lexicographical sorting in some libraries.</p>
+
+<p>For example, given sales data for various customers and their state of residence, a user might want to find the lowest 10 orders for each state.</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Customer | State | Orders
+—--------+-------+-------
+12345 | MA | 10.12
+532432 | MA | 8.44
+12345 | CA | 3.25
+56232 | WA | 6.00
+23442 | WA | 132.50
+7844 | CA | 9.33
+852353 | MA | 1.30
+</code></pre></div></div>
+
+<p>One way to do so is to order the data first by <code class="language-plaintext highlighter-rouge">State</code> and then by <code class="language-plaintext highlighter-rouge">Orders</code>:</p>
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Customer | State | Orders
+—--------+-------+-------
+12345 | CA | 3.25
+7844 | CA | 9.33
+852353 | MA | 1.30
+532432 | MA | 8.44
+12345 | MA | 10.12
+56232 | WA | 6.00
+23442 | WA | 132.50
+</code></pre></div></div>
+
+<p>(Note: While there are specialized ways for computing this particular query other than fully sorting the entire input (e.g. “TopK”), they typically need the same multi-column comparison operation described below. Thus while we will use the simplified example in this series, it applies much more broadly)</p>
+
+<h2 id="basic-implementation">Basic Implementation</h2>
+
+<p>Let us take the example of a basic sort kernel which takes a set of columns as input, and returns a list of indices identifying a sorted order.</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="o">></span> <span class="n">lexsort_to_indices</span><span class="p">([</span>
+ <span class="p">[</span><span class="s">"MA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">]</span>
+ <span class="p">])</span>
+
+<span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
+
+<span class="o">></span> <span class="n">lexsort_to_indices</span><span class="p">([</span>
+ <span class="p">[</span><span class="s">"MA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"WA"</span><span class="p">,</span> <span class="s">"CA"</span><span class="p">,</span> <span class="s">"MA"</span><span class="p">],</span>
+ <span class="p">[</span><span class="mf">10.10</span><span class="p">,</span> <span class="mf">8.44</span><span class="p">,</span> <span class="mf">3.25</span><span class="p">,</span> <span class="mf">6.00</span><span class="p">,</span> <span class="mf">132.50</span><span class="p">,</span> <span class="mf">9.33</span><span class="p">,</span> <span class="mf">1.30</span><span class="p">]</span>
+ <span class="p">])</span>
+
+<span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
+</code></pre></div></div>
+
+<p>This function returns a list of indices instead of sorting the columns directly because it:</p>
+<ol>
+ <li>Avoids expensive copying data during the sorting process</li>
+ <li>Allows deferring copying of values until the latest possible moment</li>
+ <li>Can be used to reorder additional columns that weren’t part of the sort key</li>
+</ol>
+
+<p>A straightforward implementation of lexsort_to_indices uses a comparator function,</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code> row
+ index
+ ┌─────┐ ┌─────┐ ┌─────┐ compare(left_index, right_index)
+ 0 │ │ │ │ │ │
+ ┌├─────┤─ ─├─────┤─ ─├─────┤┐ │ │
+ │ │ │ │ │ │ ◀──────────────────┘ │
+ └├─────┤─ ─├─────┤─ ─├─────┤┘ │
+ │ │ │ │ │ │Comparator function compares one │
+ ├─────┤ ├─────┤ ├─────┤ multi-column row with another. │
+ │ │ │ │ │ │ │
+ ├─────┤ ├─────┤ ├─────┤ The data types of the columns │
+ │ │ │ │ │ │ and the sort options are not │
+ └─────┘ └─────┘ └─────┘ known at compile time, only │
+ ... runtime │
+ │
+ ┌┌─────┐─ ─┌─────┐─ ─┌─────┐┐ │
+ │ │ │ │ │ │ ◀────────────────────────────────┘
+ └├─────┤─ ─├─────┤─ ─├─────┤┘
+ │ │ │ │ │ │
+ ├─────┤ ├─────┤ ├─────┤
+ N-1 │ │ │ │ │ │
+ └─────┘ └─────┘ └─────┘
+ Customer State Orders
+ UInt64 Utf8 F64
+</code></pre></div></div>
+
+<p>The comparator function compares each row a column at a time, based on the column types</p>
+
+<div class="language-text highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌────────────────────────────────┐
+ │ │
+ ▼ │
+ ┌ ─ ─ ─ ┐ ┌ ─ ─ ─ ┐ │
+ │
+ ┌─────┐ │┌─────┐│ │┌─────┐│ │
+left_index │ │ │ │ │ │ │
+ └─────┘ │└─────┘│ │└─────┘│ Step 1: Compare State
+ (UInt64)
+ │ │ │ │
+
+ │ │ │ │
+ ┌─────┐ ┌─────┐ ┌─────┐
+ right_index│ │ ││ ││ ││ ││
+ └─────┘ └─────┘ └─────┘ Step 2: If State values equal
+ │ │ │ │ compare Orders (F64)
+ Customer State Orders │
+ UInt64 │ Utf8 │ │ F64 │ │
+ ─ ─ ─ ─ ─ ─ ─ ─ │
+ ▲ │
+ │ │
+ └───────────────────────┘
+</code></pre></div></div>
+
+<p>Pseudocode for this operation might look something like</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c1"># Takes a list of columns and returns the lexicographically
+# sorted order as a list of indices
+</span><span class="k">def</span> <span class="nf">lexsort_to_indices</span><span class="p">(</span><span class="n">columns</span><span class="p">):</span>
+ <span class="n">comparator</span> <span class="o">=</span> <span class="n">build_comparator</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
+
+ <span class="c1"># Construct a list of integers from 0 to the number of rows
+</span> <span class="c1"># and sort it according to the comparator
+</span> <span class="p">[</span><span class="mf">0.</span><span class="p">.</span><span class="n">columns</span><span class="p">.</span><span class="n">num_rows</span><span class="p">()].</span><span class="n">sort_by</span><span class="p">(</span><span class="n">comparator</span><span class="p">)</span>
+
+<span class="c1"># Build a function that given indexes (left_idx, right_idx)
+# returns the comparison of the sort keys at the left
+# and right indices respectively
+</span><span class="k">def</span> <span class="nf">build_comparator</span><span class="p">(</span><span class="n">columns</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">comparator</span><span class="p">(</span><span class="n">left_idx</span><span class="p">,</span> <span class="n">right_idx</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">:</span>
+ <span class="c1"># call a compare function which performs
+</span> <span class="c1"># dynamic dispatch on type of left and right columns
+</span> <span class="n">ordering</span> <span class="o">=</span> <span class="n">compare</span><span class="p">(</span><span class="n">column</span><span class="p">,</span> <span class="n">left_idx</span><span class="p">,</span><span class="n">right_idx</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">ordering</span> <span class="o">!=</span> <span class="n">Equal</span> <span class="p">{</span>
+ <span class="k">return</span> <span class="n">ordering</span>
+ <span class="p">}</span>
+ <span class="c1"># All values equal
+</span> <span class="n">Equal</span>
+ <span class="c1"># Return comparator function
+</span> <span class="n">comparator</span>
+
+ <span class="c1"># compares the values in a single column at left_idx and right_idx
+</span> <span class="k">def</span> <span class="nf">compare</span><span class="p">(</span><span class="n">column</span><span class="p">,</span> <span class="n">left_idx</span><span class="p">,</span> <span class="n">right_idx</span><span class="p">):</span>
+ <span class="c1"># Choose comparison based on type of column ("dynamic dispatch")
+</span> <span class="k">if</span> <span class="n">column</span><span class="p">.</span><span class="nb">type</span> <span class="o">==</span> <span class="n">Int</span><span class="p">:</span>
+ <span class="nb">cmp</span><span class="p">(</span><span class="n">column</span><span class="p">[</span><span class="n">left_idx</span><span class="p">].</span><span class="n">as_int</span><span class="p">(),</span> <span class="n">column</span><span class="p">[</span><span class="n">right_idx</span><span class="p">].</span><span class="n">as_int</span><span class="p">())</span>
+ <span class="k">elif</span> <span class="n">column</span><span class="p">.</span><span class="nb">type</span> <span class="o">==</span> <span class="n">Float</span><span class="p">:</span>
+ <span class="nb">cmp</span><span class="p">(</span><span class="n">column</span><span class="p">[</span><span class="n">left_idx</span><span class="p">].</span><span class="n">as_float</span><span class="p">(),</span> <span class="n">column</span><span class="p">[</span><span class="n">right_idx</span><span class="p">].</span><span class="n">as_float</span><span class="p">())</span>
+ <span class="p">...</span>
+</code></pre></div></div>
+
+<p>Greater detail is beyond the scope of this post, but in general the more predictable the behavior of a block of code, the better its performance will be. In the case of this pseudocode, there is clear room for improvement:</p>
+
+<ol>
+ <li><code class="language-plaintext highlighter-rouge">comparator</code> performs a large number of unpredictable conditional branches, where the path execution takes depends on the data values</li>
+ <li><code class="language-plaintext highlighter-rouge">comparator</code> and <code class="language-plaintext highlighter-rouge">compare</code> use dynamic dispatch, which not only adds further conditional branches, but also function call overhead</li>
+ <li><code class="language-plaintext highlighter-rouge">comparator</code> performs a large number of reads of memory at unpredictable locations</li>
+</ol>
+
+<p>You can find the complete implementation of multi-column comparator construction in arrow-rs in <a href="https://github.com/apache/arrow-rs/blob/f629a2ebe08033e7b78585d82e98c50a4439e7a2/arrow/src/compute/kernels/sort.rs#L905-L1036">sort.rs</a> and <a href="https://github.com/apache/arrow-rs/blob/f629a2e/arrow/src/array/ord.rs#L178-L313">ord.rs</a>.</p>
+
+<h1 id="normalized-keys--byte-array-comparisons">Normalized Keys / Byte Array Comparisons</h1>
+
+<p>Now imagine we had a way to represent each logical row of data as a sequence of bytes, and that byte-wise comparison of that sequence yielded the same result as comparing the actual column values using the code above. Such a representation would require no switching on column types, and the kernel would become</p>
+
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">def</span> <span class="nf">lexsort_to_indices</span><span class="p">(</span><span class="n">columns</span><span class="p">):</span>
+ <span class="n">rows</span> <span class="o">=</span> <span class="n">convert_to_rows</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
+ <span class="p">[</span><span class="mf">0.</span><span class="p">.</span><span class="n">columns</span><span class="p">.</span><span class="n">num_rows</span><span class="p">()].</span><span class="n">sort_by</span><span class="p">(</span><span class="k">lambda</span> <span class="n">l</span><span class="p">,</span> <span class="n">r</span><span class="p">:</span> <span class="nb">cmp</span><span class="p">(</span><span class="n">rows</span><span class="p">[</span><span class="n">l</s [...]
+</code></pre></div></div>
+
+<p>While this approach does require converting to/from the byte array representation, it has some major advantages:</p>
+
+<ul>
+ <li>Rows can be compared by comparing bytes in memory, which modern computer hardware excels at with the extremely well optimized <a href="https://www.man7.org/linux/man-pages/man3/memcmp.3.html">memcmp</a></li>
+ <li>Memory accesses are largely predictable</li>
+ <li>There is no dynamic dispatch overhead</li>
+ <li>Extends straightforwardly to more sophisticated sorting strategies such as
+ <ul>
+ <li>Distribution-based sorting techniques such as radix sort</li>
+ <li>Parallel merge sort</li>
+ <li>External sort</li>
+ <li>…</li>
+ </ul>
+ </li>
+</ul>
+
+<p>You can find more information on how to leverage such representation in the “Binary String Comparison” section of the <a href="https://duckdb.org/2021/08/27/external-sorting.html">DuckDB blog post</a> on the topic as well as <a href="https://dl.acm.org/doi/10.1145/1132960.1132964">Graefe’s paper</a>. However, we found it wasn’t immediately obvious how to apply this technique to variable length string or dictionary encoded data, which we will explain in the next post in this series.</p>
+
+<h2 id="next-up-row-format">Next up: Row Format</h2>
+
+<p>This post has introduced the concept and challenges of multi column sorting, and shown why a comparable byte array representation, such as the <a href="https://docs.rs/arrow/25.0.0/arrow/row/index.html">row format</a> introduced to the <a href="https://github.com/apache/arrow-rs">Rust implementation</a> of <a href="https://arrow.apache.org/">Apache Arrow</a>, is such a compelling primitive.</p>
+
+<p>In <a href="/blog/2022/10/30/multi-column-sorts-in-arrow-rust-part-2/">the next post</a> we explain how this encoding works, but if you just want to use it, check out the <a href="https://docs.rs/arrow/latest/arrow/row/index.html">docs</a> for getting started, and report any issues on our <a href="https://github.com/apache/arrow-rs/issues">bugtracker</a>. As always, the <a href="https://github.com/apache/arrow-rs#arrow-rust-community">Arrow community</a> very much looks forward to see [...]
-->
@@ -376,7 +584,221 @@ enable the necessary components explicitly
<p>The Rust projects have moved to separate repositories outside the
main Arrow monorepo. For notes on the latest release of the Rust
-implementation, see the latest <a href="https://github.com/apache/arrow-rs/tags">Arrow Rust changelog</a>.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[The Apache Arrow team is pleased to announce the 10.0.0 release. This covers over 3 months of development work and includes 473 resolved issues from 100 distinct contributors. See the Install Page to learn how to get the libraries for your platform. The release notes below are n [...]
+implementation, see the latest <a href="https://github.com/apache/arrow-rs/tags">Arrow Rust changelog</a>.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[The Apache Arrow team is pleased to announce the 10.0.0 release. This covers over 3 months of development work and includes 473 resolved issues from 100 distinct contributors. See the Install Page to learn how to get the libraries for your platform. The release notes below are n [...]
+
+-->
+
+<h2 id="introduction">Introduction</h2>
+
+<p>In <a href="/blog/2022/11/07/multi-column-sorts-in-arrow-rust-part-1/">Part 1</a> of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new <a href="https://docs.rs/arrow/25.0.0/arrow/row/index.html">row format</a> in the <a href="https://github.com/apache/arrow-rs">Rust implementation</a> of <a href="https://arrow.apache.org/">Apache Arrow</a> works and is constructed.</p>
+
+<h2 id="row-format">Row Format</h2>
+
+<p>The row format is a variable length byte sequence created by concatenating the encoded form of each column. The encoding for each column depends on its datatype (and sort options).</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌─────┐ ┌─────┐ ┌─────┐
+ │ │ │ │ │ │
+ ├─────┤ ┌ ┼─────┼ ─ ┼─────┼ ┐ ┏━━━━━━━━━━━━━┓
+ │ │ │ │ │ │ ─────────────▶┃ ┃
+ ├─────┤ └ ┼─────┼ ─ ┼─────┼ ┘ ┗━━━━━━━━━━━━━┛
+ │ │ │ │ │ │
+ └─────┘ └─────┘ └─────┘
+ ...
+ ┌─────┐ ┌ ┬─────┬ ─ ┬─────┬ ┐ ┏━━━━━━━━┓
+ │ │ │ │ │ │ ─────────────▶┃ ┃
+ └─────┘ └ ┴─────┴ ─ ┴─────┴ ┘ ┗━━━━━━━━┛
+ Customer State Orders
+ UInt64 Utf8 F64
+
+ Input Arrays Row Format
+ (Columns)
+</code></pre></div></div>
+
+<p>The encoding is carefully designed in such a way that escaping is unnecessary: it is never ambiguous as to whether a byte is part of a sentinel (e.g. null) or a value.</p>
+
+<h3 id="unsigned-integers">Unsigned Integers</h3>
+
+<p>To encode a non-null unsigned integer, the byte <code class="language-plaintext highlighter-rouge">0x01</code> is written, followed by the integer’s bytes starting with the most significant, i.e. big endian. A null is encoded as a <code class="language-plaintext highlighter-rouge">0x00</code> byte, followed by the encoded bytes of the integer’s zero value</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 3 │03│00│00│00│ │01│00│00│00│03│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 258 │02│01│00│00│ │01│00│00│01│02│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 23423 │7F│5B│00│00│ │01│00│00│5B│7F│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ NULL │??│??│??│??│ │00│00│00│00│00│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+
+ 32-bit (4 bytes) Row Format
+ Value Little Endian
+</code></pre></div></div>
+
+<h3 id="signed-integers">Signed Integers</h3>
+
+<p>In Rust and most modern computer architectures, signed integers are encoded using <a href="https://en.wikipedia.org/wiki/Two%27s_complement">two’s complement</a>, where a number is negated by flipping all the bits, and adding 1. Therefore, flipping the top-most bit and treating the result as an unsigned integer preserves the order. This unsigned integer can then be encoded using the same encoding for unsigned integers described in the previous section. For example</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ 5 │05│00│00│00│ │05│00│00│80│ │01│80│00│00│05│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┐ ┌──┬──┬──┬──┬──┐
+ -5 │FB│FF│FF│FF│ │FB│FF│FF│7F│ │01│7F│FF│FF│FB│
+ └──┴──┴──┴──┘ └──┴──┴──┴──┘ └──┴──┴──┴──┴──┘
+
+ Value 32-bit (4 bytes) High bit flipped Row Format
+ Little Endian
+</code></pre></div></div>
+
+<h3 id="floating-point">Floating Point</h3>
+
+<p>Floating point values can be ordered according to the <a href="https://en.wikipedia.org/wiki/IEEE_754#Total-ordering_predicate">IEEE 754 totalOrder predicate</a> (implemented in Rust by <a href="https://doc.rust-lang.org/std/primitive.f32.html#method.total_cmp">f32::total_cmp</a>). This ordering interprets the bytes of the floating point value as the correspondingly sized, signed, little-endian integer, flipping all the bits except the sign bit in the case of negatives.</p>
+
+<p>Floating point values are therefore encoded to row format by converting them to the appropriate sized signed integer representation, and then using the same encoding for signed integers described in the previous section.</p>
+
+<h3 id="byte-arrays-including-strings">Byte Arrays (Including Strings)</h3>
+
+<p>Unlike primitive types above, byte arrays are variable length. For short strings, such as <code class="language-plaintext highlighter-rouge">state</code> in our example above, it is possible to pad all values to the length of the longest one with some fixed value such as <code class="language-plaintext highlighter-rouge">0x00</code> and produce a fixed length row. This is the approach described in the DuckDB blog for encoding <code class="language-plaintext highlighter-rouge">c_birth_ [...]
+
+<p>However, often values in string columns differ substantially in length or the maximum length is not known at the start of execution, making it inadvisable and/or impractical to pad the strings to a fixed length. The Rust Arrow row format therefore uses a variable length encoding.</p>
+
+<p>We need an encoding that unambiguously terminates the end of the byte array. This not only permits recovering the original value from the row format, but ensures that bytes of a longer byte array are not compared against bytes from a different column when compared against a row containing a shorter byte array.</p>
+
+<p>A null byte array is encoded as a single <code class="language-plaintext highlighter-rouge">0x00</code> byte. Similarly, an empty byte array is encoded as a single <code class="language-plaintext highlighter-rouge">0x01</code> byte.</p>
+
+<p>To encode a non-null, non-empty array, first a single <code class="language-plaintext highlighter-rouge">0x02</code> byte is written. Then the array is written in 32-byte blocks, with each complete block followed by a <code class="language-plaintext highlighter-rouge">0xFF</code> byte as a continuation token. The final block is padded to 32-bytes with <code class="language-plaintext highlighter-rouge">0x00</code>, and is then followed by the unpadded length of this final block as a s [...]
+
+<p>Note the following example encodings use a block size of 4 bytes, as opposed to 32 bytes for brevity</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌───┬───┬───┬───┬───┬───┐
+ "MEEP" │02 │'M'│'E'│'E'│'P'│04 │
+ └───┴───┴───┴───┴───┴───┘
+
+ ┌───┐
+ "" │01 |
+ └───┘
+
+ NULL ┌───┐
+ │00 │
+ └───┘
+
+"Defenestration" ┌───┬───┬───┬───┬───┬───┐
+ │02 │'D'│'e'│'f'│'e'│FF │
+ └───┼───┼───┼───┼───┼───┤
+ │'n'│'e'│'s'│'t'│FF │
+ ├───┼───┼───┼───┼───┤
+ │'r'│'a'│'t'│'r'│FF │
+ ├───┼───┼───┼───┼───┤
+ │'a'│'t'│'i'│'o'│FF │
+ ├───┼───┼───┼───┼───┤
+ │'n'│00 │00 │00 │17 │
+ └───┴───┴───┴───┴───┘
+</code></pre></div></div>
+
+<p>This approach is loosely inspired by <a href="https://en.wikipedia.org/wiki/Consistent_Overhead_Byte_Stuffing">COBS encoding</a>, and chosen over more traditional <a href="https://en.wikipedia.org/wiki/High-Level_Data_Link_Control#Asynchronous_framing">byte stuffing</a> as it is more amenable to vectorization, in particular hardware with AVX-256 can copy a 32-byte block in a single instruction.</p>
+
+<h3 id="dictionary-arrays">Dictionary Arrays</h3>
+<p>Dictionary Encoded Data (called <a href="https://pandas.pydata.org/docs/user_guide/categorical.html">categorical</a> in pandas) is increasingly important because they can store and process low cardinality data very efficiently.</p>
+
+<p>A simple approach to encoding dictionary arrays would be to encode the logical values directly using the encodings for primitive values described previously. However, this would lose the benefits of dictionary encoding to reduce memory and CPU consumption.</p>
+
+<p>To further complicate matters, the <a href="https://arrow.apache.org/docs/format/Columnar.html#dictionary-encoded-layout">Arrow implementation of Dictionary encoding</a> is quite general, and we can make no assumptions about the contents of the dictionaries. In particular, we cannot assume that the dictionary values are sorted, nor that the same dictionary is used for all arrays within a column</p>
+
+<p>The following example shows how a string column might be encoded in two arrays using two different dictionaries. The dictionary keys <code class="language-plaintext highlighter-rouge">0</code>, <code class="language-plaintext highlighter-rouge">1</code>, and <code class="language-plaintext highlighter-rouge">2</code> in the first batch correspond to different values than the same keys in the second dictionary.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
+ ┌───────────┐ ┌─────┐ │
+│ │"Fabulous" │ │ 0 │
+ ├───────────┤ ├─────┤ │
+│ │ "Bar" │ │ 2 │
+ ├───────────┤ ├─────┤ │ ┌───────────┐
+│ │ "Soup" │ │ 2 │ │"Fabulous" │
+ └───────────┘ ├─────┤ │ ├───────────┤
+│ │ 0 │ │ "Soup" │
+ ├─────┤ │ ├───────────┤
+│ │ 1 │ │ "Soup" │
+ └─────┘ │ ├───────────┤
+│ │"Fabulous" │
+ Values │ ├───────────┤
+│ Dictionary (indexes in │ "Bar" │
+ dictionary) │ ├───────────┤
+│ │ "ZZ" │
+ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ├───────────┤
+┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ "Bar" │
+ │ ├───────────┤
+│ ┌───────────┐ ┌─────┐ │ "ZZ" │
+ │"Fabulous" │ │ 1 │ │ ├───────────┤
+│ ├───────────┤ ├─────┤ │"Fabulous" │
+ │ "ZZ" │ │ 2 │ │ └───────────┘
+│ ├───────────┤ ├─────┤
+ │ "Bar" │ │ 1 │ │
+│ └───────────┘ ├─────┤
+ │ 0 │ │ Logical column
+│ └─────┘ values
+ Values │
+│ Dictionary (indexes in
+ dictionary) │
+ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘
+</code></pre></div></div>
+
+<p>The key observation which allows us to efficiently create a row format for this kind of data is that given a byte array, a new byte array can always be created which comes before or after it in the sort order by adding an additional byte.</p>
+
+<p>Therefore we can incrementally build an order-preserving mapping from dictionary values to variable length byte arrays, without needing to know all possible dictionary values beforehand, instead introducing mappings for new dictionary values as we encounter them.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>┌──────────┐ ┌─────┐
+│ "Bar" │ ───────────────▶│ 01 │
+└──────────┘ └─────┘
+┌──────────┐ ┌─────┬─────┐
+│"Fabulous"│ ───────────────▶│ 01 │ 02 │
+└──────────┘ └─────┴─────┘
+┌──────────┐ ┌─────┐
+│ "Soup" │ ───────────────▶│ 05 │
+└──────────┘ └─────┘
+┌──────────┐ ┌─────┐
+│ "ZZ" │ ───────────────▶│ 07 │
+└──────────┘ └─────┘
+
+ Example Order Preserving Mapping
+</code></pre></div></div>
+
+<p>The details of the data structure used to generate this mapping are beyond the scope of this blog post, but may be the topic of a future post. You can find <a href="https://github.com/apache/arrow-rs/blob/07024f6a16b870fda81cba5779b8817b20386ebf/arrow/src/row/interner.rs">the code here</a>.</p>
+
+<p>The data structure also ensures that no values contain <code class="language-plaintext highlighter-rouge">0x00</code> and therefore we can encode the arrays directly using <code class="language-plaintext highlighter-rouge">0x00</code> as an end-delimiter.</p>
+
+<p>A null value is encoded as a single <code class="language-plaintext highlighter-rouge">0x00</code> byte, and a non-null value encoded as a single <code class="language-plaintext highlighter-rouge">0x01</code> byte, followed by the <code class="language-plaintext highlighter-rouge">0x00</code> terminated byte array determined by the order preserving mapping</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> ┌─────┬─────┬─────┬─────┐
+ "Fabulous" │ 01 │ 03 │ 05 │ 00 │
+ └─────┴─────┴─────┴─────┘
+
+ ┌─────┬─────┬─────┐
+ "ZZ" │ 01 │ 07 │ 00 │
+ └─────┴─────┴─────┘
+
+ ┌─────┐
+ NULL │ 00 │
+ └─────┘
+
+ Input Row Format
+</code></pre></div></div>
+
+<h3 id="sort-options">Sort Options</h3>
+
+<p>One detail we have so far ignored over is how to support ascending and descending sorts (e.g. <code class="language-plaintext highlighter-rouge">ASC</code> or <code class="language-plaintext highlighter-rouge">DESC</code> in SQL). The Arrow Rust row format supports these options by simply inverting the bytes of the encoded representation, except the initial byte used for nullability encoding, on a per column basis.</p>
+
+<p>Similarly, supporting SQL compatible sorting also requires a format that can specify the order of <code class="language-plaintext highlighter-rouge">NULL</code>s (before or after all non <code class="language-plaintext highlighter-rouge">NULL</code> values). The row format supports this option by optionally encoding nulls as <code class="language-plaintext highlighter-rouge">0xFF</code> instead of <code class="language-plaintext highlighter-rouge">0x00</code> on a per column basis.</p>
+
+<h2 id="conclusion">Conclusion</h2>
+
+<p>Hopefully these two articles have given you a flavor of what is possible with a comparable row format and how it works. Feel free to check out the <a href="https://docs.rs/arrow/latest/arrow/row/index.html">docs</a> for instructions on getting started, and report any issues on our <a href="https://github.com/apache/arrow-rs/issues">bugtracker</a>.</p>
+
+<p>Using this format for lexicographic sorting is more than <a href="https://github.com/apache/arrow-rs/pull/2929">3x</a> faster than the comparator based approach, with the benefits especially pronounced for strings, dictionaries and sorts with large numbers of columns.</p>
+
+<p>We have also already used it to more than <a href="https://github.com/apache/arrow-datafusion/pull/3386">double</a> the performance of sort preserving merge in the <a href="https://arrow.apache.org/datafusion/">DataFusion project</a>, and expect similar or greater performance uplift as we apply it to sort, grouping, join, and window function operators as well.</p>
+
+<p>As always, the <a href="https://github.com/apache/arrow-rs#arrow-rust-community">Arrow community</a> very much looks forward to seeing what you build with it!</p>]]></content><author><name>tustvold and alamb</name></author><category term="arrow" /><summary type="html"><![CDATA[Introduction In Part 1 of this post, we described the problem of Multi-Column Sorting and the challenges of implementing it efficiently. This second post explains how the new row format in the Rust implementatio [...]
-->
@@ -1606,314 +2028,4 @@ eOptions</code> is added. (<a href="https://issues.apache.org/jira/browse/ARROW-
<p>The Rust projects have moved to separate repositories outside the
main Arrow monorepo. For notes on the 19.0.0 release of the Rust
-implementation, see the <a href="https://github.com/apache/arrow-rs/blob/19.0.0/CHANGELOG.md">Arrow Rust changelog</a>.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[The Apache Arrow team is pleased to announce the 9.0.0 release. This covers over 3 months of development work and includes 509 resolved issues from 114 distinct contributors. See the Install Page to learn how to get the libraries for your platform. The release notes [...]
-
--->
-
-<h1 id="introduction">Introduction</h1>
-
-<p>We recently celebrated releasing version 16.0.0 of the Rust implementation of <a href="https://arrow.apache.org/">Apache Arrow</a>. While we still get a few comments on “most rust libraries use versions 0.x.0, why are you at 16.0.0?”, our versioning scheme appears to be working well, and permits quick releases of new features and API evolution in a semver compatible way without breaking downstream projects.</p>
-
-<p>This post contains highlights from the last four months (versions
-10.0.0 to 16.0.0) of
-<a href="https://github.com/apache/arrow-rs/arrow">arrow-rs</a> and
-<a href="https://github.com/apache/arrow-rs/parquet">parquet-rs</a> development
-as well as a roadmap of future work. The full list of awesomeness can
-be found in the
-<a href="https://github.com/apache/arrow-rs/blob/master/CHANGELOG.md">CHANGELOG</a>.</p>
-
-<p>As you <a href="https://github.com/apache/arrow-rs/issues/1715">may remember</a>, the arrow and parquet implementations are in the same crate, on the same release schedule, and in this same blog. This is not for technical reasons, but helps to keep the maintenance burden for delivering great Apache software reasonable, and allows easier development of optimized conversion between Arrow <–> Parquet formats.</p>
-
-<h1 id="parquet">Parquet</h1>
-<p>The <a href="https://crates.io/crates/parquet">parquet crate</a> has seen a return to substantial improvements after being relatively dormant for several years. The current major areas of focus are</p>
-
-<ol>
- <li><strong>Performance</strong>: Improving the raw performance for reading and writing mirroring the efforts that went into the C++ version a few years ago.</li>
- <li><strong>API Ease of Use</strong>: Improving the API so it is easy to use efficiently with modern Rust for two preeminent use cases: 1) reading from local disk and 2) reading <code class="language-plaintext highlighter-rouge">async</code>hronously from remote object stores.</li>
-</ol>
-
-<p>Some Major Highlights:</p>
-
-<ul>
- <li><strong>Advanced Metadata Access</strong>: API access to advanced parquet metadata, such as <a href="https://github.com/apache/arrow-rs/pull/1322">PageEncoding</a>, <a href="https://github.com/apache/arrow-rs/pull/1309">BloomFilters</a> and <a href="https://github.com/apache/arrow-rs/pull/1762">PageIndex</a>.</li>
- <li><strong>Improved API Usability</strong>: For example, the <a href="https://github.com/apache/arrow-rs/pull/1719">parquet writer now uses <code class="language-plaintext highlighter-rouge">std:io::Write</code></a> rather than a custom <code class="language-plaintext highlighter-rouge">ParquetWriter</code> trait, making it more interoperable with the rest of the Rust ecosystem and the <a href="https://github.com/apache/arrow-rs/pull/1716">projection API is easier to use with nested t [...]
- <li><strong>Rewritten support for nested types (e.g. struct, lists)</strong> : @tustvold has revamped / rewritten support for <a href="https://github.com/apache/arrow-rs/pull/1682">reading</a> and <a href="https://github.com/apache/arrow-rs/pull/1746">writing</a> structured types, which both improved the support for arbitrary nested schemas, and is 30% faster.</li>
-</ul>
-
-<p>Looking Forward:</p>
-
-<ul>
- <li><strong>Even Faster</strong>: We are actively working to make <a href="https://github.com/apache/arrow-rs/issues/1764">writing even faster</a> and expect to see some major improvements over the next few releases.</li>
- <li><strong>Object Store Integration</strong>: Support for easily and efficiently reading/writing to/from object storage is improving, and we expect it will soon work well out of the box, fetching the minimal bytes, etc… More on this to follow in a separate blog post.</li>
- <li><strong>Parallel Decode</strong>: We intend to transparently support high performance parallel decoding of parquet to arrow arrays, when invoked from a <a href="https://crates.io/crates/rayon">rayon</a> threadpool.</li>
-</ul>
-
-<h1 id="arrow">Arrow</h1>
-
-<p>The Rust arrow implementation has also had substantial improvements, in addition to bug fixes and performance improvements.</p>
-
-<p>Some Major Highlights:</p>
-
-<ul>
- <li><strong>Ecosystem Compatibility</strong>: <a href="https://github.com/viirya">@viriya</a> has put in a massive effort to improve (and prove) compatibility with other Arrow implementations via the Rust IPC integration tests. There have been major improvements for corner cases involving nested structures, nullability, nested dictionaries, etc.</li>
- <li><strong>Safety</strong>: We continue to improve the safety of arrow, and it is not possible to trigger undefined behavior using <code class="language-plaintext highlighter-rouge">safe</code> apis – checkout the <a href="https://github.com/apache/arrow-rs/tree/master/arrow#safety">README</a> and the <a href="https://docs.rs/arrow/16.0.0/arrow/#safety-and-security">module level rustdocs</a> for more details. Among other things, we have added additional validation checking to <a href= [...]
- <li><strong>Performance</strong>: There have been several major performance improvements such as <a href="https://github.com/apache/arrow-rs/issues/1288">much faster filter kernels</a>, thanks to <a href="https://github.com/tustvold">@tustvold</a>.</li>
- <li><strong>Easier to Use APIs</strong>: Several of the APIs are now easier to use (e.g. <a href="https://github.com/apache/arrow-rs/pull/1645">#1645</a> and <a href="https://github.com/apache/arrow-rs/pull/1739">#1739</a> which lowers the barrier to entry of using <code class="language-plaintext highlighter-rouge">arrow-rs</code>, thanks to <a href="https://github.com/HaoYang670">@HaoYang670</a>.</li>
- <li><strong>DataType::Null support</strong>: is much improved, such as <a href="https://github.com/apache/arrow-rs/pull/1572">in the cast kernels</a>, thanks to <a href="https://github.com/WinkerDu">@WinkerDu</a>.</li>
- <li><strong>Improved JSON reader</strong>: The JSON reader is <a href="https://github.com/apache/arrow-rs/pull/1451">now easier to use</a> thanks to <a href="https://github.com/sum12">@sum12</a>.</li>
-</ul>
-
-<p>Looking Forward:</p>
-<ul>
- <li><strong>Make ArrayData Easier to use Safely</strong>: Some amount of <code class="language-plaintext highlighter-rouge">unsafe</code> will likely always be required in arrow (for fast IPC, for example), but we are also working to improve the underlying <code class="language-plaintext highlighter-rouge">ArrayData</code> structure to make it more compatible with the ecosystem (e.g. use <code class="language-plaintext highlighter-rouge">Bytes</code>), support faster to decode from par [...]
- <li><strong>FlightSQL</strong> – we have some <a href="https://github.com/apache/arrow-rs/pulls?q=flightsql">initial support for Flight SQL</a> thank to <a href="https://github.com/wangfenjin">@wangfenjin</a> and <a href="https://github.com/timvw">@timvw</a>, though we would love to see some additional contributors. Such help can include a basic FlightSQL server, and starting work on clients.</li>
-</ul>
-
-<p>Some areas looking for help include:</p>
-
-<ul>
- <li><strong>Decimal 256 support</strong>: <a href="https://github.com/apache/arrow-rs/issues/131">https://github.com/apache/arrow-rs/issues/131</a></li>
- <li><strong>Support for negative Decimal scale</strong>: <a href="https://github.com/apache/arrow-rs/issues/1785">https://github.com/apache/arrow-rs/issues/1785</a></li>
- <li><strong>Support IPC file compression</strong>: <a href="https://github.com/apache/arrow-rs/issues/1709">https://github.com/apache/arrow-rs/issues/1709</a></li>
- <li><strong>Zero-copy bitmap slicing</strong>: <a href="https://github.com/apache/arrow-rs/issues/1802">https://github.com/apache/arrow-rs/issues/1802</a></li>
-</ul>
-
-<h1 id="contributors">Contributors:</h1>
-
-<p>While some open source software can be created mostly by a single contributor, we believe the greatest software with the largest impact and reach is built around its community. Thus, Arrow is part of the <a href="https://www.apache.org/">Apache Software Foundation</a> and our releases both past and present are a result of our amazing community’s effort.</p>
-
-<p>We would like to thank everyone who has contributed to the arrow-rs repository since the <code class="language-plaintext highlighter-rouge">9.0.2</code> release. Keep up the great work and we look forward to continued improvements:</p>
-
-<div class="language-console highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="go">git shortlog -sn 9.0.0..16.0.0
- 47 Liang-Chi Hsieh
- 45 Raphael Taylor-Davies
- 43 Andrew Lamb
- 40 Remzi Yang
- 8 Sergey Glushchenko
- 7 Jörn Horstmann
- 6 Shani Solomon
- 6 dependabot[bot]
- 5 Yang Jiang
- 4 jakevin
- 4 Chao Sun
- 4 Yijie Shen
- 3 kazuhiko kikuchi
- 2 Sumit
- 2 Ismail-Maj
- 2 Kamil Konior
- 2 tfeda
- 2 Matthew Turner
- 1 iyupeng
- 1 ryan-jacobs1
- 1 Alex Qyoun-ae
- 1 tjwilson90
- 1 Andy Grove
- 1 Atef Sawaed
- 1 Daniël Heres
- 1 DuRipeng
- 1 Helgi Kristvin Sigurbjarnarson
- 1 Kun Liu
- 1 Kyle Barron
- 1 Marc Garcia
- 1 Peter C. Jentsch
- 1 Remco Verhoef
- 1 Sven Cattell
- 1 Thomas Peiselt
- 1 Tiphaine Ruy
- 1 Trent Feda
- 1 Wang Fenjin
- 1 Ze'ev Maor
- 1 diana
-</span></code></pre></div></div>
-
-<h1 id="join-the-community">Join the community</h1>
-
-<p>If you are interested in contributing to the Rust subproject in Apache Arrow, you can find a list of open issues
-suitable for beginners <a href="https://github.com/apache/arrow-rs/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a>
-and the full list <a href="https://github.com/apache/arrow-rs/issues">here</a>.</p>
-
-<p>Other ways to get involved include trying out Arrow on some of your data and filing bug reports, and helping to
-improve the documentation.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[Introduction We recently celebrated releasing version 16.0.0 of the Rust implementation of Apache Arrow. While we still get a few comments on “most rust libraries use versions 0.x.0, why are you at 16.0.0?”, our versioning scheme appears to be working well, and permits quick releases of new features and API evolution in a semver compatible way without break [...]
-
--->
-
-<h1 id="introduction">Introduction</h1>
-
-<p><a href="https://arrow.apache.org/datafusion/">DataFusion</a> is an extensible query execution framework, written in Rust, that
-uses Apache Arrow as its in-memory format.</p>
-
-<p>When you want to extend your Rust project with <a href="https://arrow.apache.org/datafusion/user-guide/sql/sql_status.html">SQL support</a>,
-a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth
-checking out.</p>
-
-<p>DataFusion’s SQL, <code class="language-plaintext highlighter-rouge">DataFrame</code>, and manual <code class="language-plaintext highlighter-rouge">PlanBuilder</code> API let users access a sophisticated query optimizer and
-execution engine capable of fast, resource efficient, and parallel execution that takes optimal advantage of
-today’s multicore hardware. Being written in Rust means DataFusion can offer <em>both</em> the safety of a dynamic language and
-the resource efficiency of a compiled language.</p>
-
-<p>The Apache Arrow team is pleased to announce the DataFusion 8.0.0 release (and also the release of version 0.7.0 of
-the Ballista subproject). This covers 3 months of development work and includes 279 commits from the following 49
-distinct contributors.</p>
-
-<!--
-$ git log --pretty=oneline 7.0.0..8.0.0 datafusion datafusion-cli datafusion-examples ballista ballista-cli ballista-examples | wc -l
-279
-
-$ git shortlog -sn 7.0.0..8.0.0 datafusion datafusion-cli datafusion-examples ballista ballista-cli ballista-examples | wc -l
-49
-
-(feynman han, feynman.h, Feynman Han were assumed to be the same person)
--->
-
-<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code> 39 Andy Grove
- 33 Andrew Lamb
- 21 DuRipeng
- 20 Yijie Shen
- 19 Yang Jiang
- 17 Raphael Taylor-Davies
- 11 Dan Harris
- 11 Matthew Turner
- 11 yahoNanJing
- 9 dependabot[bot]
- 8 jakevin
- 6 Kun Liu
- 5 Jiayu Liu
- 4 Daniël Heres
- 4 mingmwang
- 4 xudong.w
- 3 Carol (Nichols || Goulding)
- 3 Dmitry Patsura
- 3 Eduard Karacharov
- 3 Jeremy Dyer
- 3 Kaushik
- 3 Rich
- 3 comphead
- 3 gaojun2048
- 3 Feynman Han
- 2 Jie Han
- 2 Jon Mease
- 2 Tim Van Wassenhove
- 2 Yt
- 2 Zhang Li
- 2 silence-coding
- 1 Alexander Spies
- 1 George Andronchik
- 1 Guillaume Balaine
- 1 Hao Xin
- 1 Jiacai Liu
- 1 Jörn Horstmann
- 1 Liang-Chi Hsieh
- 1 Max Burke
- 1 NaincyKumariKnoldus
- 1 Nga Tran
- 1 Patrick More
- 1 Pierre Zemb
- 1 Remzi Yang
- 1 Sergey Melnychuk
- 1 Stephen Carman
- 1 doki
-</code></pre></div></div>
-
-<p>The following sections highlight some of the changes in this release. Of course, many other bug fixes and
-improvements have been made and we encourage you to check out the
-<a href="https://github.com/apache/arrow-datafusion/blob/8.0.0/datafusion/CHANGELOG.md">changelog</a> for full details.</p>
-
-<h1 id="summary">Summary</h1>
-
-<h2 id="ddl-support">DDL Support</h2>
-
-<p>DDL support has been expanded to include the following commands for creating databases, schemas, and views. This
-allows DataFusion to be used more effectively from the CLI.</p>
-
-<ul>
- <li><code class="language-plaintext highlighter-rouge">CREATE DATABASE</code></li>
- <li><code class="language-plaintext highlighter-rouge">CREATE VIEW</code></li>
- <li><code class="language-plaintext highlighter-rouge">CREATE SCHEMA</code></li>
- <li><code class="language-plaintext highlighter-rouge">CREATE EXTERNAL TABLE</code> now supports JSON files, <code class="language-plaintext highlighter-rouge">IF NOT EXISTS</code>, and partition columns</li>
-</ul>
-
-<h2 id="sql-support">SQL Support</h2>
-
-<p>The SQL query planner now supports a number of new SQL features, including:</p>
-
-<ul>
- <li><em>Subqueries</em>: when used via <code class="language-plaintext highlighter-rouge">IN</code>, <code class="language-plaintext highlighter-rouge">EXISTS</code>, and as scalars</li>
- <li><em>Grouping Sets</em>: <code class="language-plaintext highlighter-rouge">CUBE</code> and <code class="language-plaintext highlighter-rouge">ROLLUP</code> grouping sets.</li>
- <li><em>Aggregate functions</em>: <code class="language-plaintext highlighter-rouge">approx_percentile</code>, <code class="language-plaintext highlighter-rouge">approx_percentile_cont</code>, <code class="language-plaintext highlighter-rouge">approx_percentile_cont_with_weight</code>, <code class="language-plaintext highlighter-rouge">approx_distinct</code>, <code class="language-plaintext highlighter-rouge">approx_median</code> and <code class="language-plaintext highlighter-rouge">a [...]
- <li><em><code class="language-plaintext highlighter-rouge">null</code> literals</em></li>
- <li><em>bitwise operations</em>: for example ‘<code class="language-plaintext highlighter-rouge">|</code>’</li>
-</ul>
-
-<p>There are also many bug fixes and improvements around normalizing identifiers consistently.</p>
-
-<p>We continue our tradition of incrementally releasing support for new
-features as they are developed. Thus, while the physical plan may not yet
-support all new features, it gets more complete each release. These
-changes also make DataFusion an increasingly compelling choice for
-projects looking for a SQL parser and query planner that can produce
-optimized logical plans that can be translated to
-their own execution engine.</p>
-
-<h2 id="query-execution--internals">Query Execution & Internals</h2>
-
-<p>There are several notable improvements and new features in the query execution engine:</p>
-
-<ul>
- <li>The <code class="language-plaintext highlighter-rouge">ExecutionContext</code> has been renamed to <code class="language-plaintext highlighter-rouge">SessionContext</code> and now supports multi-tenancy</li>
- <li>The <code class="language-plaintext highlighter-rouge">ExecutionPlan</code> trait is no longer <code class="language-plaintext highlighter-rouge">async</code></li>
- <li>A new serialization API for serializing plans to bytes (based on protobuf)</li>
-</ul>
-
-<p>In addition, we have added several foundational features to drive even
-more advanced query processing into DataFusion, focusing on running
-arbitrary queries larger than available memory, and pushing the
-envelope for performance of sorting, grouping, and joining even
-further:</p>
-
-<ul>
- <li>Morsel-Driven Scheduler based on <a href="https://15721.courses.cs.cmu.edu/spring2016/papers/p743-leis.pdf">“Morsel-Driven Parallelism: A NUMA-Aware Query
-Evaluation Framework for the Many-Core Age”</a></li>
- <li>Consolidated object store implementation and integration with parquet decoding</li>
- <li>Memory Limited Spilling sort operator</li>
- <li>Memory Limited Sort-Merge join operator</li>
- <li>High performance JIT code generation for tuple comparisons</li>
- <li>Memory efficient Row Format</li>
-</ul>
-
-<h2 id="improved-file-support">Improved file support</h2>
-
-<p>DataFusion now supports JSON, both for reading and writing. There are also new DataFrame methods for writing query
-results to files in CSV, Parquet, and JSON format.</p>
-
-<h2 id="ballista">Ballista</h2>
-
-<p>Ballista continues to mature and now supports a wider range of operators and expressions. There are also improvements
-to the scheduler to support UDFs, and there are some robustness improvements, such as cleaning up work directories
-and persisting session configs to allow schedulers to restart and continue processing in-flight jobs.</p>
-
-<h2 id="upcoming-work">Upcoming Work</h2>
-
-<p>Here are some of the initiatives that the community plans on working on prior to the next release.</p>
-
-<ul>
- <li>There is a <a href="https://docs.google.com/document/d/1jNRbadyStSrV5kifwn0khufAwq6OnzGczG4z8oTQJP4/edit?usp=sharing">proposal to move Ballista to its own top-level arrow-ballista repository</a>
- to decouple DataFusion and Ballista releases and to allow each project to have documentation better targeted at
-its particular audience.</li>
- <li>We plan on increasing the frequency of DataFusion releases, with monthly releases now instead of quarterly. This
-is driven by requests from the increasing number of projects that now depend on DataFusion.</li>
- <li>There is ongoing work to implement new optimizer rules to rewrite queries containing subquery expressions as
-joins, to support a wider range of queries.</li>
- <li>The new scheduler based on morsel-driven execution will continue to evolve in this next release, with work to
-refine IO abstractions to improve performance and integration with the new scheduler.</li>
- <li>Improved performance for Sort, Grouping and Joins</li>
-</ul>
-
-<h1 id="how-to-get-involved">How to Get Involved</h1>
-
-<p>If you are interested in contributing to DataFusion, and learning about state-of-the-art query processing, we would
-love to have you join us on the journey! You can help by trying out DataFusion on some of your own data and projects
-and let us know how it goes or contribute a PR with documentation, tests or code. A list of open issues suitable
-for beginners is <a href="https://github.com/apache/arrow-datafusion/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22">here</a></p>
-
-<p>Check out our new <a href="https://arrow.apache.org/datafusion/community/communication.html">Communication Doc</a> on more
-ways to engage with the community.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[Introduction DataFusion is an extensible query execution framework, written in Rust, that uses Apache Arrow as its in-memory format. When you want to extend your Rust project with SQL support, a DataFrame API, or the ability to read and process Parquet, JSON, Avro or CSV data, DataFusion is definitely worth checking out. DataFusion’s SQL, DataFrame, [...]
\ No newline at end of file
+implementation, see the <a href="https://github.com/apache/arrow-rs/blob/19.0.0/CHANGELOG.md">Arrow Rust changelog</a>.</p>]]></content><author><name>pmc</name></author><category term="release" /><summary type="html"><![CDATA[The Apache Arrow team is pleased to announce the 9.0.0 release. This covers over 3 months of development work and includes 509 resolved issues from 114 distinct contributors. See the Install Page to learn how to get the libraries for your platform. The release notes [...]
\ No newline at end of file
diff --git a/release/0.1.0.html b/release/0.1.0.html
index 5cdcd7d5ab..f4f9614638 100644
--- a/release/0.1.0.html
+++ b/release/0.1.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="0.1.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.1.0 (10 October 2016) Download Source Release: [apache-arrow-0.1.0.tar.gz][6] Verification: [md5][3], [asc][7] Changelog Contributors $ git shortlog -sn d5aa7c46..apache-arrow-0.1.0 49 Wes McKinney 27 Uwe L. Korn 25 Julien Le Dem 13 Micah Kornfield 11 Steven Phillips 6 Jihoon Son 5 Laurent Goujon 5 adeneche 4 Dan Robin [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.1.0 (10 October 2016) Download Source Release: [apache-arrow-0.1.0.tar.gz][6] Verification: [md5][3], [asc][7] Changelog Contributors $ git shortlog -sn d5aa7c46..apache-arrow-0.1.0 49 Wes McKinney 27 Uwe L. Korn 25 Julien Le Dem 13 Micah Kornfield 11 Steven Phillips 6 Jihoon Son 5 Laurent Goujon 5 adeneche 4 Dan Robin [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.10.0.html b/release/0.10.0.html
index 5d08add006..eb3ce48e86 100644
--- a/release/0.10.0.html
+++ b/release/0.10.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.10.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.10.0 (6 August 2018) This is a major release. Download Source Artifacts Binary Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.9.0..apache-arrow-0.10.0 70 Antoine Pitrou 49 Kouhei Sutou 40 Korn, Uwe 37 Wes McKinney 32 Krisztián Szűcs 30 Andy Grove 20 Philipp Moritz 13 Phillip Cloud 11 Bryan Cutler 11 y [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.10.0 (6 August 2018) This is a major release. Download Source Artifacts Binary Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.9.0..apache-arrow-0.10.0 70 Antoine Pitrou 49 Kouhei Sutou 40 Korn, Uwe 37 Wes McKinney 32 Krisztián Szűcs 30 Andy Grove 20 Philipp Moritz 13 Phillip Cloud 11 Bryan Cutler 11 y [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.11.0.html b/release/0.11.0.html
index 7f6c383cf9..e70ae6f94b 100644
--- a/release/0.11.0.html
+++ b/release/0.11.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.11.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.11.0 (8 October 2018) This is a major release. Download Source Artifacts Binary Artifacts Git tag Contributors This includes patches from Apache Parquet that were merged. $ git shortlog -sn apache-arrow-0.10.0..apache-arrow-0.11.0 166 Wes McKinney 59 Uwe L. Korn 57 Deepak Majeti 54 Kouhei Sutou 50 Krisztián Szűcs 48 An [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.11.0 (8 October 2018) This is a major release. Download Source Artifacts Binary Artifacts Git tag Contributors This includes patches from Apache Parquet that were merged. $ git shortlog -sn apache-arrow-0.10.0..apache-arrow-0.11.0 166 Wes McKinney 59 Uwe L. Korn 57 Deepak Majeti 54 Kouhei Sutou 50 Krisztián Szűcs 48 An [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.11.1.html b/release/0.11.1.html
index d215acf866..9e3fd72944 100644
--- a/release/0.11.1.html
+++ b/release/0.11.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.11.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.11.1 (19 October 2018) This is a bugfix release to address a Python packaging issue with zlib that resulted in bug ARROW-3514. Download Source Artifacts Binary Artifacts Git tag Changelog New Features and Improvements ARROW-3353 - [Packaging] Build python 3.7 wheels ARROW-3534 - [Python] Update zlib library in manylinu [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.11.1 (19 October 2018) This is a bugfix release to address a Python packaging issue with zlib that resulted in bug ARROW-3514. Download Source Artifacts Binary Artifacts Git tag Changelog New Features and Improvements ARROW-3353 - [Packaging] Build python 3.7 wheels ARROW-3534 - [Python] Update zlib library in manylinu [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.12.0.html b/release/0.12.0.html
index 129be4bdb7..5c06a4ebfd 100644
--- a/release/0.12.0.html
+++ b/release/0.12.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.12.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.12.0 (20 January 2019) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts Git tag 8ca41384b5324bfd0ef3d3ed3f728e1d10ed73f0 Contributors This release includes 601 commits from 77 distinct contributors. $ git shortlog -sn apache-arrow-0.11.0..apache-arrow-0.12.0 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.12.0 (20 January 2019) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts Git tag 8ca41384b5324bfd0ef3d3ed3f728e1d10ed73f0 Contributors This release includes 601 commits from 77 distinct contributors. $ git shortlog -sn apache-arrow-0.11.0..apache-arrow-0.12.0 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.13.0.html b/release/0.13.0.html
index 9792ec9509..abeed461cb 100644
--- a/release/0.13.0.html
+++ b/release/0.13.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.13.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.13.0 (1 April 2019) This is a major release covering more than 2 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 550 commits from 81 distinct contributors. $ git shortlog -sn apache-arrow-0.12.0..apache-arrow-0.13.0 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.13.0 (1 April 2019) This is a major release covering more than 2 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 550 commits from 81 distinct contributors. $ git shortlog -sn apache-arrow-0.12.0..apache-arrow-0.13.0 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.14.0.html b/release/0.14.0.html
index 787fb474f8..8efe9aa6e0 100644
--- a/release/0.14.0.html
+++ b/release/0.14.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.14.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.14.0 (4 July 2019) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 570 commits from 78 distinct contributors. $ git shortlog -sn apache-arrow-0.13.0..apache-arrow-0.14.0 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.14.0 (4 July 2019) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 570 commits from 78 distinct contributors. $ git shortlog -sn apache-arrow-0.13.0..apache-arrow-0.14.0 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.14.1.html b/release/0.14.1.html
index b5680511c9..22e26cbb63 100644
--- a/release/0.14.1.html
+++ b/release/0.14.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.14.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.14.1 (22 July 2019) This is a bugfix release to address a Python wheel packaging issues and Parquet forward compatibility problems. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 52 commits from 16 distinct contributors. $ git shortlog - [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.14.1 (22 July 2019) This is a bugfix release to address a Python wheel packaging issues and Parquet forward compatibility problems. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 52 commits from 16 distinct contributors. $ git shortlog - [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.15.0.html b/release/0.15.0.html
index ddec9a58d2..0dca5f9121 100644
--- a/release/0.15.0.html
+++ b/release/0.15.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.15.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.15.0 (5 October 2019) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 672 commits from 80 distinct contributors. $ git shortlog -sn apache-arrow-0.14.0..apache-arrow-0.15 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.15.0 (5 October 2019) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 672 commits from 80 distinct contributors. $ git shortlog -sn apache-arrow-0.14.0..apache-arrow-0.15 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.15.1.html b/release/0.15.1.html
index 2e54d222e3..6dbe2d642a 100644
--- a/release/0.15.1.html
+++ b/release/0.15.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.15.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.15.1 (1 November 2019) This is a major release covering more than 1 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 41 commits from 13 distinct contributors. $ git shortlog -sn apache-arrow-0.15.0..apache-arrow-0.15 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.15.1 (1 November 2019) This is a major release covering more than 1 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 41 commits from 13 distinct contributors. $ git shortlog -sn apache-arrow-0.15.0..apache-arrow-0.15 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.16.0.html b/release/0.16.0.html
index c81ffdc1db..ede0ffad4f 100644
--- a/release/0.16.0.html
+++ b/release/0.16.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.16.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.16.0 (7 February 2020) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 709 commits from 99 distinct contributors. $ git shortlog -sn apache-arrow-0.15.1..apache-arrow-0.1 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.16.0 (7 February 2020) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 709 commits from 99 distinct contributors. $ git shortlog -sn apache-arrow-0.15.1..apache-arrow-0.1 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.17.0.html b/release/0.17.0.html
index 73a0a58e45..aad37cac75 100644
--- a/release/0.17.0.html
+++ b/release/0.17.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.17.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.17.0 (20 April 2020) This is a major release covering more than 2 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 569 commits from 79 distinct contributors. $ git shortlog -sn apache-arrow-0.16.0..apache-arrow-0.17. [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.17.0 (20 April 2020) This is a major release covering more than 2 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 569 commits from 79 distinct contributors. $ git shortlog -sn apache-arrow-0.16.0..apache-arrow-0.17. [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.17.1.html b/release/0.17.1.html
index c68767d078..8a1bca76d4 100644
--- a/release/0.17.1.html
+++ b/release/0.17.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.17.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.17.1 (18 May 2020) This is a patch release fixing bugs and regressions listed in the changelog below. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 27 commits from 11 distinct contributors. $ git shortlog -sn apache-arrow-0.17.0..apache [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.17.1 (18 May 2020) This is a patch release fixing bugs and regressions listed in the changelog below. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 27 commits from 11 distinct contributors. $ git shortlog -sn apache-arrow-0.17.0..apache [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.2.0.html b/release/0.2.0.html
index eee39e658c..f2b85851f9 100644
--- a/release/0.2.0.html
+++ b/release/0.2.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="0.2.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.2.0 (18 February 2017) Download Source Artifacts Git tag Changelog Contributors $ git shortlog -sn apache-arrow-0.1.0..apache-arrow-0.2.0 73 Wes McKinney 55 Uwe L. Korn 16 Julien Le Dem 4 Bryan Cutler 4 Nong Li 2 Christopher C. Aycock 2 Jingyuan Wang 2 Kouhei Sutou 2 Laurent Goujon 2 Leif Walsh 1 Emilio Lahr-Vivaz 1 Ho [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.2.0 (18 February 2017) Download Source Artifacts Git tag Changelog Contributors $ git shortlog -sn apache-arrow-0.1.0..apache-arrow-0.2.0 73 Wes McKinney 55 Uwe L. Korn 16 Julien Le Dem 4 Bryan Cutler 4 Nong Li 2 Christopher C. Aycock 2 Jingyuan Wang 2 Kouhei Sutou 2 Laurent Goujon 2 Leif Walsh 1 Emilio Lahr-Vivaz 1 Ho [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.3.0.html b/release/0.3.0.html
index d024f06bc9..69c3a098f0 100644
--- a/release/0.3.0.html
+++ b/release/0.3.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="0.3.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.3.0 (5 May 2017) Read more in the release blog post Download Source Artifacts Git tag d8db8f8 Changelog Contributors $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0 119 Wes McKinney 55 Kouhei Sutou 18 Uwe L. Korn 17 Julien Le Dem 9 Phillip Cloud 6 Bryan Cutler 5 Emilio Lahr-Vivaz 5 Philipp Moritz 4 Jeff Knupp [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.3.0 (5 May 2017) Read more in the release blog post Download Source Artifacts Git tag d8db8f8 Changelog Contributors $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0 119 Wes McKinney 55 Kouhei Sutou 18 Uwe L. Korn 17 Julien Le Dem 9 Phillip Cloud 6 Bryan Cutler 5 Emilio Lahr-Vivaz 5 Philipp Moritz 4 Jeff Knupp [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.4.0.html b/release/0.4.0.html
index a024098a51..20ef3f94f7 100644
--- a/release/0.4.0.html
+++ b/release/0.4.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="0.4.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.4.0 (22 May 2017) Read more in the release blog post Download Source Artifacts Git tag a8f8ba0 Changelog Contributors $ git shortlog -sn apache-arrow-0.3.0..apache-arrow-0.4.0 28 Wes McKinney 18 Kouhei Sutou 9 Uwe L. Korn 3 Brian Hulette 3 Emilio Lahr-Vivaz 3 Philipp Moritz 3 Phillip Cloud 2 Julien Le Dem 1 Bryan Cutle [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.4.0 (22 May 2017) Read more in the release blog post Download Source Artifacts Git tag a8f8ba0 Changelog Contributors $ git shortlog -sn apache-arrow-0.3.0..apache-arrow-0.4.0 28 Wes McKinney 18 Kouhei Sutou 9 Uwe L. Korn 3 Brian Hulette 3 Emilio Lahr-Vivaz 3 Philipp Moritz 3 Phillip Cloud 2 Julien Le Dem 1 Bryan Cutle [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.4.1.html b/release/0.4.1.html
index fffbc3ff8c..4e5a41316c 100644
--- a/release/0.4.1.html
+++ b/release/0.4.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="0.4.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.4.1 (9 June 2017) This is primarily a bug fix release, but also includes some packaging and documentation improvements. Read more in the release blog post. Download Source Artifacts Git tag 46315431 Changelog New Features and Improvements ARROW-1020 - [Format] Add additional language to Schema.fbs to clarify naive vs. [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.4.1 (9 June 2017) This is primarily a bug fix release, but also includes some packaging and documentation improvements. Read more in the release blog post. Download Source Artifacts Git tag 46315431 Changelog New Features and Improvements ARROW-1020 - [Format] Add additional language to Schema.fbs to clarify naive vs. [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.5.0.html b/release/0.5.0.html
index ac0bfd705d..bb426de540 100644
--- a/release/0.5.0.html
+++ b/release/0.5.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.5.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.5.0 (23 July 2017) This is a major release, with expanded features in the supported languages and additional integration test coverage between Java and C++. Read more in the release blog post. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.4.1..apache-arrow-0.5.0 42 Wes McKinney 22 Uwe [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.5.0 (23 July 2017) This is a major release, with expanded features in the supported languages and additional integration test coverage between Java and C++. Read more in the release blog post. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.4.1..apache-arrow-0.5.0 42 Wes McKinney 22 Uwe [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.6.0.html b/release/0.6.0.html
index 29d5863ca8..0a8ee54897 100644
--- a/release/0.6.0.html
+++ b/release/0.6.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.6.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.6.0 (14 August 2017) This is a major release. Read more in the release blog post. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.5.0..apache-arrow-0.6.0 48 Wes McKinney 7 siddharth 5 Matt Darwin 5 Max Risuhin 5 Philipp Moritz 4 Kouhei Sutou 3 Bryan Cutler 2 Emilio Lahr-Vivaz 2 Li Jin 2 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.6.0 (14 August 2017) This is a major release. Read more in the release blog post. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.5.0..apache-arrow-0.6.0 48 Wes McKinney 7 siddharth 5 Matt Darwin 5 Max Risuhin 5 Philipp Moritz 4 Kouhei Sutou 3 Bryan Cutler 2 Emilio Lahr-Vivaz 2 Li Jin 2 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.7.0.html b/release/0.7.0.html
index 2e755a7076..e84edeae7d 100644
--- a/release/0.7.0.html
+++ b/release/0.7.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.7.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.7.0 (17 September 2017) This is a major release. Read more in the release blog post. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.6.0..apache-arrow-0.7.0 58 Wes McKinney 14 Kouhei Sutou 11 Philipp Moritz 7 Phillip Cloud 6 siddharth 5 Uwe L. Korn 2 Bryan Cutler 2 HorimotoYasuhiro 2 La [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.7.0 (17 September 2017) This is a major release. Read more in the release blog post. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.6.0..apache-arrow-0.7.0 58 Wes McKinney 14 Kouhei Sutou 11 Philipp Moritz 7 Phillip Cloud 6 siddharth 5 Uwe L. Korn 2 Bryan Cutler 2 HorimotoYasuhiro 2 La [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.7.1.html b/release/0.7.1.html
index 3cecaa1767..68b9e018a5 100644
--- a/release/0.7.1.html
+++ b/release/0.7.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.7.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.7.1 (1 October 2017) This is a minor bug release. It was motivated by ARROW-1601, but see the complete changelog. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.7.0..apache-arrow-0.7.1 14 Wes McKinney 6 Kouhei Sutou 3 siddharth 2 Paul Taylor 2 Uwe L. Korn 1 Amir Malekpour 1 Bryan Cutle [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.7.1 (1 October 2017) This is a minor bug release. It was motivated by ARROW-1601, but see the complete changelog. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.7.0..apache-arrow-0.7.1 14 Wes McKinney 6 Kouhei Sutou 3 siddharth 2 Paul Taylor 2 Uwe L. Korn 1 Amir Malekpour 1 Bryan Cutle [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.8.0.html b/release/0.8.0.html
index c5c58eaa04..32a095e544 100644
--- a/release/0.8.0.html
+++ b/release/0.8.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.8.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.8.0 (18 December 2017) This is a major release. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.7.1..apache-arrow-0.8.0 90 Wes McKinney 23 Phillip Cloud 21 Kouhei Sutou 13 Licht-T 12 Korn, Uwe 12 Philipp Moritz 12 Uwe L. Korn 10 Bryan Cutler 5 Li Jin 5 Robert Nishihara 4 Paul Taylor 4 s [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.8.0 (18 December 2017) This is a major release. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.7.1..apache-arrow-0.8.0 90 Wes McKinney 23 Phillip Cloud 21 Kouhei Sutou 13 Licht-T 12 Korn, Uwe 12 Philipp Moritz 12 Uwe L. Korn 10 Bryan Cutler 5 Li Jin 5 Robert Nishihara 4 Paul Taylor 4 s [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/0.9.0.html b/release/0.9.0.html
index 2abbed8c5d..8deee75b41 100644
--- a/release/0.9.0.html
+++ b/release/0.9.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 0.9.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 0.9.0 (21 March 2018) This is a major release. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.8.0..apache-arrow-0.9.0 52 Wes McKinney 52 Antoine Pitrou 25 Uwe L. Korn 14 Paul Taylor 13 Kouhei Sutou 13 Phillip Cloud 9 Robert Nishihara 9 Korn, Uwe 9 Jim Crist 8 Brian Hulette 7 Philipp Mori [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 0.9.0 (21 March 2018) This is a major release. Download Source Artifacts Git tag Contributors $ git shortlog -sn apache-arrow-0.8.0..apache-arrow-0.9.0 52 Wes McKinney 52 Antoine Pitrou 25 Uwe L. Korn 14 Paul Taylor 13 Kouhei Sutou 13 Phillip Cloud 9 Robert Nishihara 9 Korn, Uwe 9 Jim Crist 8 Brian Hulette 7 Philipp Mori [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/1.0.0.html b/release/1.0.0.html
index 20cb35516c..ebac61157d 100644
--- a/release/1.0.0.html
+++ b/release/1.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 1.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 1.0.0 (24 July 2020) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 771 commits from 100 distinct contributors. $ git shortlog -sn apache-arrow-0.17.0..apache-arrow-1.0.0 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 1.0.0 (24 July 2020) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 771 commits from 100 distinct contributors. $ git shortlog -sn apache-arrow-0.17.0..apache-arrow-1.0.0 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/1.0.1.html b/release/1.0.1.html
index 393ae96883..0d59e02339 100644
--- a/release/1.0.1.html
+++ b/release/1.0.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 1.0.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 1.0.1 (21 August 2020) This is a patch release addressing bugs in the 1.0.0 release. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 39 commits from 15 distinct contributors. $ git shortlog -sn apache-arrow-1.0.0..apache-arrow-1.0.1 9 Krisz [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 1.0.1 (21 August 2020) This is a patch release addressing bugs in the 1.0.0 release. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 39 commits from 15 distinct contributors. $ git shortlog -sn apache-arrow-1.0.0..apache-arrow-1.0.1 9 Krisz [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/10.0.0.html b/release/10.0.0.html
index 383d12f02b..b9d43fe088 100644
--- a/release/10.0.0.html
+++ b/release/10.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 10.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 10.0.0 (26 October 2022) This is a major release covering more than 2 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 536 commits from 100 distinct contributors. $ git shortlog -s [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 10.0.0 (26 October 2022) This is a major release covering more than 2 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 536 commits from 100 distinct contributors. $ git shortlog -s [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/2.0.0.html b/release/2.0.0.html
index 4f9140dc92..e674035612 100644
--- a/release/2.0.0.html
+++ b/release/2.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 2.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 2.0.0 (19 October 2020) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 511 commits from 81 distinct contributors. $ git shortlog -sn apache-arrow-1.0.0..apache-arrow-2.0.0 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 2.0.0 (19 October 2020) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 511 commits from 81 distinct contributors. $ git shortlog -sn apache-arrow-1.0.0..apache-arrow-2.0.0 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/3.0.0.html b/release/3.0.0.html
index 943bcbee0b..330a237b6b 100644
--- a/release/3.0.0.html
+++ b/release/3.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 3.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 3.0.0 (26 January 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 648 commits from 106 distinct contributors. $ git shortlog -sn apache-arrow-2.0.0..apache-arrow-3.0. [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 3.0.0 (26 January 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 648 commits from 106 distinct contributors. $ git shortlog -sn apache-arrow-2.0.0..apache-arrow-3.0. [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/4.0.0.html b/release/4.0.0.html
index b1d918bc81..6976087a52 100644
--- a/release/4.0.0.html
+++ b/release/4.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 4.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 4.0.0 (26 April 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 719 commits from 114 distinct contributors. $ git shortlog -sn apache-arrow-3.0.0..apache-arrow-4.0.0 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 4.0.0 (26 April 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 719 commits from 114 distinct contributors. $ git shortlog -sn apache-arrow-3.0.0..apache-arrow-4.0.0 [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/4.0.1.html b/release/4.0.1.html
index 256cee1b54..9404999234 100644
--- a/release/4.0.1.html
+++ b/release/4.0.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 4.0.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 4.0.1 (26 May 2021) This is a patch release covering a month of development and addressing small but important bugs in the different implementations. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 21 commits from 13 distinct contributors. [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 4.0.1 (26 May 2021) This is a patch release covering a month of development and addressing small but important bugs in the different implementations. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 21 commits from 13 distinct contributors. [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/5.0.0.html b/release/5.0.0.html
index 5c0b9d0e6f..822a84e77a 100644
--- a/release/5.0.0.html
+++ b/release/5.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 5.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 5.0.0 (29 July 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 684 commits from 99 distinct contributors in 2 Arrow repositories. 77 David Li 43 Krisztián Szűcs 42 An [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 5.0.0 (29 July 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 684 commits from 99 distinct contributors in 2 Arrow repositories. 77 David Li 43 Krisztián Szűcs 42 An [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/6.0.0.html b/release/6.0.0.html
index 49a21dc987..4d6f570373 100644
--- a/release/6.0.0.html
+++ b/release/6.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 6.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 6.0.0 (26 October 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 592 commits from 88 distinct contributors. 58 David Li 56 Antoine Pitrou 46 Neal Richardson 42 Sutou [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 6.0.0 (26 October 2021) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For CentOS For Debian For Python For Ubuntu Git tag Contributors This release includes 592 commits from 88 distinct contributors. 58 David Li 56 Antoine Pitrou 46 Neal Richardson 42 Sutou [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/6.0.1.html b/release/6.0.1.html
index a255c684f4..fb9b551455 100644
--- a/release/6.0.1.html
+++ b/release/6.0.1.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 6.0.1 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 6.0.1 (18 November 2021) This is a patch release covering more than 0 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 34 commits from 16 distinct contributors. $ git shortlog -sn [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 6.0.1 (18 November 2021) This is a patch release covering more than 0 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 34 commits from 16 distinct contributors. $ git shortlog -sn [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/7.0.0.html b/release/7.0.0.html
index f57a006c7a..917312c648 100644
--- a/release/7.0.0.html
+++ b/release/7.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 7.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 7.0.0 (3 February 2022) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 650 commits from 105 distinct contributors. $ git shortlog -sn [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 7.0.0 (3 February 2022) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 650 commits from 105 distinct contributors. $ git shortlog -sn [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/8.0.0.html b/release/8.0.0.html
index d25b82db3f..5ab291756f 100644
--- a/release/8.0.0.html
+++ b/release/8.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 8.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 8.0.0 (6 May 2022) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 636 commits from 127 distinct contributors. $ git shortlog -sn apac [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 8.0.0 (6 May 2022) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 636 commits from 127 distinct contributors. $ git shortlog -sn apac [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/9.0.0.html b/release/9.0.0.html
index 85154fd120..fa2f93a89a 100644
--- a/release/9.0.0.html
+++ b/release/9.0.0.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Apache Arrow 9.0.0 Release" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow 9.0.0 (3 August 2022) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 529 commits from 114 distinct contributors. $ git shortlog -sn a [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow 9.0.0 (3 August 2022) This is a major release covering more than 3 months of development. Download Source Artifacts Binary Artifacts For AlmaLinux For Amazon Linux For CentOS For C# For Debian For Python For Ubuntu Git tag Contributors This release includes 529 commits from 114 distinct contributors. $ git shortlog -sn a [...]
<!-- End Jekyll SEO tag -->
diff --git a/release/index.html b/release/index.html
index 07501e5b23..c5559b0641 100644
--- a/release/index.html
+++ b/release/index.html
@@ -20,13 +20,13 @@
<meta property="og:site_name" content="Apache Arrow" />
<meta property="og:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="og:type" content="article" />
-<meta property="article:published_time" content="2022-11-05T05:58:12-04:00" />
+<meta property="article:published_time" content="2022-11-07T07:19:02-05:00" />
<meta name="twitter:card" content="summary_large_image" />
<meta property="twitter:image" content="https://arrow.apache.org/img/arrow-logo_horizontal_black-txt_white-bg.png" />
<meta property="twitter:title" content="Releases" />
<meta name="twitter:site" content="@ApacheArrow" />
<script type="application/ld+json">
-{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-05T05:58:12-04:00","datePublished":"2022-11-05T05:58:12-04:00","description":"Apache Arrow Releases Navigate to the release page for downloads and the changelog. 10.0.0 (26 October 2022) 9.0.0 (3 August 2022) 8.0.0 (6 May 2022) 7.0.0 (3 February 2022) 6.0.1 (18 November 2021) 6.0.0 (26 October 2021) 5.0.0 (29 July 2021) 4.0.1 (26 May 2021) 4.0.0 (26 April 2021) 3.0.0 (26 January 2021) 2.0.0 (19 October 2020) 1 [...]
+{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2022-11-07T07:19:02-05:00","datePublished":"2022-11-07T07:19:02-05:00","description":"Apache Arrow Releases Navigate to the release page for downloads and the changelog. 10.0.0 (26 October 2022) 9.0.0 (3 August 2022) 8.0.0 (6 May 2022) 7.0.0 (3 February 2022) 6.0.1 (18 November 2021) 6.0.0 (26 October 2021) 5.0.0 (29 July 2021) 4.0.1 (26 May 2021) 4.0.0 (26 April 2021) 3.0.0 (26 January 2021) 2.0.0 (19 October 2020) 1 [...]
<!-- End Jekyll SEO tag -->