You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/09/30 22:08:50 UTC
[07/12] arrow-site git commit: Update to latest website
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/bd12ebcd/blog/index.html
----------------------------------------------------------------------
diff --git a/blog/index.html b/blog/index.html
index 794eb8c..d48efcd 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -5,7 +5,7 @@
<title>Apache Arrow Homepage</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <meta name="generator" content="Jekyll v3.4.3">
+ <meta name="generator" content="Jekyll v3.8.4">
<!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
<link rel="icon" type="image/x-icon" href="/favicon.ico">
@@ -13,10 +13,8 @@
<link href="/css/main.css" rel="stylesheet">
<link href="/css/syntax.css" rel="stylesheet">
- <script src="https://code.jquery.com/jquery-3.2.1.min.js"
- integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
- crossorigin="anonymous"></script>
- <script src="/assets/javascripts/bootstrap.min.js"></script>
+ <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<!-- Global Site Tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script>
@@ -33,96 +31,98 @@
<body class="wrap">
- <div class="container">
- <nav class="navbar navbar-default">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#arrow-navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="/">Apache Arrow™ </a>
- </div>
+ <header>
+ <nav class="navbar navbar-expand-md navbar-dark bg-dark">
+ <a class="navbar-brand" href="/">Apache Arrow™ </a>
+ <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="arrow-navbar">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Project Links<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownProjectLinks" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Project Links
</a>
- <ul class="dropdown-menu">
- <li><a href="/install/">Install</a></li>
- <li><a href="/blog/">Blog</a></li>
- <li><a href="/release/">Releases</a></li>
- <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
- <li><a href="https://github.com/apache/arrow">Source Code</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownProjectLinks">
+ <a class="dropdown-item" href="/install/">Install</a>
+ <a class="dropdown-item" href="/blog/">Blog</a>
+ <a class="dropdown-item" href="/release/">Releases</a>
+ <a class="dropdown-item" href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow">Source Code</a>
+ </div>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Community<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownCommunity" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Community
</a>
- <ul class="dropdown-menu">
- <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
- <li><a href="https://cwiki.apache.org/confluence/display/ARROW">Developer Wiki</a></li>
- <li><a href="/committers/">Committers</a></li>
- <li><a href="/powered_by/">Powered By</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownCommunity">
+ <a class="dropdown-item" href="http://mail-archives.apache.org/mod_mbox/arrow-user/">User Mailing List</a>
+ <a class="dropdown-item" href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Dev Mailing List</a>
+ <a class="dropdown-item" href="https://cwiki.apache.org/confluence/display/ARROW">Developer Wiki</a>
+ <a class="dropdown-item" href="/committers/">Committers</a>
+ <a class="dropdown-item" href="/powered_by/">Powered By</a>
+ </div>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Specification<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownSpecification" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Specification
</a>
- <ul class="dropdown-menu">
- <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
- <li><a href="/docs/metadata.html">Metadata</a></li>
- <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownSpecification">
+ <a class="dropdown-item" href="/docs/memory_layout.html">Memory Layout</a>
+ <a class="dropdown-item" href="/docs/metadata.html">Metadata</a>
+ <a class="dropdown-item" href="/docs/ipc.html">Messaging / IPC</a>
+ </div>
</li>
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Documentation<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownDocumentation" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Documentation
</a>
- <ul class="dropdown-menu">
- <li><a href="/docs/python">Python</a></li>
- <li><a href="/docs/cpp">C++ API</a></li>
- <li><a href="/docs/java">Java API</a></li>
- <li><a href="/docs/c_glib">C GLib API</a></li>
- <li><a href="/docs/js">Javascript API</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownDocumentation">
+ <a class="dropdown-item" href="/docs/python">Python</a>
+ <a class="dropdown-item" href="/docs/cpp">C++ API</a>
+ <a class="dropdown-item" href="/docs/java">Java API</a>
+ <a class="dropdown-item" href="/docs/c_glib">C GLib API</a>
+ <a class="dropdown-item" href="/docs/js">Javascript API</a>
+ </div>
</li>
<!-- <li><a href="/blog">Blog</a></li> -->
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">ASF Links<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownASF" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ ASF Links
</a>
- <ul class="dropdown-menu">
- <li><a href="http://www.apache.org/">ASF Website</a></li>
- <li><a href="http://www.apache.org/licenses/">License</a></li>
- <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
- <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
- <li><a href="http://www.apache.org/security/">Security</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownASF">
+ <a class="dropdown-item" href="http://www.apache.org/">ASF Website</a>
+ <a class="dropdown-item" href="http://www.apache.org/licenses/">License</a>
+ <a class="dropdown-item" href="http://www.apache.org/foundation/sponsorship.html">Donate</a>
+ <a class="dropdown-item" href="http://www.apache.org/foundation/thanks.html">Thanks</a>
+ <a class="dropdown-item" href="http://www.apache.org/security/">Security</a>
+ </div>
</li>
</ul>
- <a href="http://www.apache.org/">
+ <a href="http://www.apache.org/" class="flex-row ml-md-auto">
<img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
</a>
</div><!-- /.navbar-collapse -->
</div>
</nav>
+ </header>
-
+ <div class="container p-lg-4">
+ <main role="main">
+
<h2>Project News and Blog</h2>
<hr>
@@ -130,29 +130,23 @@
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.10.0 Release
<a href="/blog/2018/08/07/0.10.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 07 Aug 2018
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 07 Aug 2018
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -199,29 +193,23 @@ languages.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Faster, scalable memory allocations in Apache Arrow with jemalloc
<a href="/blog/2018/07/20/jemalloc/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 20 Jul 2018
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://github.com/xhochy"><i class="fa fa-user"></i> Uwe Korn (uwe)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 20 Jul 2018
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://github.com/xhochy">Uwe Korn (uwe)</a>
+ </p>
<!--
-->
@@ -268,7 +256,7 @@ to expand our allocations as the data is loaded.</p>
<p>To incrementally build a vector using memory expansion of a factor of 2, we
would use the following C-code with the standard POSIX APIs:</p>
-<div class="language-c highlighter-rouge"><pre class="highlight"><code><span class="kt">size_t</span> <span class="n">size</span> <span class="o">=</span> <span class="mi">128</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">;</span>
+<div class="language-c highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kt">size_t</span> <span class="n">size</span> <span class="o">=</span> <span class="mi">128</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">;</span>
<span class="kt">void</span><span class="o">*</span> <span class="n">ptr</span> <span class="o">=</span> <span class="n">aligned_alloc</span><span class="p">(</span><span class="mi">64</span><span class="p">,</span> <span class="n">size</span><span class="p">);</span>
<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o"><</span> <span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span>
<span class="kt">size_t</span> <span class="n">new_size</span> <span class="o">=</span> <span class="n">size</span> <span class="o">*</span> <span class="mi">2</span><span class="p">;</span>
@@ -279,23 +267,21 @@ would use the following C-code with the standard POSIX APIs:</p>
<span class="n">size</span> <span class="o">=</span> <span class="n">new_size</span><span class="p">;</span>
<span class="p">}</span>
<span class="n">free</span><span class="p">(</span><span class="n">ptr</span><span class="p">);</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>With jemalloc’s special APIs, we are able to omit the explicit call to <code class="highlighter-rouge">memcpy</code>.
In the case where a memory expansion cannot be done in-place, it is still called
by the allocator but not needed on all occasions. This simplifies our user code
to:</p>
-<div class="language-c highlighter-rouge"><pre class="highlight"><code><span class="kt">size_t</span> <span class="n">size</span> <span class="o">=</span> <span class="mi">128</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">;</span>
+<div class="language-c highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kt">size_t</span> <span class="n">size</span> <span class="o">=</span> <span class="mi">128</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">;</span>
<span class="kt">void</span><span class="o">*</span> <span class="n">ptr</span> <span class="o">=</span> <span class="n">mallocx</span><span class="p">(</span><span class="n">size</span><span class="p">,</span> <span class="n">MALLOCX_ALIGN</span><span class="p">(</span><span class="mi">64</span><span class="p">));</span>
<span class="k">for</span> <span class="p">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o"><</span> <span class="mi">10</span><span class="p">;</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span>
<span class="n">size</span> <span class="o">*=</span> <span class="mi">2</span><span class="p">;</span>
<span class="n">ptr</span> <span class="o">=</span> <span class="n">rallocx</span><span class="p">(</span><span class="n">ptr</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">MALLOCX_ALIGN</span><span class="p">(</span><span class="mi">64</span><span class="p">));</span>
<span class="p">}</span>
<span class="n">dallocx</span><span class="p">(</span><span class="n">ptr</span><span class="p">,</span> <span class="n">MALLOCX_ALIGN</span><span class="p">(</span><span class="mi">64</span><span class="p">));</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>To see the real world benefits of using jemalloc, we look at the benchmarks in
Arrow C++. There we have modeled a typical use case of incrementally building up
@@ -307,25 +293,23 @@ of the <code class="highlighter-rouge">builder-benchmark</code> in the Arrow C++
<p>Runtimes without <code class="highlighter-rouge">jemalloc</code>:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>BM_BuildPrimitiveArrayNoNulls/repeats:3 636726 us 804.114MB/s
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>BM_BuildPrimitiveArrayNoNulls/repeats:3 636726 us 804.114MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3 621345 us 824.019MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3 625008 us 819.19MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_mean 627693 us 815.774MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_median 625008 us 819.19MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_stddev 8034 us 10.3829MB/s
-</code></pre>
-</div>
+</code></pre></div></div>
<p>Runtimes with <code class="highlighter-rouge">jemalloc</code>:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>BM_BuildPrimitiveArrayNoNulls/repeats:3 630881 us 811.563MB/s
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>BM_BuildPrimitiveArrayNoNulls/repeats:3 630881 us 811.563MB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3 352891 us 1.41687GB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3 351039 us 1.42434GB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_mean 444937 us 1.21125GB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_median 352891 us 1.41687GB/s
BM_BuildPrimitiveArrayNoNulls/repeats:3_stddev 161035 us 371.335MB/s
-</code></pre>
-</div>
+</code></pre></div></div>
<p>The benchmark was run three times for each configuration to see the performance
differences. The first run in each configuration yielded the same performance but
@@ -344,29 +328,23 @@ reallocation.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
A Native Go Library for Apache Arrow
<a href="/blog/2018/03/22/go-code-donation/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 22 Mar 2018
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://github.com/pmc"><i class="fa fa-user"></i> The Apache Arrow PMC (pmc)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 22 Mar 2018
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://github.com/pmc">The Apache Arrow PMC (pmc)</a>
+ </p>
<!--
-->
@@ -422,29 +400,23 @@ at <a href="https://arrow.apache.org">https://arrow.apache.org</a> and join the
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.9.0 Release
<a href="/blog/2018/03/22/0.9.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 22 Mar 2018
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 22 Mar 2018
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -523,29 +495,23 @@ computational libraries within the project.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.8.0 Release
<a href="/blog/2017/12/18/0.8.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 18 Dec 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 18 Dec 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -708,29 +674,23 @@ implementations and bindings to more languages.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Improvements to Java Vector API in Apache Arrow 0.8.0
- <a href="/blog/2017/12/19/java-vector-improvements/" class="permalink" title="Permalink">∞</a>
- </h2>
+ <a href="/blog/2017/12/18/java-vector-improvements/" class="permalink" title="Permalink">∞</a>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 18 Dec 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (Siddharth Teotia)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 18 Dec 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (Siddharth Teotia)</a>
+ </p>
<!--
-->
@@ -821,29 +781,23 @@ In the new implementation, all vectors in Java are nullable in nature.</li>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Fast Python Serialization with Ray and Apache Arrow
<a href="/blog/2017/10/15/fast-python-serialization-with-ray-and-arrow/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 15 Oct 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (Philipp Moritz, Robert Nishihara)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 15 Oct 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (Philipp Moritz, Robert Nishihara)</a>
+ </p>
<!--
-->
@@ -965,19 +919,17 @@ Python classes and deeply nested objects.</p>
<p>The serialization library can be used directly through pyarrow as follows. More
documentation is available <a href="https://arrow.apache.org/docs/python/ipc.html#arbitrary-object-serialization">here</a>.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="n">x</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="s">'hello'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">5.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">])]</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">x</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="s">'hello'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">5.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">])]</span>
<span class="n">serialized_x</span> <span class="o">=</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">serialize</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">.</span><span class="n">to_buffer</span><span class="p">()</span>
<span class="n">deserialized_x</span> <span class="o">=</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">serialized_x</span><span class="p">)</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>It can be used directly through the Ray API as follows.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="n">x</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="s">'hello'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">5.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">])]</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">x</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="s">'hello'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">5.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">])]</span>
<span class="n">x_id</span> <span class="o">=</span> <span class="n">ray</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="n">deserialized_x</span> <span class="o">=</span> <span class="n">ray</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">x_id</span><span class="p">)</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<h2 id="data-representation">Data Representation</h2>
@@ -996,21 +948,19 @@ serialized object, and the UnionArray contains references to these tensors.</p>
<p>To give a concrete example, consider the following object.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="s">'hello'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">5.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">])]</span>
-</code></pre>
-</div>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">[(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="s">'hello'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">5.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">])]</span>
+</code></pre></div></div>
<p>It would be represented in Arrow with the following structure.</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>UnionArray(type_ids=[tuple, string, int, int, ndarray],
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>UnionArray(type_ids=[tuple, string, int, int, ndarray],
tuples=ListArray(offsets=[0, 2],
UnionArray(type_ids=[int, int],
ints=[1, 2])),
strings=['hello'],
ints=[3, 4],
ndarrays=[<offset of numpy array>])
-</code></pre>
-</div>
+</code></pre></div></div>
<p>Arrow uses Flatbuffers to encode serialized schemas. <strong>Using only the schema, we
can compute the offsets of each value in the data blob without scanning through
@@ -1053,10 +1003,10 @@ Benchmarking <code class="highlighter-rouge">ray.put</code> and <code class="hig
<code class="highlighter-rouge">pyarrow.deserialize</code> gives similar figures. The plots were generated at this
<a href="https://github.com/apache/arrow/tree/894f7400977693b4e0e8f4b9845fd89481f6bf29">commit</a>.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="kn">import</span> <span class="nn">pickle</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="kn">import</span> <span class="nn">pickle</span>
<span class="kn">import</span> <span class="nn">pyarrow</span>
-<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="kn">as</span> <span class="nn">plt</span>
-<span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
+<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="n">plt</span>
+<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="n">np</span>
<span class="kn">import</span> <span class="nn">timeit</span>
@@ -1120,8 +1070,7 @@ Benchmarking <code class="highlighter-rouge">ray.put</code> and <code class="hig
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">test_objects</span><span class="p">)):</span>
<span class="n">plot</span><span class="p">(</span><span class="o">*</span><span class="n">benchmark_object</span><span class="p">(</span><span class="n">test_objects</span><span class="p">[</span><span class="n">i</span><span class="p">]),</span> <span class="n">titles</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">i</span><span class="p">)</span>
-</code></pre>
-</div>
+</code></pre></div></div>
</div>
@@ -1130,29 +1079,23 @@ Benchmarking <code class="highlighter-rouge">ray.put</code> and <code class="hig
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.7.0 Release
<a href="/blog/2017/09/19/0.7.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 19 Sep 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 19 Sep 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -1199,7 +1142,7 @@ library, we implemented a variety of type conversion functions. These functions
are essential in ETL tasks when conforming one table schema to another. These
are similar to the <code class="highlighter-rouge">astype</code> function in NumPy.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="n">In</span> <span class="p">[</span><span class="mi">17</span><span class="p">]:</span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">In</span> <span class="p">[</span><span class="mi">17</span><span class="p">]:</span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="n">pa</span>
<span class="n">In</span> <span class="p">[</span><span class="mi">18</span><span class="p">]:</span> <span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="bp">True</span><span class="p">,</span> <span class="bp">False</span><span class="p">,</span> <span class="bp">None</span><span class="p">,</span> <span class="bp">True</span><span class="p">])</span>
@@ -1222,8 +1165,7 @@ are similar to the <code class="highlighter-rouge">astype</code> function in Num
<span class="n">NA</span><span class="p">,</span>
<span class="mi">1</span>
<span class="p">]</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>Over time these will expand to support as many input-and-output type
combinations with optimized conversions.</p>
@@ -1239,7 +1181,7 @@ columnar memory.</p>
<p>For example, we could write a record batch from CPU memory to GPU device memory
like so (some error checking omitted):</p>
-<div class="language-c++ highlighter-rouge"><pre class="highlight"><code><span class="cp">#include <arrow/api.h>
+<div class="language-c++ highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="cp">#include <arrow/api.h>
#include <arrow/gpu/cuda_api.h>
</span>
<span class="k">using</span> <span class="k">namespace</span> <span class="n">arrow</span><span class="p">;</span>
@@ -1254,20 +1196,18 @@ like so (some error checking omitted):</p>
<span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">gpu</span><span class="o">::</span><span class="n">CudaBuffer</span><span class="o">></span> <span class="n">device_serialized</span><span class="p">;</span>
<span class="n">gpu</span><span class="o">::</span><span class="n">SerializeRecordBatch</span><span class="p">(</span><span class="o">*</span><span class="n">batch</span><span class="p">,</span> <span class="n">context_</span><span class="p">.</span><span class="n">get</span><span class="p">(),</span> <span class="o">&</span><span class="n">device_serialized</span><span class="p">));</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>We can then “read” the GPU record batch, but the returned <code class="highlighter-rouge">arrow::RecordBatch</code>
internally will contain GPU device pointers that you can use for CUDA kernel
calls:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>std::shared_ptr<RecordBatch> device_batch;
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>std::shared_ptr<RecordBatch> device_batch;
gpu::ReadRecordBatch(batch->schema(), device_serialized,
default_memory_pool(), &device_batch));
// Now run some CUDA kernels on device_batch
-</code></pre>
-</div>
+</code></pre></div></div>
<h2 id="decimal-integration-tests">Decimal Integration Tests</h2>
@@ -1321,29 +1261,23 @@ bindings to more languages.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.6.0 Release
<a href="/blog/2017/08/16/0.6.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 16 Aug 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 16 Aug 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -1384,13 +1318,12 @@ be done in a future release.</p>
<p>As an example, the Python data:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>data = [{'a': 1, 'bb': 2, 'cc': 3}, {'dddd': 4}]
-</code></pre>
-</div>
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>data = [{'a': 1, 'bb': 2, 'cc': 3}, {'dddd': 4}]
+</code></pre></div></div>
<p>Could be represented in an Arrow <code class="highlighter-rouge">Map<String, Int32></code> as:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>Map<String, Int32> = List<Struct<keys: String, values: Int32>>
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Map<String, Int32> = List<Struct<keys: String, values: Int32>>
is_valid: [true, true]
offsets: [0, 3, 4]
values: Struct<keys: String, values: Int32>
@@ -1402,8 +1335,7 @@ be done in a future release.</p>
- values: Int32
is_valid: [true, true, true, true]
data: [1, 2, 3, 4]
-</code></pre>
-</div>
+</code></pre></div></div>
<h2 id="python-changes">Python Changes</h2>
<p>Some highlights of Python development outside of bug fixes and general API
@@ -1435,29 +1367,23 @@ Java and C++. Please join the discussion there.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Plasma In-Memory Object Store
<a href="/blog/2017/08/08/plasma-in-memory-object-store/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 08 Aug 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (Philipp Moritz and Robert Nishihara)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 08 Aug 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (Philipp Moritz and Robert Nishihara)</a>
+ </p>
<!--
-->
@@ -1513,7 +1439,7 @@ the client can write to the buffer and construct the object within the allocated
buffer. When the client is done, the client <em>seals</em> the buffer making the object
immutable and making it available to other Plasma clients.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="c"># Create an object.</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Create an object.</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="mi">20</span> <span class="o">*</span> <span class="n">b</span><span class="s">'a'</span><span class="p">)</span>
<span class="n">object_size</span> <span class="o">=</span> <span class="mi">1000</span>
<span class="nb">buffer</span> <span class="o">=</span> <span class="n">memoryview</span><span class="p">(</span><span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">object_id</span><span class="p">,</span> <span class="n">object_size</span><span class="p">))</span>
@@ -1524,18 +1450,16 @@ immutable and making it available to other Plasma clients.</p>
<span class="c"># Seal the object making it immutable and available to other clients.</span>
<span class="n">client</span><span class="o">.</span><span class="n">seal</span><span class="p">(</span><span class="n">object_id</span><span class="p">)</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p><strong>Getting an object:</strong> After an object has been sealed, any client who knows the
object ID can get the object.</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="c"># Get the object from the store. This blocks until the object has been sealed.</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c"># Get the object from the store. This blocks until the object has been sealed.</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">pyarrow</span><span class="o">.</span><span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="mi">20</span> <span class="o">*</span> <span class="n">b</span><span class="s">'a'</span><span class="p">)</span>
<span class="p">[</span><span class="n">buff</span><span class="p">]</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">get</span><span class="p">([</span><span class="n">object_id</span><span class="p">])</span>
<span class="nb">buffer</span> <span class="o">=</span> <span class="n">memoryview</span><span class="p">(</span><span class="n">buff</span><span class="p">)</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>If the object has not been sealed yet, then the call to <code class="highlighter-rouge">client.get</code> will block
until the object has been sealed.</p>
@@ -1588,29 +1512,23 @@ if you are interested in getting involved with the project.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Speeding up PySpark with Apache Arrow
<a href="/blog/2017/07/26/spark-arrow/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 26 Jul 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (BryanCutler)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 26 Jul 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (BryanCutler)</a>
+ </p>
<!--
-->
@@ -1651,7 +1569,7 @@ executors to perform in parallel, drastically reducing the load on the driver.</
needs to be enabled by setting the SQLConf “spark.sql.execution.arrow.enabled” to
“true”. Let’s look at a simple usage example.</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>Welcome to
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
@@ -1691,8 +1609,7 @@ min 0.000000e+00 8.291929e-07
50% 2.097152e+06 4.999210e-01
75% 3.145727e+06 7.498380e-01
max 4.194303e+06 9.999996e-01
-</code></pre>
-</div>
+</code></pre></div></div>
<p>This example was run locally on my laptop using Spark defaults so the times
shown should not be taken precisely. Even though, it is clear there is a huge
@@ -1739,29 +1656,23 @@ helped push this effort forwards.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.5.0 Release
<a href="/blog/2017/07/25/0.5.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 25 Jul 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 25 Jul 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -1783,16 +1694,14 @@ contain integers which correspond to the dictionary.</p>
<p>So we might have:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>data (string): ['foo', 'bar', 'foo', 'bar']
-</code></pre>
-</div>
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>data (string): ['foo', 'bar', 'foo', 'bar']
+</code></pre></div></div>
<p>In dictionary-encoded form, this could be represented as:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>indices (int8): [0, 1, 0, 1]
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>indices (int8): [0, 1, 0, 1]
dictionary (string): ['foo', 'bar']
-</code></pre>
-</div>
+</code></pre></div></div>
<p>In upcoming releases, we plan to complete integration testing for the remaining
data types (including some more complicated types like unions and decimals) on
@@ -1854,29 +1763,23 @@ mailing list</a>. Please join the discussion there.</p>
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Connecting Relational Databases to the Apache Arrow World with turbodbc
<a href="/blog/2017/06/16/turbodbc-arrow/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 16 Jun 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://github.com/MathMagique"><i class="fa fa-user"></i> Michael König (MathMagique)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 16 Jun 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://github.com/MathMagique">Michael König (MathMagique)</a>
+ </p>
<!--
-->
@@ -1903,7 +1806,7 @@ that exceed the capabilities of the row-wise API mandated by Python’s <a href=
Turbodbc has adopted Apache Arrow for this very task with the recently released
version 2.0.0:</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="o">>>></span> <span class="kn">from</span> <span class="nn">turbodbc</span> <span class="kn">import</span> <span class="n">connect</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="o">>>></span> <span class="kn">from</span> <span class="nn">turbodbc</span> <span class="kn">import</span> <span class="n">connect</span>
<span class="o">>>></span> <span class="n">connection</span> <span class="o">=</span> <span class="n">connect</span><span class="p">(</span><span class="n">dsn</span><span class="o">=</span><span class="s">"My columnar database"</span><span class="p">)</span>
<span class="o">>>></span> <span class="n">cursor</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="o">>>></span> <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s">"SELECT some_integers, some_strings FROM my_table"</span><span class="p">)</span>
@@ -1911,8 +1814,7 @@ version 2.0.0:</p>
<span class="n">pyarrow</span><span class="o">.</span><span class="n">Table</span>
<span class="n">some_integers</span><span class="p">:</span> <span class="n">int64</span>
<span class="n">some_strings</span><span class="p">:</span> <span class="n">string</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>With this new addition, the data flow for a result set of a typical SELECT query
is like this:</p>
@@ -1965,29 +1867,23 @@ nitty-gritty details, check out parts <a href="https://tech.blue-yonder.com/maki
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.4.1 Release
<a href="/blog/2017/06/14/0.4.1-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 14 Jun 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 14 Jun 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -2007,10 +1903,9 @@ PyArrow including bundled <a href="http://parquet.apache.org">Apache Parquet</a>
with either conda or pip across the 3 major platforms: Linux, macOS, and
Windows. Use one of:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>pip install pyarrow
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>pip install pyarrow
conda install pyarrow -c conda-forge
-</code></pre>
-</div>
+</code></pre></div></div>
<h3 id="turbodbc-200-with-apache-arrow-support">Turbodbc 2.0.0 with Apache Arrow Support</h3>
@@ -2020,10 +1915,9 @@ team used the PyArrow C++ API introduced in version 0.4.0 to construct
<code class="highlighter-rouge">pyarrow.Table</code> objects inside the <code class="highlighter-rouge">turbodbc</code> library. Learn more in their
<a href="http://turbodbc.readthedocs.io/en/latest/pages/advanced_usage.html#apache-arrow-support">documentation</a> and install with one of:</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>pip install turbodbc
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>pip install turbodbc
conda install turbodbc -c conda-forge
-</code></pre>
-</div>
+</code></pre></div></div>
</div>
@@ -2032,29 +1926,23 @@ conda install turbodbc -c conda-forge
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.4.0 Release
<a href="/blog/2017/05/23/0.4.0-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 23 May 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 23 May 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -2114,8 +2002,8 @@ Python wrapper objects to be constructed from C++ extension code:</p>
<figure class="highlight"><pre><code class="language-cpp" data-lang="cpp"><span class="cp">#include "arrow/python/pyarrow.h"
</span>
<span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">arrow</span><span class="o">::</span><span class="n">py</span><span class="o">::</span><span class="n">import_pyarrow</span><span class="p">())</span> <span class="p">{</span>
- <span class="c1">// Error
-</span><span class="p">}</span>
+ <span class="c1">// Error</span>
+<span class="p">}</span>
<span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">RecordBatch</span><span class="o">></span> <span class="n">cpp_batch</span> <span class="o">=</span> <span class="n">GetData</span><span class="p">(...);</span>
<span class="n">PyObject</span><span class="o">*</span> <span class="n">py_batch</span> <span class="o">=</span> <span class="n">arrow</span><span class="o">::</span><span class="n">py</span><span class="o">::</span><span class="n">wrap_batch</span><span class="p">(</span><span class="n">cpp_batch</span><span class="p">);</span></code></pre></figure>
@@ -2137,29 +2025,23 @@ Linux. We are working on providing binary wheel installers for Windows as well.<
- <div class="container">
- <h2>
+ <div class="blog-post" style="margin-bottom: 4rem">
+ <h1>
Apache Arrow 0.3.0 Release
<a href="/blog/2017/05/08/0.3-release/" class="permalink" title="Permalink">∞</a>
- </h2>
+ </h1>
- <div class="panel">
- <div class="panel-body">
- <div>
- <span class="label label-default">Published</span>
- <span class="published">
- <i class="fa fa-calendar"></i>
- 08 May 2017
- </span>
- </div>
- <div>
- <span class="label label-default">By</span>
- <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
- </div>
- </div>
- </div>
+ <p>
+ <span class="badge badge-secondary">Published</span>
+ <span class="published">
+ 08 May 2017
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+ <a href="http://wesmckinney.com">Wes McKinney (wesm)</a>
+ </p>
<!--
-->
@@ -2248,9 +2130,8 @@ for building from source on Windows: <a href="https://github.com/apache/arrow/bl
<p>For the first time, you can install the Arrow Python library on Windows from
<a href="https://conda-forge.github.io">conda-forge</a>:</p>
-<div class="language-shell highlighter-rouge"><pre class="highlight"><code>conda install pyarrow -c conda-forge
-</code></pre>
-</div>
+<div class="language-shell highlighter-rouge"><div class="highlight"><pre class="highlight"><code>conda <span class="nb">install </span>pyarrow <span class="nt">-c</span> conda-forge
+</code></pre></div></div>
<h3 id="c-glib-bindings-with-support-for-ruby-lua-and-more">C (GLib) Bindings, with support for Ruby, Lua, and more</h3>
@@ -2283,7 +2164,7 @@ managed memory view supporting zero-copy reads and slices. <a href="https://gith
contributed integration between Arrow buffers and the Python buffer protocol
and memoryviews, so now code like this is possible:</p>
-<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="n">In</span> <span class="p">[</span><span class="mi">6</span><span class="p">]:</span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
+<div class="language-python highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="n">In</span> <span class="p">[</span><span class="mi">6</span><span class="p">]:</span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="n">pa</span>
<span class="n">In</span> <span class="p">[</span><span class="mi">7</span><span class="p">]:</span> <span class="n">buf</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">b</span><span class="s">'foobarbaz'</span><span class="p">)</span>
@@ -2295,8 +2176,7 @@ and memoryviews, so now code like this is possible:</p>
<span class="n">In</span> <span class="p">[</span><span class="mi">10</span><span class="p">]:</span> <span class="n">buf</span><span class="o">.</span><span class="n">to_pybytes</span><span class="p">()</span>
<span class="n">Out</span><span class="p">[</span><span class="mi">10</span><span class="p">]:</span> <span class="n">b</span><span class="s">'foobarbaz'</span>
-</code></pre>
-</div>
+</code></pre></div></div>
<p>We have significantly expanded <a href="http://parquet.apache.org"><strong>Apache Parquet</strong></a> support via the C++
Parquet implementation <a href="https://github.com/apache/parquet-cpp">parquet-cpp</a>. This includes support for partitioned
@@ -2347,7 +2227,7 @@ instructions for getting started.</p>
<p>Thanks to all who contributed patches to this release.</p>
-<div class="highlighter-rouge"><pre class="highlight"><code>$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
+<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
119 Wes McKinney
55 Kouhei Sutou
18 Uwe L. Korn
@@ -2371,8 +2251,7 @@ instructions for getting started.</p>
1 Itai Incze
1 Holden Karau
1 Deepak Majeti
-</code></pre>
-</div>
+</code></pre></div></div>
</div>
@@ -2382,11 +2261,13 @@ instructions for getting started.</p>
+ </main>
<hr/>
<footer class="footer">
<p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
<p>© 2017 Apache Software Foundation</p>
+ <script src="/assets/main-8d2a359fd27a888246eb638b36a4e8b68ac65b9f11c48b9fac601fa0c9a7d796.js" integrity="sha256-jSo1n9J6iIJG62OLNqTotorGW58RxIufrGAfoMmn15Y=" crossorigin="anonymous" type="text/javascript"></script>
</footer>
</div>
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/bd12ebcd/committers/index.html
----------------------------------------------------------------------
diff --git a/committers/index.html b/committers/index.html
index 548f554..266071c 100644
--- a/committers/index.html
+++ b/committers/index.html
@@ -5,7 +5,7 @@
<title>Apache Arrow Homepage</title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
- <meta name="generator" content="Jekyll v3.4.3">
+ <meta name="generator" content="Jekyll v3.8.4">
<!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
<link rel="icon" type="image/x-icon" href="/favicon.ico">
@@ -13,10 +13,8 @@
<link href="/css/main.css" rel="stylesheet">
<link href="/css/syntax.css" rel="stylesheet">
- <script src="https://code.jquery.com/jquery-3.2.1.min.js"
- integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
- crossorigin="anonymous"></script>
- <script src="/assets/javascripts/bootstrap.min.js"></script>
+ <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<!-- Global Site Tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script>
@@ -33,96 +31,98 @@
<body class="wrap">
- <div class="container">
- <nav class="navbar navbar-default">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#arrow-navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="/">Apache Arrow™ </a>
- </div>
+ <header>
+ <nav class="navbar navbar-expand-md navbar-dark bg-dark">
+ <a class="navbar-brand" href="/">Apache Arrow™ </a>
+ <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="arrow-navbar">
<ul class="nav navbar-nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Project Links<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownProjectLinks" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Project Links
</a>
- <ul class="dropdown-menu">
- <li><a href="/install/">Install</a></li>
- <li><a href="/blog/">Blog</a></li>
- <li><a href="/release/">Releases</a></li>
- <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
- <li><a href="https://github.com/apache/arrow">Source Code</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownProjectLinks">
+ <a class="dropdown-item" href="/install/">Install</a>
+ <a class="dropdown-item" href="/blog/">Blog</a>
+ <a class="dropdown-item" href="/release/">Releases</a>
+ <a class="dropdown-item" href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a>
+ <a class="dropdown-item" href="https://github.com/apache/arrow">Source Code</a>
+ </div>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Community<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownCommunity" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Community
</a>
- <ul class="dropdown-menu">
- <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
- <li><a href="https://cwiki.apache.org/confluence/display/ARROW">Developer Wiki</a></li>
- <li><a href="/committers/">Committers</a></li>
- <li><a href="/powered_by/">Powered By</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownCommunity">
+ <a class="dropdown-item" href="http://mail-archives.apache.org/mod_mbox/arrow-user/">User Mailing List</a>
+ <a class="dropdown-item" href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Dev Mailing List</a>
+ <a class="dropdown-item" href="https://cwiki.apache.org/confluence/display/ARROW">Developer Wiki</a>
+ <a class="dropdown-item" href="/committers/">Committers</a>
+ <a class="dropdown-item" href="/powered_by/">Powered By</a>
+ </div>
</li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Specification<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownSpecification" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Specification
</a>
- <ul class="dropdown-menu">
- <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
- <li><a href="/docs/metadata.html">Metadata</a></li>
- <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownSpecification">
+ <a class="dropdown-item" href="/docs/memory_layout.html">Memory Layout</a>
+ <a class="dropdown-item" href="/docs/metadata.html">Metadata</a>
+ <a class="dropdown-item" href="/docs/ipc.html">Messaging / IPC</a>
+ </div>
</li>
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">Documentation<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownDocumentation" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Documentation
</a>
- <ul class="dropdown-menu">
- <li><a href="/docs/python">Python</a></li>
- <li><a href="/docs/cpp">C++ API</a></li>
- <li><a href="/docs/java">Java API</a></li>
- <li><a href="/docs/c_glib">C GLib API</a></li>
- <li><a href="/docs/js">Javascript API</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownDocumentation">
+ <a class="dropdown-item" href="/docs/python">Python</a>
+ <a class="dropdown-item" href="/docs/cpp">C++ API</a>
+ <a class="dropdown-item" href="/docs/java">Java API</a>
+ <a class="dropdown-item" href="/docs/c_glib">C GLib API</a>
+ <a class="dropdown-item" href="/docs/js">Javascript API</a>
+ </div>
</li>
<!-- <li><a href="/blog">Blog</a></li> -->
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown"
- role="button" aria-haspopup="true"
- aria-expanded="false">ASF Links<span class="caret"></span>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownASF" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ ASF Links
</a>
- <ul class="dropdown-menu">
- <li><a href="http://www.apache.org/">ASF Website</a></li>
- <li><a href="http://www.apache.org/licenses/">License</a></li>
- <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
- <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
- <li><a href="http://www.apache.org/security/">Security</a></li>
- </ul>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownASF">
+ <a class="dropdown-item" href="http://www.apache.org/">ASF Website</a>
+ <a class="dropdown-item" href="http://www.apache.org/licenses/">License</a>
+ <a class="dropdown-item" href="http://www.apache.org/foundation/sponsorship.html">Donate</a>
+ <a class="dropdown-item" href="http://www.apache.org/foundation/thanks.html">Thanks</a>
+ <a class="dropdown-item" href="http://www.apache.org/security/">Security</a>
+ </div>
</li>
</ul>
- <a href="http://www.apache.org/">
+ <a href="http://www.apache.org/" class="flex-row ml-md-auto">
<img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
</a>
</div><!-- /.navbar-collapse -->
</div>
</nav>
+ </header>
- <div class="container">
+ <div class="container p-lg-4">
+ <main role="main">
+ <div class="container">
<h2>Committers</h2>
<table class="table"><thead>
<tr>
@@ -336,11 +336,13 @@
</body>
</html>
+ </main>
<hr/>
<footer class="footer">
<p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
<p>© 2017 Apache Software Foundation</p>
+ <script src="/assets/main-8d2a359fd27a888246eb638b36a4e8b68ac65b9f11c48b9fac601fa0c9a7d796.js" integrity="sha256-jSo1n9J6iIJG62OLNqTotorGW58RxIufrGAfoMmn15Y=" crossorigin="anonymous" type="text/javascript"></script>
</footer>
</div>