You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/08 12:51:06 UTC
[3/3] arrow-site git commit: Update site for 0.3.0
Update site for 0.3.0
Project: http://git-wip-us.apache.org/repos/asf/arrow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow-site/commit/1e5c92cb
Tree: http://git-wip-us.apache.org/repos/asf/arrow-site/tree/1e5c92cb
Diff: http://git-wip-us.apache.org/repos/asf/arrow-site/diff/1e5c92cb
Branch: refs/heads/asf-site
Commit: 1e5c92cb1365f33a8d0ad8fd6635060268a45a79
Parents: 6360599
Author: Wes McKinney <we...@twosigma.com>
Authored: Mon May 8 08:50:56 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Mon May 8 08:50:56 2017 -0400
----------------------------------------------------------------------
blog/2017/05/08/0.3-release/index.html | 346 +++++++++++++++++++++++++++
blog/index.html | 358 ++++++++++++++++++++++++++++
committers/index.html | 209 ++++++++++++++++
css/main.css | 2 +-
docs/ipc.html | 34 ++-
docs/memory_layout.html | 34 ++-
docs/metadata.html | 34 ++-
feed.xml | 212 +++++++++++++++-
index.html | 154 +++---------
install/index.html | 160 +++++++++++++
10 files changed, 1399 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/1e5c92cb/blog/2017/05/08/0.3-release/index.html
----------------------------------------------------------------------
diff --git a/blog/2017/05/08/0.3-release/index.html b/blog/2017/05/08/0.3-release/index.html
new file mode 100644
index 0000000..dac95f1
--- /dev/null
+++ b/blog/2017/05/08/0.3-release/index.html
@@ -0,0 +1,346 @@
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <title>Apache Arrow 0.3 Release</title>
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="generator" content="Jekyll v3.4.3">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+
+ <title>Apache Arrow Homepage</title>
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+ <link href="/css/main.css" rel="stylesheet">
+ <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+ integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+ crossorigin="anonymous"></script>
+ <script src="/assets/javascripts/bootstrap.min.js"></script>
+ </head>
+
+
+
+<body class="wrap">
+ <div class="container">
+ <nav class="navbar navbar-default">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#arrow-navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/">Apache Arrow™ </a>
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="arrow-navbar">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Project Links<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/install/">Install</a></li>
+ <li><a href="/blog/">Blog</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
+ <li><a href="https://github.com/apache/arrow">Source Code</a></li>
+ <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
+ <li><a href="https://apachearrowslackin.herokuapp.com">Slack Channel</a></li>
+ <li><a href="/committers/">Committers</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Specification<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+ <li><a href="/docs/metadata.html">Metadata</a></li>
+ <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Documentation<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/python">Python</a></li>
+ <li><a href="/docs/cpp">C++ API</a></li>
+ <li><a href="/docs/java">Java API</a></li>
+ <li><a href="/docs/c_glib">C GLib API</a></li>
+ </ul>
+ </li>
+ <!-- <li><a href="/blog">Blog</a></li> -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">ASF Links<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/">ASF Website</a></li>
+ <li><a href="http://www.apache.org/licenses/">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+ <li><a href="http://www.apache.org/security/">Security</a></li>
+ </ul>
+ </li>
+ </ul>
+ <a href="http://www.apache.org/">
+ <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+ </a>
+ </div><!-- /.navbar-collapse -->
+ </div>
+ </nav>
+
+
+ <h2>
+ Apache Arrow 0.3 Release
+ <a href="/blog/2017/05/08/0.3-release/" class="permalink" title="Permalink">∞</a>
+ </h2>
+
+ <div class="panel">
+ <div class="panel-body">
+ <div>
+ <span class="label label-default">Published</span>
+ <span class="published">
+ <i class="fa fa-calendar"></i>
+ 08 May 2017
+ </span>
+ </div>
+ <div>
+ <span class="label label-default">By</span>
+ <a href="http://people.apache.org/~wesm"><i class="fa fa-user"></i> wesm</a>
+ </div>
+ </div>
+ </div>
+
+ <!--
+
+-->
+
+<p>The Apache Arrow team is pleased to announce the 0.3.0 release of the
+project. It is the product of an intense 10 weeks of development since the
+0.2.0 release from this past February. It includes <a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.3.0"><strong>306 resolved JIRAs</strong></a>
+from <a href="https://github.com/apache/arrow/graphs/contributors"><strong>23 contributors</strong></a>.</p>
+
+<p>While we have added many new features to the different Arrow implementations,
+one of the major development focuses in 2017 has been hardening the in-memory
+format, type metadata, and messaging protocol to provide a <strong>stable,
+production-ready foundation</strong> for big data applications. We are excited to be
+collaborating with the <a href="http://spark.apache.org">Apache Spark</a> and <a href="http://www.geomesa.org/">GeoMesa</a> communities on
+utilizing Arrow for high performance IO and in-memory data processing.</p>
+
+<p>See the <a href="http://arrow.apache.org/install">Install Page</a> to learn how to get the libraries for your platform.</p>
+
+<p>We will be publishing more information about the Apache Arrow roadmap as we
+forge ahead with using Arrow to accelerate big data systems.</p>
+
+<p>We are looking for more contributors from within our existing communities and
+from other communities (such as Go, R, or Julia) to get involved in Arrow
+development.</p>
+
+<h3 id="file-and-streaming-format-hardening">File and Streaming Format Hardening</h3>
+
+<p>The 0.2.0 release brought with it the first iterations of the <strong>random access</strong>
+and <strong>streaming</strong> Arrow wire formats. See the <a href="http://arrow.apache.org/docs/ipc.html">IPC specification</a> for
+implementation details and <a href="http://wesmckinney.com/blog/arrow-streaming-columnar/">example blog post</a> with some use cases. These
+provide low-overhead, zero-copy access to Arrow record batch payloads.</p>
+
+<p>In 0.3.0 we have solidified a number of small details with the binary format
+and improved our integration and unit testing particularly in the Java, C++,
+and Python libraries. Using the <a href="http://github.com/google/flatbuffers">Google Flatbuffers</a> project has helped with
+adding new features to our metadata without breaking forward compatibility.</p>
+
+<p>We are not yet ready to make a firm commitment to strong forward compatibility
+(in case we find something needs to change) in the binary format, but we will
+make efforts between major releases to not make unnecessary
+breakages. Contributions to the website and component user and API
+documentation would also be most welcome.</p>
+
+<h3 id="dictionary-encoding-support">Dictionary Encoding Support</h3>
+
+<p><a href="https://github.com/elahrvivaz">Emilio Lahr-Vivaz</a> from the <a href="http://www.geomesa.org/">GeoMesa</a> project contributed Java support
+for dictionary-encoded Arrow vectors. We followed up with C++ and Python
+support (and <code class="highlighter-rouge">pandas.Categorical</code> integration). We have not yet implemented
+full integration tests for dictionaries (for sending this data between C++ and
+Java), but hope to achieve this in the 0.4.0 Arrow release.</p>
+
+<p>This common data representation technique for categorical data allows multiple
+record batches to share a common “dictionary”, with the values in the batches
+being represented as integers referencing the dictionary. This data is called
+“categorical” or “factor” in statistical languages, while in file formats like
+Apache Parquet it is strictly used for data compression.</p>
+
+<h3 id="expanded-date-time-and-fixed-size-types">Expanded Date, Time, and Fixed Size Types</h3>
+
+<p>A notable omission from the 0.2.0 release was complete and integration-tested
+support for the gamut of date and time types that occur in the wild. These are
+needed for <a href="http://parquet.apache.org">Apache Parquet</a> and Apache Spark integration.</p>
+
+<ul>
+ <li><strong>Date</strong>: 32-bit (days unit) and 64-bit (milliseconds unit)</li>
+ <li><strong>Time</strong>: 64-bit integer with unit (second, millisecond, microsecond, nanosecond)</li>
+ <li><strong>Timestamp</strong>: 64-bit integer with unit, with or without timezone</li>
+ <li><strong>Fixed Size Binary</strong>: Primitive values occupying certain number of bytes</li>
+ <li><strong>Fixed Size List</strong>: List values with constant size (no separate offsets vector)</li>
+</ul>
+
+<p>We have additionally added experimental support for exact decimals in C++ using
+<a href="https://github.com/boostorg/multiprecision">Boost.Multiprecision</a>, though we have not yet hardened the Decimal memory
+format between the Java and C++ implementations.</p>
+
+<h3 id="c-and-python-support-on-windows">C++ and Python Support on Windows</h3>
+
+<p>We have made many general improvements to development and packaging for general
+C++ and Python development. 0.3.0 is the first release to bring full C++ and
+Python support for Windows on Visual Studio (MSVC) 2015 and 2017. In addition
+to adding Appveyor continuous integration for MSVC, we have also written guides
+for building from source on Windows: <a href="https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md">C++</a> and <a href="https://github.com/apache/arrow/blob/master/python/doc/source/development.rst">Python</a>.</p>
+
+<p>For the first time, you can install the Arrow Python library on Windows from
+[conda-forge][31]:</p>
+
+<div class="language-shell highlighter-rouge"><pre class="highlight"><code>conda install pyarrow -c conda-forge
+</code></pre>
+</div>
+
+<h3 id="c-glib-bindings-with-support-for-ruby-lua-and-more">C (GLib) Bindings, with support for Ruby, Lua, and more</h3>
+
+<p><a href="http://github.com/kou">Kouhei Sutou</a> is a new Apache Arrow contributor and has contributed GLib C
+bindings (to the C++ libraries) for Linux. Using a C middleware framework
+called <a href="https://wiki.gnome.org/Projects/GObjectIntrospection">GObject Introspection</a>, it is possible to use these bindings
+seamlessly in Ruby, Lua, Go, and <a href="https://wiki.gnome.org/Projects/GObjectIntrospection/Users">other programming languages</a>. We will
+probably need to publish some follow up blogs explaining how these bindings
+work and how to use them.</p>
+
+<h3 id="apache-spark-integration-for-pyspark">Apache Spark Integration for PySpark</h3>
+
+<p>We have been collaborating with the Apache Spark community on <a href="https://issues.apache.org/jira/browse/SPARK-13534">SPARK-13534</a>
+to add support for using Arrow to accelerate <code class="highlighter-rouge">DataFrame.toPandas</code> in
+PySpark. We have observed over <a href="https://github.com/apache/spark/pull/15821#issuecomment-282175163"><strong>40x speedup</strong></a> from the more efficient
+data serialization.</p>
+
+<p>Using Arrow in PySpark opens the door to many other performance optimizations,
+particularly around UDF evaluation (e.g. <code class="highlighter-rouge">map</code> and <code class="highlighter-rouge">filter</code> operations with
+Python lambda functions).</p>
+
+<h3 id="new-python-feature-memory-views-feather-apache-parquet-support">New Python Feature: Memory Views, Feather, Apache Parquet support</h3>
+
+<p>Arrow’s Python library <code class="highlighter-rouge">pyarrow</code> is a Cython binding for the <code class="highlighter-rouge">libarrow</code> and
+<code class="highlighter-rouge">libarrow_python</code> C++ libraries, which handle inteoperability with NumPy,
+<a href="http://pandas.pydata.org">pandas</a>, and the Python standard library.</p>
+
+<p>At the heart of Arrow’s C++ libraries is the <code class="highlighter-rouge">arrow::Buffer</code> object, which is a
+managed memory view supporting zero-copy reads and slices. <a href="https://github.com/JeffKnupp">Jeff Knupp</a>
+contributed integration between Arrow buffers and the Python buffer protocol
+and memoryviews, so now code like this is possible:</p>
+
+<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="n">In</span> <span class="p">[</span><span class="mi">6</span><span class="p">]:</span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">7</span><span class="p">]:</span> <span class="n">buf</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">b</span><span class="s">'foobarbaz'</span><span class="p">)</span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span class="n">buf</span>
+<span class="n">Out</span><span class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span class="o"><</span><span class="n">pyarrow</span><span class="o">.</span><span class="n">_io</span><span class="o">.</span><span class="n">Buffer</span> <span class="n">at</span> <span class="mh">0x7f6c0a84b538</span><span class="o">></span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">9</span><span class="p">]:</span> <span class="n">memoryview</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
+<span class="n">Out</span><span class="p">[</span><span class="mi">9</span><span class="p">]:</span> <span class="o"><</span><span class="n">memory</span> <span class="n">at</span> <span class="mh">0x7f6c0a8c5e88</span><span class="o">></span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">10</span><span class="p">]:</span> <span class="n">buf</span><span class="o">.</span><span class="n">to_pybytes</span><span class="p">()</span>
+<span class="n">Out</span><span class="p">[</span><span class="mi">10</span><span class="p">]:</span> <span class="n">b</span><span class="s">'foobarbaz'</span>
+</code></pre>
+</div>
+
+<p>We have significantly expanded <a href="http://parquet.apache.org"><strong>Apache Parquet</strong></a> support via the C++
+Parquet implementation <a href="https://github.com/apache/parquet-cpp">parquet-cpp</a>. This includes support for partitioned
+datasets on disk or in HDFS. We added initial Arrow-powered Parquet support <a href="https://github.com/dask/dask/commit/68f9e417924a985c1f2e2a587126833c70a2e9f4">in
+the Dask project</a>, and look forward to more collaborations with the Dask
+developers on distributed processing of pandas data.</p>
+
+<p>With Arrow’s support for pandas maturing, we were able to merge in the
+<a href="https://github.com/wesm/feather"><strong>Feather format</strong></a> implementation, which is essentially a special case of
+the Arrow random access format. We’ll be continuing Feather development within
+the Arrow codebase. For example, Feather can now read and write with Python
+file objects using Arrow’s Python binding layer.</p>
+
+<p>We also implemented more robust support for pandas-specific data types, like
+<code class="highlighter-rouge">DatetimeTZ</code> and <code class="highlighter-rouge">Categorical</code>.</p>
+
+<h3 id="support-for-tensors-and-beyond-in-c-library">Support for Tensors and beyond in C++ Library</h3>
+
+<p>There has been increased interest in using Apache Arrow as a tool for zero-copy
+shared memory management for machine learning applications. A flagship example
+is the <a href="https://github.com/ray-project/ray">Ray project</a> from the UC Berkeley <a href="https://rise.cs.berkeley.edu/">RISELab</a>.</p>
+
+<p>Machine learning deals in additional kinds of data structures beyond what the
+Arrow columnar format supports, like multidimensional arrays aka “tensors”. As
+such, we implemented the <a href="http://arrow.apache.org/docs/cpp/classarrow_1_1_tensor.html"><code class="highlighter-rouge">arrow::Tensor</code></a> C++ type which can utilize the
+rest of Arrow’s zero-copy shared memory machinery (using <code class="highlighter-rouge">arrow::Buffer</code> for
+managing memory lifetime). In C++ in particular, we will want to provide for
+additional data structures utilizing common IO and memory management tools.</p>
+
+<h3 id="start-of-javascript-typescript-implementation">Start of JavaScript (TypeScript) Implementation</h3>
+
+<p><a href="https://github.com/TheNeuralBit">Brian Hulette</a> started developing an Arrow implementation in
+<a href="https://conda-forge.github.io">TypeScript</a> for use in NodeJS and browser-side applications. We are
+benefitting from Flatbuffers’ first class support for JavaScript.</p>
+
+<h3 id="improved-website-and-developer-documentation">Improved Website and Developer Documentation</h3>
+
+<p>Since 0.2.0 we have implemented a new website stack for publishing
+documentation and blogs based on <a href="https://jekyllrb.com">Jekyll</a>. Kouhei Sutou developed a <a href="https://github.com/red-data-tools/jekyll-jupyter-notebook">Jekyll
+Jupyter Notebook plugin</a> so that we can use Jupyter to author content for
+the Arrow website.</p>
+
+<p>On the website, we have now published API documentation for the C, C++, Java,
+and Python subcomponents. Within these you will find easier-to-follow developer
+instructions for getting started.</p>
+
+<h3 id="contributors">Contributors</h3>
+
+<p>Thanks to all who contributed patches to this release.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
+ 119 Wes McKinney
+ 55 Kouhei Sutou
+ 18 Uwe L. Korn
+ 17 Julien Le Dem
+ 9 Phillip Cloud
+ 6 Bryan Cutler
+ 5 Philipp Moritz
+ 5 Emilio Lahr-Vivaz
+ 4 Max Risuhin
+ 4 Johan Mabille
+ 4 Jeff Knupp
+ 3 Steven Phillips
+ 3 Miki Tebeka
+ 2 Leif Walsh
+ 2 Jeff Reback
+ 2 Brian Hulette
+ 1 Tsuyoshi Ozawa
+ 1 rvernica
+ 1 Nong Li
+ 1 Julien Lafaye
+ 1 Itai Incze
+ 1 Holden Karau
+ 1 Deepak Majeti
+</code></pre>
+</div>
+
+
+
+ <hr/>
+<footer class="footer">
+ <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ <p>© 2017 Apache Software Foundation</p>
+</footer>
+
+ </div>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/1e5c92cb/blog/index.html
----------------------------------------------------------------------
diff --git a/blog/index.html b/blog/index.html
new file mode 100644
index 0000000..44c4dcd
--- /dev/null
+++ b/blog/index.html
@@ -0,0 +1,358 @@
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <title>Blog / News</title>
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="generator" content="Jekyll v3.4.3">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+
+ <title>Apache Arrow Homepage</title>
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+ <link href="/css/main.css" rel="stylesheet">
+ <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+ integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+ crossorigin="anonymous"></script>
+ <script src="/assets/javascripts/bootstrap.min.js"></script>
+ </head>
+
+
+<body class="wrap">
+ <div class="container">
+ <nav class="navbar navbar-default">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#arrow-navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/">Apache Arrow™ </a>
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="arrow-navbar">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Project Links<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/install/">Install</a></li>
+ <li><a href="/blog/">Blog</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
+ <li><a href="https://github.com/apache/arrow">Source Code</a></li>
+ <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
+ <li><a href="https://apachearrowslackin.herokuapp.com">Slack Channel</a></li>
+ <li><a href="/committers/">Committers</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Specification<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+ <li><a href="/docs/metadata.html">Metadata</a></li>
+ <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Documentation<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/python">Python</a></li>
+ <li><a href="/docs/cpp">C++ API</a></li>
+ <li><a href="/docs/java">Java API</a></li>
+ <li><a href="/docs/c_glib">C GLib API</a></li>
+ </ul>
+ </li>
+ <!-- <li><a href="/blog">Blog</a></li> -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">ASF Links<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/">ASF Website</a></li>
+ <li><a href="http://www.apache.org/licenses/">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+ <li><a href="http://www.apache.org/security/">Security</a></li>
+ </ul>
+ </li>
+ </ul>
+ <a href="http://www.apache.org/">
+ <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+ </a>
+ </div><!-- /.navbar-collapse -->
+ </div>
+ </nav>
+
+
+
+
+<h2>Project News and Blog</h2>
+<hr>
+
+
+
+ <div class="container">
+ <h2>
+ Apache Arrow 0.3 Release
+ <a href="/blog/2017/05/08/0.3-release/" class="permalink" title="Permalink">∞</a>
+ </h2>
+
+
+
+ <div class="panel">
+ <div class="panel-body">
+ <div>
+ <span class="label label-default">Published</span>
+ <span class="published">
+ <i class="fa fa-calendar"></i>
+ 08 May 2017
+ </span>
+ </div>
+ <div>
+ <span class="label label-default">By</span>
+ <a href="http://wesmckinney.com"><i class="fa fa-user"></i> Wes McKinney (wesm)</a>
+ </div>
+ </div>
+ </div>
+ <!--
+
+-->
+
+<p>The Apache Arrow team is pleased to announce the 0.3.0 release of the
+project. It is the product of an intense 10 weeks of development since the
+0.2.0 release from this past February. It includes <a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.3.0"><strong>306 resolved JIRAs</strong></a>
+from <a href="https://github.com/apache/arrow/graphs/contributors"><strong>23 contributors</strong></a>.</p>
+
+<p>While we have added many new features to the different Arrow implementations,
+one of the major development focuses in 2017 has been hardening the in-memory
+format, type metadata, and messaging protocol to provide a <strong>stable,
+production-ready foundation</strong> for big data applications. We are excited to be
+collaborating with the <a href="http://spark.apache.org">Apache Spark</a> and <a href="http://www.geomesa.org/">GeoMesa</a> communities on
+utilizing Arrow for high performance IO and in-memory data processing.</p>
+
+<p>See the <a href="http://arrow.apache.org/install">Install Page</a> to learn how to get the libraries for your platform.</p>
+
+<p>We will be publishing more information about the Apache Arrow roadmap as we
+forge ahead with using Arrow to accelerate big data systems.</p>
+
+<p>We are looking for more contributors from within our existing communities and
+from other communities (such as Go, R, or Julia) to get involved in Arrow
+development.</p>
+
+<h3 id="file-and-streaming-format-hardening">File and Streaming Format Hardening</h3>
+
+<p>The 0.2.0 release brought with it the first iterations of the <strong>random access</strong>
+and <strong>streaming</strong> Arrow wire formats. See the <a href="http://arrow.apache.org/docs/ipc.html">IPC specification</a> for
+implementation details and <a href="http://wesmckinney.com/blog/arrow-streaming-columnar/">example blog post</a> with some use cases. These
+provide low-overhead, zero-copy access to Arrow record batch payloads.</p>
+
+<p>In 0.3.0 we have solidified a number of small details with the binary format
+and improved our integration and unit testing particularly in the Java, C++,
+and Python libraries. Using the <a href="http://github.com/google/flatbuffers">Google Flatbuffers</a> project has helped with
+adding new features to our metadata without breaking forward compatibility.</p>
+
+<p>We are not yet ready to make a firm commitment to strong forward compatibility
+(in case we find something needs to change) in the binary format, but we will
+make efforts between major releases to not make unnecessary
+breakages. Contributions to the website and component user and API
+documentation would also be most welcome.</p>
+
+<h3 id="dictionary-encoding-support">Dictionary Encoding Support</h3>
+
+<p><a href="https://github.com/elahrvivaz">Emilio Lahr-Vivaz</a> from the <a href="http://www.geomesa.org/">GeoMesa</a> project contributed Java support
+for dictionary-encoded Arrow vectors. We followed up with C++ and Python
+support (and <code class="highlighter-rouge">pandas.Categorical</code> integration). We have not yet implemented
+full integration tests for dictionaries (for sending this data between C++ and
+Java), but hope to achieve this in the 0.4.0 Arrow release.</p>
+
+<p>This common data representation technique for categorical data allows multiple
+record batches to share a common “dictionary”, with the values in the batches
+being represented as integers referencing the dictionary. This data is called
+“categorical” or “factor” in statistical languages, while in file formats like
+Apache Parquet it is strictly used for data compression.</p>
+
+<h3 id="expanded-date-time-and-fixed-size-types">Expanded Date, Time, and Fixed Size Types</h3>
+
+<p>A notable omission from the 0.2.0 release was complete and integration-tested
+support for the gamut of date and time types that occur in the wild. These are
+needed for <a href="http://parquet.apache.org">Apache Parquet</a> and Apache Spark integration.</p>
+
+<ul>
+ <li><strong>Date</strong>: 32-bit (days unit) and 64-bit (milliseconds unit)</li>
+ <li><strong>Time</strong>: 64-bit integer with unit (second, millisecond, microsecond, nanosecond)</li>
+ <li><strong>Timestamp</strong>: 64-bit integer with unit, with or without timezone</li>
+ <li><strong>Fixed Size Binary</strong>: Primitive values occupying certain number of bytes</li>
+ <li><strong>Fixed Size List</strong>: List values with constant size (no separate offsets vector)</li>
+</ul>
+
+<p>We have additionally added experimental support for exact decimals in C++ using
+<a href="https://github.com/boostorg/multiprecision">Boost.Multiprecision</a>, though we have not yet hardened the Decimal memory
+format between the Java and C++ implementations.</p>
+
+<h3 id="c-and-python-support-on-windows">C++ and Python Support on Windows</h3>
+
+<p>We have made many general improvements to development and packaging for general
+C++ and Python development. 0.3.0 is the first release to bring full C++ and
+Python support for Windows on Visual Studio (MSVC) 2015 and 2017. In addition
+to adding Appveyor continuous integration for MSVC, we have also written guides
+for building from source on Windows: <a href="https://github.com/apache/arrow/blob/master/cpp/doc/Windows.md">C++</a> and <a href="https://github.com/apache/arrow/blob/master/python/doc/source/development.rst">Python</a>.</p>
+
+<p>For the first time, you can install the Arrow Python library on Windows from
+[conda-forge][31]:</p>
+
+<div class="language-shell highlighter-rouge"><pre class="highlight"><code>conda install pyarrow -c conda-forge
+</code></pre>
+</div>
+
+<h3 id="c-glib-bindings-with-support-for-ruby-lua-and-more">C (GLib) Bindings, with support for Ruby, Lua, and more</h3>
+
+<p><a href="http://github.com/kou">Kouhei Sutou</a> is a new Apache Arrow contributor and has contributed GLib C
+bindings (to the C++ libraries) for Linux. Using a C middleware framework
+called <a href="https://wiki.gnome.org/Projects/GObjectIntrospection">GObject Introspection</a>, it is possible to use these bindings
+seamlessly in Ruby, Lua, Go, and <a href="https://wiki.gnome.org/Projects/GObjectIntrospection/Users">other programming languages</a>. We will
+probably need to publish some follow up blogs explaining how these bindings
+work and how to use them.</p>
+
+<h3 id="apache-spark-integration-for-pyspark">Apache Spark Integration for PySpark</h3>
+
+<p>We have been collaborating with the Apache Spark community on <a href="https://issues.apache.org/jira/browse/SPARK-13534">SPARK-13534</a>
+to add support for using Arrow to accelerate <code class="highlighter-rouge">DataFrame.toPandas</code> in
+PySpark. We have observed over <a href="https://github.com/apache/spark/pull/15821#issuecomment-282175163"><strong>40x speedup</strong></a> from the more efficient
+data serialization.</p>
+
+<p>Using Arrow in PySpark opens the door to many other performance optimizations,
+particularly around UDF evaluation (e.g. <code class="highlighter-rouge">map</code> and <code class="highlighter-rouge">filter</code> operations with
+Python lambda functions).</p>
+
+<h3 id="new-python-feature-memory-views-feather-apache-parquet-support">New Python Feature: Memory Views, Feather, Apache Parquet support</h3>
+
+<p>Arrow’s Python library <code class="highlighter-rouge">pyarrow</code> is a Cython binding for the <code class="highlighter-rouge">libarrow</code> and
+<code class="highlighter-rouge">libarrow_python</code> C++ libraries, which handle inteoperability with NumPy,
+<a href="http://pandas.pydata.org">pandas</a>, and the Python standard library.</p>
+
+<p>At the heart of Arrow’s C++ libraries is the <code class="highlighter-rouge">arrow::Buffer</code> object, which is a
+managed memory view supporting zero-copy reads and slices. <a href="https://github.com/JeffKnupp">Jeff Knupp</a>
+contributed integration between Arrow buffers and the Python buffer protocol
+and memoryviews, so now code like this is possible:</p>
+
+<div class="language-python highlighter-rouge"><pre class="highlight"><code><span class="n">In</span> <span class="p">[</span><span class="mi">6</span><span class="p">]:</span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">7</span><span class="p">]:</span> <span class="n">buf</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">b</span><span class="s">'foobarbaz'</span><span class="p">)</span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span class="n">buf</span>
+<span class="n">Out</span><span class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span class="o"><</span><span class="n">pyarrow</span><span class="o">.</span><span class="n">_io</span><span class="o">.</span><span class="n">Buffer</span> <span class="n">at</span> <span class="mh">0x7f6c0a84b538</span><span class="o">></span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">9</span><span class="p">]:</span> <span class="n">memoryview</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
+<span class="n">Out</span><span class="p">[</span><span class="mi">9</span><span class="p">]:</span> <span class="o"><</span><span class="n">memory</span> <span class="n">at</span> <span class="mh">0x7f6c0a8c5e88</span><span class="o">></span>
+
+<span class="n">In</span> <span class="p">[</span><span class="mi">10</span><span class="p">]:</span> <span class="n">buf</span><span class="o">.</span><span class="n">to_pybytes</span><span class="p">()</span>
+<span class="n">Out</span><span class="p">[</span><span class="mi">10</span><span class="p">]:</span> <span class="n">b</span><span class="s">'foobarbaz'</span>
+</code></pre>
+</div>
+
+<p>We have significantly expanded <a href="http://parquet.apache.org"><strong>Apache Parquet</strong></a> support via the C++
+Parquet implementation <a href="https://github.com/apache/parquet-cpp">parquet-cpp</a>. This includes support for partitioned
+datasets on disk or in HDFS. We added initial Arrow-powered Parquet support <a href="https://github.com/dask/dask/commit/68f9e417924a985c1f2e2a587126833c70a2e9f4">in
+the Dask project</a>, and look forward to more collaborations with the Dask
+developers on distributed processing of pandas data.</p>
+
+<p>With Arrow’s support for pandas maturing, we were able to merge in the
+<a href="https://github.com/wesm/feather"><strong>Feather format</strong></a> implementation, which is essentially a special case of
+the Arrow random access format. We’ll be continuing Feather development within
+the Arrow codebase. For example, Feather can now read and write with Python
+file objects using Arrow’s Python binding layer.</p>
+
+<p>We also implemented more robust support for pandas-specific data types, like
+<code class="highlighter-rouge">DatetimeTZ</code> and <code class="highlighter-rouge">Categorical</code>.</p>
+
+<h3 id="support-for-tensors-and-beyond-in-c-library">Support for Tensors and beyond in C++ Library</h3>
+
+<p>There has been increased interest in using Apache Arrow as a tool for zero-copy
+shared memory management for machine learning applications. A flagship example
+is the <a href="https://github.com/ray-project/ray">Ray project</a> from the UC Berkeley <a href="https://rise.cs.berkeley.edu/">RISELab</a>.</p>
+
+<p>Machine learning deals in additional kinds of data structures beyond what the
+Arrow columnar format supports, like multidimensional arrays aka “tensors”. As
+such, we implemented the <a href="http://arrow.apache.org/docs/cpp/classarrow_1_1_tensor.html"><code class="highlighter-rouge">arrow::Tensor</code></a> C++ type which can utilize the
+rest of Arrow’s zero-copy shared memory machinery (using <code class="highlighter-rouge">arrow::Buffer</code> for
+managing memory lifetime). In C++ in particular, we will want to provide for
+additional data structures utilizing common IO and memory management tools.</p>
+
+<h3 id="start-of-javascript-typescript-implementation">Start of JavaScript (TypeScript) Implementation</h3>
+
+<p><a href="https://github.com/TheNeuralBit">Brian Hulette</a> started developing an Arrow implementation in
+<a href="https://conda-forge.github.io">TypeScript</a> for use in NodeJS and browser-side applications. We are
+benefitting from Flatbuffers’ first class support for JavaScript.</p>
+
+<h3 id="improved-website-and-developer-documentation">Improved Website and Developer Documentation</h3>
+
+<p>Since 0.2.0 we have implemented a new website stack for publishing
+documentation and blogs based on <a href="https://jekyllrb.com">Jekyll</a>. Kouhei Sutou developed a <a href="https://github.com/red-data-tools/jekyll-jupyter-notebook">Jekyll
+Jupyter Notebook plugin</a> so that we can use Jupyter to author content for
+the Arrow website.</p>
+
+<p>On the website, we have now published API documentation for the C, C++, Java,
+and Python subcomponents. Within these you will find easier-to-follow developer
+instructions for getting started.</p>
+
+<h3 id="contributors">Contributors</h3>
+
+<p>Thanks to all who contributed patches to this release.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>$ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
+ 119 Wes McKinney
+ 55 Kouhei Sutou
+ 18 Uwe L. Korn
+ 17 Julien Le Dem
+ 9 Phillip Cloud
+ 6 Bryan Cutler
+ 5 Philipp Moritz
+ 5 Emilio Lahr-Vivaz
+ 4 Max Risuhin
+ 4 Johan Mabille
+ 4 Jeff Knupp
+ 3 Steven Phillips
+ 3 Miki Tebeka
+ 2 Leif Walsh
+ 2 Jeff Reback
+ 2 Brian Hulette
+ 1 Tsuyoshi Ozawa
+ 1 rvernica
+ 1 Nong Li
+ 1 Julien Lafaye
+ 1 Itai Incze
+ 1 Holden Karau
+ 1 Deepak Majeti
+</code></pre>
+</div>
+
+
+ </div>
+
+
+
+
+ <hr/>
+<footer class="footer">
+ <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ <p>© 2017 Apache Software Foundation</p>
+</footer>
+
+ </div>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/1e5c92cb/committers/index.html
----------------------------------------------------------------------
diff --git a/committers/index.html b/committers/index.html
new file mode 100644
index 0000000..eb56d13
--- /dev/null
+++ b/committers/index.html
@@ -0,0 +1,209 @@
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <title></title>
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="generator" content="Jekyll v3.4.3">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+
+ <title>Apache Arrow Homepage</title>
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+ <link href="/css/main.css" rel="stylesheet">
+ <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+ integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+ crossorigin="anonymous"></script>
+ <script src="/assets/javascripts/bootstrap.min.js"></script>
+ </head>
+
+
+<body class="wrap">
+ <div class="container">
+ <nav class="navbar navbar-default">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#arrow-navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/">Apache Arrow™ </a>
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="arrow-navbar">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Project Links<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/install/">Install</a></li>
+ <li><a href="/blog/">Blog</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/ARROW">Issue Tracker</a></li>
+ <li><a href="https://github.com/apache/arrow">Source Code</a></li>
+ <li><a href="http://mail-archives.apache.org/mod_mbox/arrow-dev/">Mailing List</a></li>
+ <li><a href="https://apachearrowslackin.herokuapp.com">Slack Channel</a></li>
+ <li><a href="/committers/">Committers</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Specification<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+ <li><a href="/docs/metadata.html">Metadata</a></li>
+ <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+ </ul>
+ </li>
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">Documentation<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/python">Python</a></li>
+ <li><a href="/docs/cpp">C++ API</a></li>
+ <li><a href="/docs/java">Java API</a></li>
+ <li><a href="/docs/c_glib">C GLib API</a></li>
+ </ul>
+ </li>
+ <!-- <li><a href="/blog">Blog</a></li> -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+ role="button" aria-haspopup="true"
+ aria-expanded="false">ASF Links<span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://www.apache.org/">ASF Website</a></li>
+ <li><a href="http://www.apache.org/licenses/">License</a></li>
+ <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+ <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+ <li><a href="http://www.apache.org/security/">Security</a></li>
+ </ul>
+ </li>
+ </ul>
+ <a href="http://www.apache.org/">
+ <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+ </a>
+ </div><!-- /.navbar-collapse -->
+ </div>
+ </nav>
+
+
+ <div class="container">
+<h2>Committers</h2>
+<table class="table"><thead>
+<tr>
+<th>Name</th>
+<th>Alias (email is <alias>@apache.org)</th>
+</tr>
+</thead><tbody>
+<tr>
+<td>Jacques Nadeau</td>
+<td>jacques</td>
+</tr>
+<tr>
+<td>Todd Lipcon</td>
+<td>todd</td>
+</tr>
+<tr>
+<td>Ted Dunning</td>
+<td>tdunning</td>
+</tr>
+<tr>
+<td>Michael Stack</td>
+<td>stack</td>
+</tr>
+<tr>
+<td>P. Taylor Goetz</td>
+<td>ptgoetz</td>
+</tr>
+<tr>
+<td>Julian Hyde</td>
+<td>jhyde</td>
+</tr>
+<tr>
+<td>Reynold Xin</td>
+<td>rxin</td>
+</tr>
+<tr>
+<td>James Taylor</td>
+<td>jamestaylor</td>
+</tr>
+<tr>
+<td>Julien Le Dem</td>
+<td>julien</td>
+</tr>
+<tr>
+<td>Jake Luciani</td>
+<td>jake</td>
+</tr>
+<tr>
+<td>Jason Altekruse</td>
+<td>json</td>
+</tr>
+<tr>
+<td>Alex Levenson</td>
+<td>alexlevenson</td>
+</tr>
+<tr>
+<td>Parth Chandra</td>
+<td>parthc</td>
+</tr>
+<tr>
+<td>Marcel Kornacker</td>
+<td>marcel</td>
+</tr>
+<tr>
+<td>Steven Phillips</td>
+<td>smp</td>
+</tr>
+<tr>
+<td>Hanifi Gunes</td>
+<td>hg</td>
+</tr>
+<tr>
+<td>Abdelhakim Deneche</td>
+<td>adeneche</td>
+</tr>
+<tr>
+<td>Wes McKinney</td>
+<td>wesm</td>
+</tr>
+<tr>
+<td>David Alves</td>
+<td>dralves</td>
+</tr>
+<tr>
+<td>Ippokratis Pandis</td>
+<td>ippokratis</td>
+</tr>
+<tr>
+<td>Uwe L. Korn</td>
+<td>uwe</td>
+</tr>
+</tbody></table>
+
+ </div> <!-- /container -->
+ </body>
+</html>
+
+
+ <hr/>
+<footer class="footer">
+ <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ <p>© 2017 Apache Software Foundation</p>
+</footer>
+
+ </div>
+</body>
+</html>