You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2018/12/23 16:31:56 UTC
[49/51] [partial] arrow-site git commit: Upload nightly docs
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/_modules/index.html
----------------------------------------------------------------------
diff --git a/docs/latest/_modules/index.html b/docs/latest/_modules/index.html
new file mode 100644
index 0000000..befce19
--- /dev/null
+++ b/docs/latest/_modules/index.html
@@ -0,0 +1,226 @@
+
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>Overview: module code — Apache Arrow v0.11.1.dev473+g6ed02454</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../genindex.html" />
+ <link rel="search" title="Search" href="../search.html" />
+
+
+ <script src="../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../index.html" class="icon icon-home"> Apache Arrow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 0.11.1.dev473+g6ed02454
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <p class="caption"><span class="caption-text">Memory Format</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li>
+</ul>
+<p class="caption"><span class="caption-text">Languages</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../cpp/index.html">C++ Implementation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../index.html">Apache Arrow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../index.html">Docs</a> »</li>
+
+ <li>Overview: module code</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>All modules for which code is available</h1>
+<ul><li><a href="pyarrow.html">pyarrow</a></li>
+<ul><li><a href="pyarrow/_csv.html">pyarrow._csv</a></li>
+<li><a href="pyarrow/_plasma.html">pyarrow._plasma</a></li>
+<li><a href="pyarrow/feather.html">pyarrow.feather</a></li>
+<li><a href="pyarrow/filesystem.html">pyarrow.filesystem</a></li>
+<li><a href="pyarrow/hdfs.html">pyarrow.hdfs</a></li>
+<li><a href="pyarrow/ipc.html">pyarrow.ipc</a></li>
+<li><a href="pyarrow/lib.html">pyarrow.lib</a></li>
+<li><a href="pyarrow/parquet.html">pyarrow.parquet</a></li>
+<li><a href="pyarrow/types.html">pyarrow.types</a></li>
+</ul></ul>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+ © Copyright 2016-2018 Apache Software Foundation
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../_static/jquery.js"></script>
+ <script type="text/javascript" src="../_static/underscore.js"></script>
+ <script type="text/javascript" src="../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script>
+<script>
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments);}
+ gtag('js', new Date());
+
+ gtag('config', 'UA-107500873-1');
+</script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/_modules/pyarrow.html
----------------------------------------------------------------------
diff --git a/docs/latest/_modules/pyarrow.html b/docs/latest/_modules/pyarrow.html
new file mode 100644
index 0000000..7b0004b
--- /dev/null
+++ b/docs/latest/_modules/pyarrow.html
@@ -0,0 +1,478 @@
+
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>pyarrow — Apache Arrow v0.11.1.dev473+g6ed02454</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../genindex.html" />
+ <link rel="search" title="Search" href="../search.html" />
+
+
+ <script src="../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../index.html" class="icon icon-home"> Apache Arrow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 0.11.1.dev473+g6ed02454
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <p class="caption"><span class="caption-text">Memory Format</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li>
+</ul>
+<p class="caption"><span class="caption-text">Languages</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../cpp/index.html">C++ Implementation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../index.html">Apache Arrow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../index.html">Docs</a> »</li>
+
+ <li><a href="index.html">Module code</a> »</li>
+
+ <li>pyarrow</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for pyarrow</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="c1"># flake8: noqa</span>
+
+<span class="kn">import</span> <span class="nn">os</span> <span class="k">as</span> <span class="nn">_os</span>
+<span class="kn">import</span> <span class="nn">sys</span> <span class="k">as</span> <span class="nn">_sys</span>
+
+<span class="k">try</span><span class="p">:</span>
+ <span class="kn">from</span> <span class="nn">._generated_version</span> <span class="k">import</span> <span class="n">version</span> <span class="k">as</span> <span class="n">__version__</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="c1"># Package is not installed, parse git tag at runtime</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="kn">import</span> <span class="nn">setuptools_scm</span>
+ <span class="c1"># Code duplicated from setup.py to avoid a dependency on each other</span>
+ <span class="k">def</span> <span class="nf">parse_git</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Parse function for setuptools_scm that ignores tags for non-C++</span>
+<span class="sd"> subprojects, e.g. apache-arrow-js-XXX tags.</span>
+<span class="sd"> """</span>
+ <span class="kn">from</span> <span class="nn">setuptools_scm.git</span> <span class="k">import</span> <span class="n">parse</span>
+ <span class="n">kwargs</span><span class="p">[</span><span class="s1">'describe_command'</span><span class="p">]</span> <span class="o">=</span> \
+ <span class="s2">"git describe --dirty --tags --long --match 'apache-arrow-[0-9].*'"</span>
+ <span class="k">return</span> <span class="n">parse</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="n">__version__</span> <span class="o">=</span> <span class="n">setuptools_scm</span><span class="o">.</span><span class="n">get_version</span><span class="p">(</span><span class="s1">'../'</span><span class="p">,</span>
+ <span class="n">parse</span><span class="o">=</span><span class="n">parse_git</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+ <span class="n">__version__</span> <span class="o">=</span> <span class="kc">None</span>
+
+
+<span class="kn">import</span> <span class="nn">pyarrow.compat</span> <span class="k">as</span> <span class="nn">compat</span>
+
+<span class="c1"># Workaround for https://issues.apache.org/jira/browse/ARROW-2657</span>
+<span class="c1"># and https://issues.apache.org/jira/browse/ARROW-2920</span>
+<span class="k">if</span> <span class="n">_sys</span><span class="o">.</span><span class="n">platform</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'linux'</span><span class="p">,</span> <span class="s1">'linux2'</span><span class="p">):</span>
+ <span class="n">compat</span><span class="o">.</span><span class="n">import_tensorflow_extension</span><span class="p">()</span>
+ <span class="n">compat</span><span class="o">.</span><span class="n">import_pytorch_extension</span><span class="p">()</span>
+
+
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="n">cpu_count</span><span class="p">,</span> <span class="n">set_cpu_count</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">null</span><span class="p">,</span> <span class="n">bool_</span><span class="p">,</span>
+ <span class="n">int8</span><span class="p">,</span> <span class="n">int16</span><span class="p">,</span> <span class="n">int32</span><span class="p">,</span> <span class="n">int64</span><span class="p">,</span>
+ <span class="n">uint8</span><span class="p">,</span> <span class="n">uint16</span><span class="p">,</span> <span class="n">uint32</span><span class="p">,</span> <span class="n">uint64</span><span class="p">,</span>
+ <span class="n">time32</span><span class="p">,</span> <span class="n">time64</span><span class="p">,</span> <span class="n">timestamp</span><span class="p">,</span> <span class="n">date32</span><span class="p">,</span> <span class="n">date64</span><span class="p">,</span>
+ <span class="n">float16</span><span class="p">,</span> <span class="n">float32</span><span class="p">,</span> <span class="n">float64</span><span class="p">,</span>
+ <span class="n">binary</span><span class="p">,</span> <span class="n">string</span><span class="p">,</span> <span class="n">utf8</span><span class="p">,</span> <span class="n">decimal128</span><span class="p">,</span>
+ <span class="n">list_</span><span class="p">,</span> <span class="n">struct</span><span class="p">,</span> <span class="n">union</span><span class="p">,</span> <span class="n">dictionary</span><span class="p">,</span> <span class="n">field</span><span class="p">,</span>
+ <span class="n">type_for_alias</span><span class="p">,</span>
+ <span class="n">DataType</span><span class="p">,</span>
+ <span class="n">Field</span><span class="p">,</span>
+ <span class="n">Schema</span><span class="p">,</span>
+ <span class="n">schema</span><span class="p">,</span>
+ <span class="n">Array</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">,</span>
+ <span class="n">array</span><span class="p">,</span> <span class="n">chunked_array</span><span class="p">,</span> <span class="n">column</span><span class="p">,</span>
+ <span class="n">from_numpy_dtype</span><span class="p">,</span>
+ <span class="n">NullArray</span><span class="p">,</span>
+ <span class="n">NumericArray</span><span class="p">,</span> <span class="n">IntegerArray</span><span class="p">,</span> <span class="n">FloatingPointArray</span><span class="p">,</span>
+ <span class="n">BooleanArray</span><span class="p">,</span>
+ <span class="n">Int8Array</span><span class="p">,</span> <span class="n">UInt8Array</span><span class="p">,</span>
+ <span class="n">Int16Array</span><span class="p">,</span> <span class="n">UInt16Array</span><span class="p">,</span>
+ <span class="n">Int32Array</span><span class="p">,</span> <span class="n">UInt32Array</span><span class="p">,</span>
+ <span class="n">Int64Array</span><span class="p">,</span> <span class="n">UInt64Array</span><span class="p">,</span>
+ <span class="n">ListArray</span><span class="p">,</span> <span class="n">UnionArray</span><span class="p">,</span>
+ <span class="n">BinaryArray</span><span class="p">,</span> <span class="n">StringArray</span><span class="p">,</span>
+ <span class="n">FixedSizeBinaryArray</span><span class="p">,</span>
+ <span class="n">DictionaryArray</span><span class="p">,</span>
+ <span class="n">Date32Array</span><span class="p">,</span> <span class="n">Date64Array</span><span class="p">,</span>
+ <span class="n">TimestampArray</span><span class="p">,</span> <span class="n">Time32Array</span><span class="p">,</span> <span class="n">Time64Array</span><span class="p">,</span>
+ <span class="n">Decimal128Array</span><span class="p">,</span> <span class="n">StructArray</span><span class="p">,</span>
+ <span class="n">ArrayValue</span><span class="p">,</span> <span class="n">Scalar</span><span class="p">,</span> <span class="n">NA</span><span class="p">,</span> <span class="n">_NULL</span> <span class="k">as</span> <span class="n">NULL</span><span class="p">,</span>
+ <span class="n">BooleanValue</span><span class="p">,</span>
+ <span class="n">Int8Value</span><span class="p">,</span> <span class="n">Int16Value</span><span class="p">,</span> <span class="n">Int32Value</span><span class="p">,</span> <span class="n">Int64Value</span><span class="p">,</span>
+ <span class="n">UInt8Value</span><span class="p">,</span> <span class="n">UInt16Value</span><span class="p">,</span> <span class="n">UInt32Value</span><span class="p">,</span> <span class="n">UInt64Value</span><span class="p">,</span>
+ <span class="n">HalfFloatValue</span><span class="p">,</span> <span class="n">FloatValue</span><span class="p">,</span> <span class="n">DoubleValue</span><span class="p">,</span> <span class="n">ListValue</span><span class="p">,</span>
+ <span class="n">BinaryValue</span><span class="p">,</span> <span class="n">StringValue</span><span class="p">,</span> <span class="n">FixedSizeBinaryValue</span><span class="p">,</span>
+ <span class="n">DecimalValue</span><span class="p">,</span> <span class="n">UnionValue</span><span class="p">,</span> <span class="n">StructValue</span><span class="p">,</span> <span class="n">DictionaryValue</span><span class="p">,</span>
+ <span class="n">Date32Value</span><span class="p">,</span> <span class="n">Date64Value</span><span class="p">,</span>
+ <span class="n">Time32Value</span><span class="p">,</span> <span class="n">Time64Value</span><span class="p">,</span>
+ <span class="n">TimestampValue</span><span class="p">)</span>
+
+<span class="c1"># Buffers, allocation</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">Buffer</span><span class="p">,</span> <span class="n">ResizableBuffer</span><span class="p">,</span> <span class="n">foreign_buffer</span><span class="p">,</span> <span class="n">py_buffer</span><span class="p">,</span>
+ <span class="n">compress</span><span class="p">,</span> <span class="n">decompress</span><span class="p">,</span> <span class="n">allocate_buffer</span><span class="p">)</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">MemoryPool</span><span class="p">,</span> <span class="n">LoggingMemoryPool</span><span class="p">,</span> <span class="n">ProxyMemoryPool</span><span class="p">,</span>
+ <span class="n">total_allocated_bytes</span><span class="p">,</span> <span class="n">set_memory_pool</span><span class="p">,</span>
+ <span class="n">default_memory_pool</span><span class="p">,</span> <span class="n">logging_memory_pool</span><span class="p">,</span>
+ <span class="n">proxy_memory_pool</span><span class="p">,</span> <span class="n">log_memory_allocations</span><span class="p">)</span>
+
+<span class="c1"># I/O</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">HdfsFile</span><span class="p">,</span> <span class="n">NativeFile</span><span class="p">,</span> <span class="n">PythonFile</span><span class="p">,</span>
+ <span class="n">CompressedInputStream</span><span class="p">,</span> <span class="n">CompressedOutputStream</span><span class="p">,</span>
+ <span class="n">FixedSizeBufferWriter</span><span class="p">,</span>
+ <span class="n">BufferReader</span><span class="p">,</span> <span class="n">BufferOutputStream</span><span class="p">,</span>
+ <span class="n">OSFile</span><span class="p">,</span> <span class="n">MemoryMappedFile</span><span class="p">,</span> <span class="n">memory_map</span><span class="p">,</span>
+ <span class="n">create_memory_map</span><span class="p">,</span> <span class="n">have_libhdfs</span><span class="p">,</span> <span class="n">have_libhdfs3</span><span class="p">,</span>
+ <span class="n">MockOutputStream</span><span class="p">,</span> <span class="n">input_stream</span><span class="p">,</span> <span class="n">output_stream</span><span class="p">)</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">ChunkedArray</span><span class="p">,</span> <span class="n">Column</span><span class="p">,</span> <span class="n">RecordBatch</span><span class="p">,</span> <span class="n">Table</span><span class="p">,</span>
+ <span class="n">concat_tables</span><span class="p">)</span>
+
+<span class="c1"># Exceptions</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">ArrowException</span><span class="p">,</span>
+ <span class="n">ArrowKeyError</span><span class="p">,</span>
+ <span class="n">ArrowInvalid</span><span class="p">,</span>
+ <span class="n">ArrowIOError</span><span class="p">,</span>
+ <span class="n">ArrowMemoryError</span><span class="p">,</span>
+ <span class="n">ArrowNotImplementedError</span><span class="p">,</span>
+ <span class="n">ArrowTypeError</span><span class="p">,</span>
+ <span class="n">ArrowSerializationError</span><span class="p">,</span>
+ <span class="n">PlasmaObjectExists</span><span class="p">)</span>
+
+<span class="c1"># Serialization</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="p">(</span><span class="n">deserialize_from</span><span class="p">,</span> <span class="n">deserialize</span><span class="p">,</span>
+ <span class="n">deserialize_components</span><span class="p">,</span>
+ <span class="n">serialize</span><span class="p">,</span> <span class="n">serialize_to</span><span class="p">,</span> <span class="n">read_serialized</span><span class="p">,</span>
+ <span class="n">SerializedPyObject</span><span class="p">,</span> <span class="n">SerializationContext</span><span class="p">,</span>
+ <span class="n">SerializationCallbackError</span><span class="p">,</span>
+ <span class="n">DeserializationCallbackError</span><span class="p">)</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.filesystem</span> <span class="k">import</span> <span class="n">FileSystem</span><span class="p">,</span> <span class="n">LocalFileSystem</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.hdfs</span> <span class="k">import</span> <span class="n">HadoopFileSystem</span>
+<span class="kn">import</span> <span class="nn">pyarrow.hdfs</span> <span class="k">as</span> <span class="nn">hdfs</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.ipc</span> <span class="k">import</span> <span class="p">(</span><span class="n">Message</span><span class="p">,</span> <span class="n">MessageReader</span><span class="p">,</span>
+ <span class="n">RecordBatchFileReader</span><span class="p">,</span> <span class="n">RecordBatchFileWriter</span><span class="p">,</span>
+ <span class="n">RecordBatchStreamReader</span><span class="p">,</span> <span class="n">RecordBatchStreamWriter</span><span class="p">,</span>
+ <span class="n">read_message</span><span class="p">,</span> <span class="n">read_record_batch</span><span class="p">,</span> <span class="n">read_schema</span><span class="p">,</span>
+ <span class="n">read_tensor</span><span class="p">,</span> <span class="n">write_tensor</span><span class="p">,</span>
+ <span class="n">get_record_batch_size</span><span class="p">,</span> <span class="n">get_tensor_size</span><span class="p">,</span>
+ <span class="n">open_stream</span><span class="p">,</span>
+ <span class="n">open_file</span><span class="p">,</span>
+ <span class="n">serialize_pandas</span><span class="p">,</span> <span class="n">deserialize_pandas</span><span class="p">)</span>
+<span class="kn">import</span> <span class="nn">pyarrow.ipc</span> <span class="k">as</span> <span class="nn">ipc</span>
+
+
+<div class="viewcode-block" id="open_stream"><a class="viewcode-back" href="../python/generated/pyarrow.open_stream.html#pyarrow.open_stream">[docs]</a><span class="k">def</span> <span class="nf">open_stream</span><span class="p">(</span><span class="n">source</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> pyarrow.open_stream deprecated since 0.12, use pyarrow.ipc.open_stream</span>
+<span class="sd"> """</span>
+ <span class="kn">import</span> <span class="nn">warnings</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"pyarrow.open_stream is deprecated, please use "</span>
+ <span class="s2">"pyarrow.ipc.open_stream"</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">ipc</span><span class="o">.</span><span class="n">open_stream</span><span class="p">(</span><span class="n">source</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="open_file"><a class="viewcode-back" href="../python/generated/pyarrow.open_file.html#pyarrow.open_file">[docs]</a><span class="k">def</span> <span class="nf">open_file</span><span class="p">(</span><span class="n">source</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> pyarrow.open_file deprecated since 0.12, use pyarrow.ipc.open_file</span>
+<span class="sd"> """</span>
+ <span class="kn">import</span> <span class="nn">warnings</span>
+ <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"pyarrow.open_file is deprecated, please use "</span>
+ <span class="s2">"pyarrow.ipc.open_file"</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">ipc</span><span class="o">.</span><span class="n">open_file</span><span class="p">(</span><span class="n">source</span><span class="p">)</span></div>
+
+
+<span class="n">localfs</span> <span class="o">=</span> <span class="n">LocalFileSystem</span><span class="o">.</span><span class="n">get_instance</span><span class="p">()</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.serialization</span> <span class="k">import</span> <span class="p">(</span><span class="n">default_serialization_context</span><span class="p">,</span>
+ <span class="n">register_default_serialization_handlers</span><span class="p">,</span>
+ <span class="n">register_torch_serialization_handlers</span><span class="p">)</span>
+
+<span class="kn">import</span> <span class="nn">pyarrow.types</span> <span class="k">as</span> <span class="nn">types</span>
+
+<span class="c1"># Entry point for starting the plasma store</span>
+
+<span class="k">def</span> <span class="nf">_plasma_store_entry_point</span><span class="p">():</span>
+ <span class="sd">"""Entry point for starting the plasma store.</span>
+
+<span class="sd"> This can be used by invoking e.g.</span>
+<span class="sd"> ``plasma_store -s /tmp/plasma -m 1000000000``</span>
+<span class="sd"> from the command line and will start the plasma_store executable with the</span>
+<span class="sd"> given arguments.</span>
+<span class="sd"> """</span>
+ <span class="kn">import</span> <span class="nn">pyarrow</span>
+ <span class="n">plasma_store_executable</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">pyarrow</span><span class="o">.</span><span class="n">__path__</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
+ <span class="s2">"plasma_store_server"</span><span class="p">)</span>
+ <span class="n">_os</span><span class="o">.</span><span class="n">execv</span><span class="p">(</span><span class="n">plasma_store_executable</span><span class="p">,</span> <span class="n">_sys</span><span class="o">.</span><span class="n">argv</span><span class="p">)</span>
+
+<span class="c1"># ----------------------------------------------------------------------</span>
+<span class="c1"># Deprecations</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.util</span> <span class="k">import</span> <span class="n">_deprecate_api</span> <span class="c1"># noqa</span>
+
+<span class="c1"># ----------------------------------------------------------------------</span>
+<span class="c1"># Returning absolute path to the pyarrow include directory (if bundled, e.g. in</span>
+<span class="c1"># wheels)</span>
+
+<div class="viewcode-block" id="get_include"><a class="viewcode-back" href="../python/generated/pyarrow.get_include.html#pyarrow.get_include">[docs]</a><span class="k">def</span> <span class="nf">get_include</span><span class="p">():</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return absolute path to directory containing Arrow C++ include</span>
+<span class="sd"> headers. Similar to numpy.get_include</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">),</span> <span class="s1">'include'</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="get_libraries"><a class="viewcode-back" href="../python/generated/pyarrow.get_libraries.html#pyarrow.get_libraries">[docs]</a><span class="k">def</span> <span class="nf">get_libraries</span><span class="p">():</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return list of library names to include in the `libraries` argument for C</span>
+<span class="sd"> or Cython extensions using pyarrow</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s1">'arrow'</span><span class="p">,</span> <span class="s1">'arrow_python'</span><span class="p">]</span></div>
+
+
+<div class="viewcode-block" id="get_library_dirs"><a class="viewcode-back" href="../python/generated/pyarrow.get_library_dirs.html#pyarrow.get_library_dirs">[docs]</a><span class="k">def</span> <span class="nf">get_library_dirs</span><span class="p">():</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return lists of directories likely to contain Arrow C++ libraries for</span>
+<span class="sd"> linking C or Cython extensions using pyarrow</span>
+<span class="sd"> """</span>
+ <span class="n">package_cwd</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span>
+
+ <span class="n">library_dirs</span> <span class="o">=</span> <span class="p">[</span><span class="n">package_cwd</span><span class="p">]</span>
+
+ <span class="c1"># Search library paths via pkg-config. This is necessary if the user</span>
+ <span class="c1"># installed libarrow and the other shared libraries manually and they</span>
+ <span class="c1"># are not shipped inside the pyarrow package (see also ARROW-2976).</span>
+ <span class="kn">from</span> <span class="nn">subprocess</span> <span class="k">import</span> <span class="n">call</span><span class="p">,</span> <span class="n">PIPE</span><span class="p">,</span> <span class="n">Popen</span>
+ <span class="n">pkg_config_executable</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PKG_CONFIG'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">'pkg-config'</span>
+ <span class="k">for</span> <span class="n">package</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"arrow"</span><span class="p">,</span> <span class="s2">"plasma"</span><span class="p">,</span> <span class="s2">"arrow_python"</span><span class="p">]:</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{0}</span><span class="s1"> --exists </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pkg_config_executable</span><span class="p">,</span> <span class="n">package</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">call</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">pkg_config_executable</span><span class="p">,</span> <span class="s2">"--libs-only-L"</span><span class="p">,</span> <span class="n">package</span><span class="p">]</span>
+ <span class="n">proc</span> <span class="o">=</span> <span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">PIPE</span><span class="p">)</span>
+ <span class="n">out</span><span class="p">,</span> <span class="n">err</span> <span class="o">=</span> <span class="n">proc</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span>
+ <span class="n">library_dir</span> <span class="o">=</span> <span class="n">out</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)[</span><span class="mi">2</span><span class="p">:]</span> <span class="c1"># strip "-L"</span>
+ <span class="k">if</span> <span class="n">library_dir</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">library_dirs</span><span class="p">:</span>
+ <span class="n">library_dirs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">library_dir</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+ <span class="k">pass</span>
+
+ <span class="k">if</span> <span class="n">_sys</span><span class="o">.</span><span class="n">platform</span> <span class="o">==</span> <span class="s1">'win32'</span><span class="p">:</span>
+ <span class="c1"># TODO(wesm): Is this necessary, or does setuptools within a conda</span>
+ <span class="c1"># installation add Library\lib to the linker path for MSVC?</span>
+ <span class="n">python_base_install</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">_sys</span><span class="o">.</span><span class="n">executable</span><span class="p">)</span>
+ <span class="n">library_lib</span> <span class="o">=</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">python_base_install</span><span class="p">,</span> <span class="s1">'Library'</span><span class="p">,</span> <span class="s1">'lib'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">library_lib</span><span class="p">,</span> <span class="s1">'arrow.lib'</span><span class="p">)):</span>
+ <span class="n">library_dirs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">library_lib</span><span class="p">)</span>
+
+ <span class="c1"># ARROW-4074: Allow for ARROW_HOME to be set to some other directory</span>
+ <span class="k">if</span> <span class="s1">'ARROW_HOME'</span> <span class="ow">in</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="p">:</span>
+ <span class="n">library_dirs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'ARROW_HOME'</span><span class="p">],</span> <span class="s1">'lib'</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="n">library_dirs</span></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+ © Copyright 2016-2018 Apache Software Foundation
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../_static/jquery.js"></script>
+ <script type="text/javascript" src="../_static/underscore.js"></script>
+ <script type="text/javascript" src="../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script>
+<script>
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments);}
+ gtag('js', new Date());
+
+ gtag('config', 'UA-107500873-1');
+</script>
+
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/_modules/pyarrow/feather.html
----------------------------------------------------------------------
diff --git a/docs/latest/_modules/pyarrow/feather.html b/docs/latest/_modules/pyarrow/feather.html
new file mode 100644
index 0000000..0e38446
--- /dev/null
+++ b/docs/latest/_modules/pyarrow/feather.html
@@ -0,0 +1,455 @@
+
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>pyarrow.feather — Apache Arrow v0.11.1.dev473+g6ed02454</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../genindex.html" />
+ <link rel="search" title="Search" href="../../search.html" />
+
+
+ <script src="../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../index.html" class="icon icon-home"> Apache Arrow
+
+
+
+ </a>
+
+
+
+
+ <div class="version">
+ 0.11.1.dev473+g6ed02454
+ </div>
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <p class="caption"><span class="caption-text">Memory Format</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../format/README.html">Arrow specification documents</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../format/Guidelines.html">Implementation guidelines</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../format/Layout.html">Physical memory layout</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../format/IPC.html">Interprocess messaging / communication (IPC)</a></li>
+</ul>
+<p class="caption"><span class="caption-text">Languages</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../cpp/index.html">C++ Implementation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../python/index.html">Python bindings</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../index.html">Apache Arrow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../index.html">Docs</a> »</li>
+
+ <li><a href="../index.html">Module code</a> »</li>
+
+ <li><a href="../pyarrow.html">pyarrow</a> »</li>
+
+ <li>pyarrow.feather</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for pyarrow.feather</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">distutils.version</span> <span class="k">import</span> <span class="n">LooseVersion</span>
+<span class="kn">import</span> <span class="nn">os</span>
+
+<span class="kn">import</span> <span class="nn">six</span>
+<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.compat</span> <span class="k">import</span> <span class="n">pdapi</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="n">FeatherError</span> <span class="c1"># noqa</span>
+<span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="k">import</span> <span class="n">Table</span><span class="p">,</span> <span class="n">concat_tables</span>
+<span class="kn">import</span> <span class="nn">pyarrow.lib</span> <span class="k">as</span> <span class="nn">ext</span>
+
+
+<span class="k">try</span><span class="p">:</span>
+ <span class="n">infer_dtype</span> <span class="o">=</span> <span class="n">pdapi</span><span class="o">.</span><span class="n">infer_dtype</span>
+<span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+ <span class="n">infer_dtype</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">lib</span><span class="o">.</span><span class="n">infer_dtype</span>
+
+
+<span class="k">if</span> <span class="n">LooseVersion</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">__version__</span><span class="p">)</span> <span class="o"><</span> <span class="s1">'0.17.0'</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ImportError</span><span class="p">(</span><span class="s2">"feather requires pandas >= 0.17.0"</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">FeatherReader</span><span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">FeatherReader</span><span class="p">):</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">source</span> <span class="o">=</span> <span class="n">source</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">source</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">read_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">columns</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read</span><span class="p">()</span>
+ <span class="n">column_types</span> <span class="o">=</span> <span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="n">column</span><span class="p">)</span> <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">]</span>
+ <span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span> <span class="o">==</span> <span class="nb">int</span><span class="p">,</span> <span class="n">column_types</span><span class="p">)):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_indices</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="nb">all</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span> <span class="o">==</span> <span class="nb">str</span><span class="p">,</span> <span class="n">column_types</span><span class="p">)):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_names</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
+
+ <span class="n">column_type_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="vm">__name__</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">column_types</span><span class="p">]</span>
+ <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Columns must be indices or names. "</span>
+ <span class="s2">"Got columns </span><span class="si">{}</span><span class="s2"> of types </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">columns</span><span class="p">,</span> <span class="n">column_type_names</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">read_pandas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span>
+ <span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">check_chunked_overflow</span><span class="p">(</span><span class="n">col</span><span class="p">):</span>
+ <span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">num_chunks</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+ <span class="k">return</span>
+
+ <span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">binary</span><span class="p">(),</span> <span class="n">ext</span><span class="o">.</span><span class="n">string</span><span class="p">()):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Column '</span><span class="si">{0}</span><span class="s2">' exceeds 2GB maximum capacity of "</span>
+ <span class="s2">"a Feather binary column. This restriction may be "</span>
+ <span class="s2">"lifted in the future"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">col</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># TODO(wesm): Not sure when else this might be reached</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Column '</span><span class="si">{0}</span><span class="s2">' of type </span><span class="si">{1}</span><span class="s2"> was chunked on conversion "</span>
+ <span class="s2">"to Arrow and cannot be currently written to "</span>
+ <span class="s2">"Feather format"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">col</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">col</span><span class="o">.</span><span class="n">type</span><span class="p">)))</span>
+
+
+<span class="k">class</span> <span class="nc">FeatherWriter</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">dest</span> <span class="o">=</span> <span class="n">dest</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">writer</span> <span class="o">=</span> <span class="n">ext</span><span class="o">.</span><span class="n">FeatherWriter</span><span class="p">()</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">writer</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">dest</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">):</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">SparseDataFrame</span><span class="p">):</span>
+ <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">to_dense</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">is_unique</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"cannot serialize duplicate column names"</span><span class="p">)</span>
+
+ <span class="c1"># TODO(wesm): Remove this length check, see ARROW-1732</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">table</span> <span class="o">=</span> <span class="n">Table</span><span class="o">.</span><span class="n">from_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">preserve_index</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">names</span><span class="p">):</span>
+ <span class="n">col</span> <span class="o">=</span> <span class="n">table</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
+ <span class="n">check_chunked_overflow</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">writer</span><span class="o">.</span><span class="n">write_array</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">col</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">writer</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+
+<span class="k">class</span> <span class="nc">FeatherDataset</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Encapsulates details of reading a list of Feather files.</span>
+
+<span class="sd"> Parameters</span>
+<span class="sd"> ----------</span>
+<span class="sd"> path_or_paths : List[str]</span>
+<span class="sd"> A list of file names</span>
+<span class="sd"> validate_schema : boolean, default True</span>
+<span class="sd"> Check that individual file schemas are all the same / compatible</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path_or_paths</span><span class="p">,</span> <span class="n">validate_schema</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">paths</span> <span class="o">=</span> <span class="n">path_or_paths</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_schema</span> <span class="o">=</span> <span class="n">validate_schema</span>
+
+ <span class="k">def</span> <span class="nf">read_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Read multiple feather files as a single pyarrow.Table</span>
+
+<span class="sd"> Parameters</span>
+<span class="sd"> ----------</span>
+<span class="sd"> columns : List[str]</span>
+<span class="sd"> Names of columns to read from the file</span>
+
+<span class="sd"> Returns</span>
+<span class="sd"> -------</span>
+<span class="sd"> pyarrow.Table</span>
+<span class="sd"> Content of the file as a table (of columns)</span>
+<span class="sd"> """</span>
+ <span class="n">_fil</span> <span class="o">=</span> <span class="n">FeatherReader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">paths</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_tables</span> <span class="o">=</span> <span class="p">[</span><span class="n">_fil</span><span class="p">]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">_fil</span><span class="o">.</span><span class="n">schema</span>
+
+ <span class="k">for</span> <span class="n">fil</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">paths</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
+ <span class="n">fil_table</span> <span class="o">=</span> <span class="n">FeatherReader</span><span class="p">(</span><span class="n">fil</span><span class="p">)</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_schema</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">validate_schemas</span><span class="p">(</span><span class="n">fil</span><span class="p">,</span> <span class="n">fil_table</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_tables</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fil_table</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">concat_tables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_tables</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">validate_schemas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">piece</span><span class="p">,</span> <span class="n">table</span><span class="p">):</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">equals</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">):</span>
+ <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Schema in </span><span class="si">{0!s}</span><span class="s1"> was different. </span><span class="se">\n</span><span class="s1">'</span>
+ <span class="s1">'</span><span class="si">{1!s}</span><span class="se">\n\n</span><span class="s1">vs</span><span class="se">\n\n</span><span class="si">{2!s}</span><span class="s1">'</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">piece</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+ <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">))</span>
+
+ <span class="k">def</span> <span class="nf">read_pandas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Read multiple Parquet files as a single pandas DataFrame</span>
+
+<span class="sd"> Parameters</span>
+<span class="sd"> ----------</span>
+<span class="sd"> columns : List[str]</span>
+<span class="sd"> Names of columns to read from the file</span>
+<span class="sd"> use_threads : boolean, default True</span>
+<span class="sd"> Use multiple threads when converting to pandas</span>
+
+<span class="sd"> Returns</span>
+<span class="sd"> -------</span>
+<span class="sd"> pandas.DataFrame</span>
+<span class="sd"> Content of the file as a pandas DataFrame (of columns)</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span>
+ <span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="write_feather"><a class="viewcode-back" href="../../python/generated/pyarrow.feather.write_feather.html#pyarrow.feather.write_feather">[docs]</a><span class="k">def</span> <span class="nf">write_feather</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Write a pandas.DataFrame to Feather format</span>
+
+<span class="sd"> Parameters</span>
+<span class="sd"> ----------</span>
+<span class="sd"> df : pandas.DataFrame</span>
+<span class="sd"> dest : string</span>
+<span class="sd"> Local file path</span>
+<span class="sd"> """</span>
+ <span class="n">writer</span> <span class="o">=</span> <span class="n">FeatherWriter</span><span class="p">(</span><span class="n">dest</span><span class="p">)</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">writer</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+ <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+ <span class="c1"># Try to make sure the resource is closed</span>
+ <span class="kn">import</span> <span class="nn">gc</span>
+ <span class="n">writer</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="n">gc</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
+ <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dest</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
+ <span class="k">try</span><span class="p">:</span>
+ <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">dest</span><span class="p">)</span>
+ <span class="k">except</span> <span class="n">os</span><span class="o">.</span><span class="n">error</span><span class="p">:</span>
+ <span class="k">pass</span>
+ <span class="k">raise</span></div>
+
+
+<div class="viewcode-block" id="read_feather"><a class="viewcode-back" href="../../python/generated/pyarrow.feather.read_feather.html#pyarrow.feather.read_feather">[docs]</a><span class="k">def</span> <span class="nf">read_feather</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Read a pandas.DataFrame from Feather format</span>
+
+<span class="sd"> Parameters</span>
+<span class="sd"> ----------</span>
+<span class="sd"> source : string file path, or file-like object</span>
+<span class="sd"> columns : sequence, optional</span>
+<span class="sd"> Only read a specific set of columns. If not provided, all columns are</span>
+<span class="sd"> read</span>
+<span class="sd"> use_threads: bool, default True</span>
+<span class="sd"> Whether to parallelize reading using multiple threads</span>
+
+<span class="sd"> Returns</span>
+<span class="sd"> -------</span>
+<span class="sd"> df : pandas.DataFrame</span>
+<span class="sd"> """</span>
+ <span class="n">reader</span> <span class="o">=</span> <span class="n">FeatherReader</span><span class="p">(</span><span class="n">source</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_pandas</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">read_table</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Read a pyarrow.Table from Feather format</span>
+
+<span class="sd"> Parameters</span>
+<span class="sd"> ----------</span>
+<span class="sd"> source : string file path, or file-like object</span>
+<span class="sd"> columns : sequence, optional</span>
+<span class="sd"> Only read a specific set of columns. If not provided, all columns are</span>
+<span class="sd"> read</span>
+
+<span class="sd"> Returns</span>
+<span class="sd"> -------</span>
+<span class="sd"> table : pyarrow.Table</span>
+<span class="sd"> """</span>
+ <span class="n">reader</span> <span class="o">=</span> <span class="n">FeatherReader</span><span class="p">(</span><span class="n">source</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+ © Copyright 2016-2018 Apache Software Foundation
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+
+
+ <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../_static/doctools.js"></script>
+
+
+
+
+ <script type="text/javascript" src="../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script>
+<script>
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments);}
+ gtag('js', new Date());
+
+ gtag('config', 'UA-107500873-1');
+</script>
+
+
+</body>
+</html>
\ No newline at end of file