You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2019/01/10 20:05:38 UTC
[arrow] branch master updated: ARROW-4216: [Python] Add CUDA API
docs
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 5a502d2 ARROW-4216: [Python] Add CUDA API docs
5a502d2 is described below
commit 5a502d281545402240e818d5fd97a9aaf36363f2
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Thu Jan 10 21:05:31 2019 +0100
ARROW-4216: [Python] Add CUDA API docs
Also reorganize the API docs into several documents, and add/improve docstrings.
To allow building the docs without CUDA enabled, I added some conditional inclusion logic. When CUDA isn't enabled, the API docs are still generated but the docstrings are empty. This seems to be the only sane setting that doesn't produce Sphinx errors, one way or the other.
Author: Antoine Pitrou <an...@python.org>
Closes #3372 from pitrou/ARROW-4216-cuda-py-docs and squashes the following commits:
80600da5 <Antoine Pitrou> ARROW-4216: Add CUDA API docs
---
docs/source/conf.py | 33 +++
docs/source/python/api.rst | 389 ++---------------------------------
docs/source/python/api/arrays.rst | 109 ++++++++++
docs/source/python/api/cuda.rst | 62 ++++++
docs/source/python/api/datatypes.rst | 134 ++++++++++++
docs/source/python/api/files.rst | 65 ++++++
docs/source/python/api/formats.rst | 70 +++++++
docs/source/python/api/ipc.rst | 59 ++++++
docs/source/python/api/memory.rst | 68 ++++++
docs/source/python/api/misc.rst | 40 ++++
docs/source/python/api/plasma.rst | 33 +++
docs/source/python/api/tables.rst | 54 +++++
python/pyarrow/__init__.py | 4 +-
python/pyarrow/_cuda.pyx | 70 ++++---
python/pyarrow/array.pxi | 132 +++++++++---
python/pyarrow/io.pxi | 86 ++++++--
python/pyarrow/memory.pxi | 14 +-
python/pyarrow/scalar.pxi | 106 +++++++++-
python/pyarrow/types.pxi | 103 +++++++++-
19 files changed, 1167 insertions(+), 464 deletions(-)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 1cadef1..d525fa9 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -53,6 +53,7 @@ extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.autosummary',
'sphinx.ext.doctest',
+ 'sphinx.ext.ifconfig',
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
'sphinx.ext.napoleon',
@@ -69,6 +70,9 @@ autodoc_default_options = {
'inherited-members': None
}
+# Overriden conditionally below
+autodoc_mock_imports = []
+
# ipython directive options
ipython_mplbackend = ''
@@ -387,3 +391,32 @@ texinfo_documents = [
# If true, do not generate a @detailmenu in the "Top" node's menu.
#
# texinfo_no_detailmenu = False
+
+
+# -- Customization --------------------------------------------------------
+
+# Conditional API doc generation
+
+# Sphinx has two features for conditional inclusion:
+# - The "only" directive
+# https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#including-content-based-on-tags
+# - The "ifconfig" extension
+# https://www.sphinx-doc.org/en/master/usage/extensions/ifconfig.html
+#
+# Both have issues, but "ifconfig" seems to work in this setting.
+
+try:
+ import pyarrow.cuda
+ cuda_enabled = True
+except ImportError:
+ cuda_enabled = False
+ # Mock pyarrow.cuda to avoid autodoc warnings.
+ # XXX I can't get autodoc_mock_imports to work, so mock manually instead
+ # (https://github.com/sphinx-doc/sphinx/issues/2174#issuecomment-453177550)
+ from unittest import mock
+ pyarrow.cuda = sys.modules['pyarrow.cuda'] = mock.Mock()
+
+def setup(app):
+ # Use a config value to indicate whether CUDA API docs can be generated.
+ # This will also rebuild appropriately when the value changes.
+ app.add_config_value('cuda_enabled', cuda_enabled, 'env')
diff --git a/docs/source/python/api.rst b/docs/source/python/api.rst
index 0bad76f..b06509f 100644
--- a/docs/source/python/api.rst
+++ b/docs/source/python/api.rst
@@ -15,385 +15,22 @@
.. specific language governing permissions and limitations
.. under the License.
-.. currentmodule:: pyarrow
.. _api:
*************
API Reference
*************
-.. _api.types:
-
-Type and Schema Factory Functions
----------------------------------
-
-.. autosummary::
- :toctree: generated/
-
- null
- bool_
- int8
- int16
- int32
- int64
- uint8
- uint16
- uint32
- uint64
- float16
- float32
- float64
- time32
- time64
- timestamp
- date32
- date64
- binary
- string
- utf8
- decimal128
- list_
- struct
- dictionary
- field
- schema
- from_numpy_dtype
-
-.. currentmodule:: pyarrow.types
-.. _api.types.checking:
-
-Type checking functions
------------------------
-
-.. autosummary::
- :toctree: generated/
-
- is_boolean
- is_integer
- is_signed_integer
- is_unsigned_integer
- is_int8
- is_int16
- is_int32
- is_int64
- is_uint8
- is_uint16
- is_uint32
- is_uint64
- is_floating
- is_float16
- is_float32
- is_float64
- is_decimal
- is_list
- is_struct
- is_union
- is_nested
- is_temporal
- is_timestamp
- is_date
- is_date32
- is_date64
- is_time
- is_time32
- is_time64
- is_null
- is_binary
- is_unicode
- is_string
- is_fixed_size_binary
- is_map
- is_dictionary
-
-.. currentmodule:: pyarrow
-
-.. _api.value:
-
-Scalar Value Types
-------------------
-
-.. autosummary::
- :toctree: generated/
-
- NA
- Scalar
- ArrayValue
- BooleanValue
- Int8Value
- Int16Value
- Int32Value
- Int64Value
- UInt8Value
- UInt16Value
- UInt32Value
- UInt64Value
- FloatValue
- DoubleValue
- ListValue
- BinaryValue
- StringValue
- FixedSizeBinaryValue
- Date32Value
- Date64Value
- TimestampValue
- DecimalValue
-
-.. _api.array:
-
-.. currentmodule:: pyarrow
-
-Array Types
------------
-
-.. autosummary::
- :toctree: generated/
-
- array
- Array
- BooleanArray
- DictionaryArray
- FloatingPointArray
- IntegerArray
- Int8Array
- Int16Array
- Int32Array
- Int64Array
- NullArray
- NumericArray
- UInt8Array
- UInt16Array
- UInt32Array
- UInt64Array
- BinaryArray
- FixedSizeBinaryArray
- StringArray
- Time32Array
- Time64Array
- Date32Array
- Date64Array
- TimestampArray
- Decimal128Array
- ListArray
-
-.. _api.table:
-
-.. currentmodule:: pyarrow
-
-Tables and Record Batches
--------------------------
-
-.. autosummary::
- :toctree: generated/
-
- column
- chunked_array
- concat_tables
- ChunkedArray
- Column
- RecordBatch
- Table
-
-.. _api.tensor:
-
-Tensor type and Functions
--------------------------
-
-.. autosummary::
- :toctree: generated/
-
- Tensor
-
-.. _api.io:
-
-In-Memory Buffers
------------------
-
-.. autosummary::
- :toctree: generated/
-
- allocate_buffer
- compress
- decompress
- py_buffer
- foreign_buffer
- Buffer
- ResizableBuffer
-
-Input / Output and Shared Memory
---------------------------------
-
-.. autosummary::
- :toctree: generated/
-
- input_stream
- output_stream
- BufferReader
- BufferOutputStream
- FixedSizeBufferWriter
- NativeFile
- OSFile
- MemoryMappedFile
- CompressedInputStream
- CompressedOutputStream
- memory_map
- create_memory_map
- PythonFile
-
-File Systems
-------------
-
-.. autosummary::
- :toctree: generated/
-
- hdfs.connect
- LocalFileSystem
-
-.. class:: HadoopFileSystem
- :noindex:
-
-.. _api.ipc:
-
-Serialization and IPC
----------------------
-
-.. autosummary::
- :toctree: generated/
-
- ipc.open_file
- ipc.open_stream
- Message
- MessageReader
- RecordBatchFileReader
- RecordBatchFileWriter
- RecordBatchStreamReader
- RecordBatchStreamWriter
- read_message
- read_record_batch
- get_record_batch_size
- read_tensor
- write_tensor
- get_tensor_size
- serialize
- serialize_to
- deserialize
- deserialize_components
- deserialize_from
- read_serialized
- SerializedPyObject
- SerializationContext
-
-.. _api.memory_pool:
-
-Memory Pools
-------------
-
-.. currentmodule:: pyarrow
-
-.. autosummary::
- :toctree: generated/
-
- MemoryPool
- default_memory_pool
- total_allocated_bytes
- set_memory_pool
- log_memory_allocations
-
-.. _api.type_classes:
-
-.. currentmodule:: pyarrow
-
-Type Classes
-------------
-
-.. autosummary::
- :toctree: generated/
-
- DataType
- Field
- Schema
-
-.. currentmodule:: pyarrow.plasma
-
-.. _api.plasma:
-
-Plasma In-Memory Object Store
------------------------------
-
-.. autosummary::
- :toctree: generated/
-
- ObjectID
- PlasmaClient
- PlasmaBuffer
-
-.. currentmodule:: pyarrow.csv
-
-.. _api.csv:
-
-CSV Files
----------
-
-.. autosummary::
- :toctree: generated/
-
- ReadOptions
- ParseOptions
- ConvertOptions
- read_csv
-
-.. _api.feather:
-
-Feather Files
--------------
-
-.. currentmodule:: pyarrow.feather
-
-.. autosummary::
- :toctree: generated/
-
- read_feather
- write_feather
-
-.. currentmodule:: pyarrow
-
-.. _api.parquet:
-
-Parquet Files
--------------
-
-.. currentmodule:: pyarrow.parquet
-
-.. autosummary::
- :toctree: generated/
-
- ParquetDataset
- ParquetFile
- ParquetWriter
- read_table
- read_metadata
- read_pandas
- read_schema
- write_metadata
- write_table
- write_to_dataset
-
-.. currentmodule:: pyarrow
-
-Multi-Threading
----------------
-
-.. autosummary::
- :toctree: generated/
-
- cpu_count
- set_cpu_count
-
-Using with C extensions
------------------------
-
-.. autosummary::
- :toctree: generated/
-
- get_include
- get_libraries
- get_library_dirs
+.. toctree::
+ :maxdepth: 2
+
+ api/datatypes
+ api/arrays
+ api/memory
+ api/files
+ api/tables
+ api/ipc
+ api/formats
+ api/plasma
+ api/cuda
+ api/misc
diff --git a/docs/source/python/api/arrays.rst b/docs/source/python/api/arrays.rst
new file mode 100644
index 0000000..db45eef
--- /dev/null
+++ b/docs/source/python/api/arrays.rst
@@ -0,0 +1,109 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. _api.array:
+.. currentmodule:: pyarrow
+
+Arrays and Scalars
+==================
+
+Factory Function
+----------------
+
+This function is the main entry point to create an Arrow array from Python.
+
+.. autosummary::
+ :toctree: ../generated/
+
+ array
+
+Array Types
+-----------
+
+An array's Python class depends on its data type. Concrete array classes
+may expose data type-specific methods or properties.
+
+.. autosummary::
+ :toctree: ../generated/
+
+ Array
+ BooleanArray
+ FloatingPointArray
+ IntegerArray
+ Int8Array
+ Int16Array
+ Int32Array
+ Int64Array
+ NullArray
+ NumericArray
+ UInt8Array
+ UInt16Array
+ UInt32Array
+ UInt64Array
+ BinaryArray
+ StringArray
+ FixedSizeBinaryArray
+ Time32Array
+ Time64Array
+ Date32Array
+ Date64Array
+ TimestampArray
+ Decimal128Array
+ DictionaryArray
+ ListArray
+ StructArray
+ UnionArray
+
+.. _api.scalar:
+
+Array Scalars
+-------------
+
+Indexing an array wraps the represented value in a scalar object whose
+concrete type depends on the array data type. You shouldn't instantiate
+any of those classes directly.
+
+.. autosummary::
+ :toctree: ../generated/
+
+ NA
+ Scalar
+ ArrayValue
+ BooleanValue
+ Int8Value
+ Int16Value
+ Int32Value
+ Int64Value
+ UInt8Value
+ UInt16Value
+ UInt32Value
+ UInt64Value
+ FloatValue
+ DoubleValue
+ BinaryValue
+ StringValue
+ FixedSizeBinaryValue
+ Time32Value
+ Time64Value
+ Date32Value
+ Date64Value
+ TimestampValue
+ DecimalValue
+ DictionaryValue
+ ListValue
+ StructValue
+ UnionValue
diff --git a/docs/source/python/api/cuda.rst b/docs/source/python/api/cuda.rst
new file mode 100644
index 0000000..364f032
--- /dev/null
+++ b/docs/source/python/api/cuda.rst
@@ -0,0 +1,62 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow.cuda
+
+CUDA Integration
+================
+
+.. ifconfig:: not cuda_enabled
+
+ .. error::
+ This documentation was built without CUDA enabled. The CUDA
+ API docs are not available.
+
+.. NOTE We still generate those API docs (with empty docstrings)
+.. when CUDA is disabled and `pyarrow.cuda` mocked (see conf.py).
+.. Otherwise we'd get autodoc warnings, see https://github.com/sphinx-doc/sphinx/issues/4770
+
+CUDA Contexts
+-------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ Context
+
+CUDA Buffers
+------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ CudaBuffer
+ new_host_buffer
+ HostBuffer
+ BufferReader
+ BufferWriter
+
+Serialization and IPC
+---------------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ serialize_record_batch
+ read_record_batch
+ read_message
+ IpcMemHandle
diff --git a/docs/source/python/api/datatypes.rst b/docs/source/python/api/datatypes.rst
new file mode 100644
index 0000000..5ad0204
--- /dev/null
+++ b/docs/source/python/api/datatypes.rst
@@ -0,0 +1,134 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. _api.types:
+.. currentmodule:: pyarrow
+
+Data Types and Schemas
+======================
+
+Factory Functions
+-----------------
+
+These should be used to create Arrow data types and schemas.
+
+.. autosummary::
+ :toctree: ../generated/
+
+ null
+ bool_
+ int8
+ int16
+ int32
+ int64
+ uint8
+ uint16
+ uint32
+ uint64
+ float16
+ float32
+ float64
+ time32
+ time64
+ timestamp
+ date32
+ date64
+ binary
+ string
+ utf8
+ decimal128
+ list_
+ struct
+ dictionary
+ field
+ schema
+ from_numpy_dtype
+
+.. _api.type_classes:
+.. currentmodule:: pyarrow
+
+Type Classes
+------------
+
+Do not instantiate these classes directly. Instead, call one of the factory
+functions above.
+
+.. autosummary::
+ :toctree: ../generated/
+
+ DataType
+ DictionaryType
+ ListType
+ StructType
+ UnionType
+ TimestampType
+ Time32Type
+ Time64Type
+ FixedSizeBinaryType
+ Decimal128Type
+ Field
+ Schema
+
+.. _api.types.checking:
+.. currentmodule:: pyarrow.types
+
+Type Checking
+-------------
+
+These functions are predicates to check whether a :class:`DataType` instance
+represents a given data type (such as ``int32``) or general category
+(such as "is a signed integer").
+
+.. autosummary::
+ :toctree: ../generated/
+
+ is_boolean
+ is_integer
+ is_signed_integer
+ is_unsigned_integer
+ is_int8
+ is_int16
+ is_int32
+ is_int64
+ is_uint8
+ is_uint16
+ is_uint32
+ is_uint64
+ is_floating
+ is_float16
+ is_float32
+ is_float64
+ is_decimal
+ is_list
+ is_struct
+ is_union
+ is_nested
+ is_temporal
+ is_timestamp
+ is_date
+ is_date32
+ is_date64
+ is_time
+ is_time32
+ is_time64
+ is_null
+ is_binary
+ is_unicode
+ is_string
+ is_fixed_size_binary
+ is_map
+ is_dictionary
diff --git a/docs/source/python/api/files.rst b/docs/source/python/api/files.rst
new file mode 100644
index 0000000..106dfde
--- /dev/null
+++ b/docs/source/python/api/files.rst
@@ -0,0 +1,65 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+
+Streams and File Access
+=======================
+
+.. _api.io:
+
+Factory Functions
+-----------------
+
+These factory functions are the recommended way to create a Arrow stream.
+They accept various kinds of sources, such as in-memory buffers or on-disk files.
+
+.. autosummary::
+ :toctree: ../generated/
+
+ input_stream
+ output_stream
+ memory_map
+ create_memory_map
+
+Stream Classes
+--------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ NativeFile
+ OSFile
+ PythonFile
+ BufferReader
+ BufferOutputStream
+ FixedSizeBufferWriter
+ MemoryMappedFile
+ CompressedInputStream
+ CompressedOutputStream
+
+File Systems
+------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ hdfs.connect
+ LocalFileSystem
+
+.. class:: HadoopFileSystem
+ :noindex:
diff --git a/docs/source/python/api/formats.rst b/docs/source/python/api/formats.rst
new file mode 100644
index 0000000..8de30ec
--- /dev/null
+++ b/docs/source/python/api/formats.rst
@@ -0,0 +1,70 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Tabular File Formats
+====================
+
+.. currentmodule:: pyarrow.csv
+
+.. _api.csv:
+
+CSV Files
+---------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ ReadOptions
+ ParseOptions
+ ConvertOptions
+ read_csv
+
+.. _api.feather:
+
+Feather Files
+-------------
+
+.. currentmodule:: pyarrow.feather
+
+.. autosummary::
+ :toctree: ../generated/
+
+ read_feather
+ write_feather
+
+.. currentmodule:: pyarrow
+
+.. _api.parquet:
+
+Parquet Files
+-------------
+
+.. currentmodule:: pyarrow.parquet
+
+.. autosummary::
+ :toctree: ../generated/
+
+ ParquetDataset
+ ParquetFile
+ ParquetWriter
+ read_table
+ read_metadata
+ read_pandas
+ read_schema
+ write_metadata
+ write_table
+ write_to_dataset
diff --git a/docs/source/python/api/ipc.rst b/docs/source/python/api/ipc.rst
new file mode 100644
index 0000000..bd14d30
--- /dev/null
+++ b/docs/source/python/api/ipc.rst
@@ -0,0 +1,59 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+
+.. _api.ipc:
+
+Serialization and IPC
+=====================
+
+Inter-Process Communication
+---------------------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ ipc.open_file
+ ipc.open_stream
+ Message
+ MessageReader
+ RecordBatchFileReader
+ RecordBatchFileWriter
+ RecordBatchStreamReader
+ RecordBatchStreamWriter
+ read_message
+ read_record_batch
+ get_record_batch_size
+ read_tensor
+ write_tensor
+ get_tensor_size
+
+Serialization
+-------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ serialize
+ serialize_to
+ deserialize
+ deserialize_components
+ deserialize_from
+ read_serialized
+ SerializedPyObject
+ SerializationContext
diff --git a/docs/source/python/api/memory.rst b/docs/source/python/api/memory.rst
new file mode 100644
index 0000000..da9156f
--- /dev/null
+++ b/docs/source/python/api/memory.rst
@@ -0,0 +1,68 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+
+.. _api.memory:
+
+Buffers and Memory
+==================
+
+In-Memory Buffers
+-----------------
+
+Factory Functions
+~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+ :toctree: ../generated/
+
+ allocate_buffer
+ py_buffer
+ foreign_buffer
+
+Classes
+~~~~~~~
+
+.. autosummary::
+ :toctree: ../generated/
+
+ Buffer
+ ResizableBuffer
+
+Miscellaneous
+~~~~~~~~~~~~~
+
+.. autosummary::
+ :toctree: ../generated/
+
+ compress
+ decompress
+
+.. _api.memory_pool:
+
+Memory Pools
+------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ MemoryPool
+ default_memory_pool
+ total_allocated_bytes
+ set_memory_pool
+ log_memory_allocations
diff --git a/docs/source/python/api/misc.rst b/docs/source/python/api/misc.rst
new file mode 100644
index 0000000..c13b806
--- /dev/null
+++ b/docs/source/python/api/misc.rst
@@ -0,0 +1,40 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+
+Miscellaneous
+=============
+
+Multi-Threading
+---------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ cpu_count
+ set_cpu_count
+
+Using with C extensions
+-----------------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ get_include
+ get_libraries
+ get_library_dirs
diff --git a/docs/source/python/api/plasma.rst b/docs/source/python/api/plasma.rst
new file mode 100644
index 0000000..8df9e4e
--- /dev/null
+++ b/docs/source/python/api/plasma.rst
@@ -0,0 +1,33 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow.plasma
+
+.. _api.plasma:
+
+Plasma In-Memory Object Store
+=============================
+
+Classes
+-------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ ObjectID
+ PlasmaClient
+ PlasmaBuffer
diff --git a/docs/source/python/api/tables.rst b/docs/source/python/api/tables.rst
new file mode 100644
index 0000000..5a229d2
--- /dev/null
+++ b/docs/source/python/api/tables.rst
@@ -0,0 +1,54 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. currentmodule:: pyarrow
+
+.. _api.table:
+
+Tables and Tensors
+==================
+
+Factory Functions
+-----------------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ column
+ chunked_array
+ concat_tables
+
+Classes
+-------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ ChunkedArray
+ Column
+ RecordBatch
+ Table
+
+.. _api.tensor:
+
+Tensors
+-------
+
+.. autosummary::
+ :toctree: ../generated/
+
+ Tensor
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 0d1c1be..dabcdf1 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -60,7 +60,9 @@ from pyarrow.lib import (null, bool_,
binary, string, utf8, decimal128,
list_, struct, union, dictionary, field,
type_for_alias,
- DataType,
+ DataType, DictionaryType, ListType, StructType,
+ UnionType, TimestampType, Time32Type, Time64Type,
+ FixedSizeBinaryType, Decimal128Type,
Field,
Schema,
schema,
diff --git a/python/pyarrow/_cuda.pyx b/python/pyarrow/_cuda.pyx
index cd57049..c2d95a6 100644
--- a/python/pyarrow/_cuda.pyx
+++ b/python/pyarrow/_cuda.pyx
@@ -23,21 +23,29 @@ cimport cpython as cp
cdef class Context:
- """ CUDA driver context.
+ """
+ CUDA driver context.
"""
- def __cinit__(self, int device_number=0, uintptr_t handle=0):
- """Construct the shared CUDA driver context for a particular device.
+ def __init__(self, *args, **kwargs):
+ """
+ Create a CUDA driver context for a particular device.
+
+ If a CUDA context handle is passed, it is wrapped, otherwise
+ a default CUDA context for the given device is requested.
Parameters
----------
- device_number : int
- Specify the gpu device for which the CUDA driver context is
+ device_number : int (default 0)
+ Specify the GPU device for which the CUDA driver context is
requested.
- handle : int
- Specify handle for a shared context that has been created by
- another library.
+ handle : int, optional
+ Specify CUDA handle for a shared context that has been created
+ by another library.
"""
+ # This method exposed because autodoc doesn't pick __cinit__
+
+ def __cinit__(self, int device_number=0, uintptr_t handle=0):
cdef CCudaDeviceManager* manager
check_status(CCudaDeviceManager.GetInstance(&manager))
cdef int n = manager.num_devices()
@@ -55,13 +63,14 @@ cdef class Context:
@staticmethod
def from_numba(context=None):
- """Create Context instance from a numba CUDA context.
+ """
+ Create a Context instance from a Numba CUDA context.
Parameters
----------
context : {numba.cuda.cudadrv.driver.Context, None}
- Specify numba CUDA context instance. When None, use the
- current numba context.
+ A Numba CUDA context instance.
+ If None, the current Numba context is used.
Returns
-------
@@ -75,7 +84,8 @@ cdef class Context:
handle=context.handle.value)
def to_numba(self):
- """Convert Context to numba CUDA context.
+ """
+ Convert Context to a Numba CUDA context.
Returns
-------
@@ -238,7 +248,7 @@ cdef class Context:
cdef class IpcMemHandle:
- """A container for a CUDA IPC handle.
+ """A serializable container for a CUDA IPC handle.
"""
cdef void init(self, shared_ptr[CCudaIpcMemHandle]& h):
self.handle = h
@@ -285,14 +295,10 @@ cdef class IpcMemHandle:
cdef class CudaBuffer(Buffer):
"""An Arrow buffer with data located in a GPU device.
- To create a CudaBuffer instance, use
-
- <Context instance>.device_buffer(data=<object>, offset=<offset>,
- size=<nbytes>)
-
- The memory allocated in CudaBuffer instance is freed when the
- instance is deleted.
+ To create a CudaBuffer instance, use Context.device_buffer().
+ The memory allocated in a CudaBuffer is freed when the buffer object
+ is deleted.
"""
def __init__(self):
@@ -529,7 +535,7 @@ cdef class CudaBuffer(Buffer):
After calling this function, this device memory will not be
freed when the CudaBuffer is destructed.
- Results
+ Returns
-------
ipc_handle : IpcMemHandle
The exported IPC handle
@@ -774,9 +780,9 @@ def serialize_record_batch(object batch, object ctx):
Parameters
----------
batch : RecordBatch
- Specify record batch to write
+ Record batch to write
ctx : Context
- Specify context to allocate device memory from
+ CUDA Context to allocate device memory from
Returns
-------
@@ -797,14 +803,14 @@ def read_message(object source, pool=None):
Parameters
----------
source : {CudaBuffer, cuda.BufferReader}
- Specify device buffer or reader of device buffer.
- pool : {MemoryPool, None}
- Specify pool to allocate CPU memory for the metadata
+ Device buffer or reader of device buffer.
+ pool : MemoryPool (optional)
+ Pool to allocate CPU memory for the metadata
Returns
-------
message : Message
- the deserialized message, body still on device
+ The deserialized message, body still on device
"""
cdef:
Message result = Message.__new__(Message)
@@ -824,16 +830,16 @@ def read_record_batch(object buffer, object schema, pool=None):
Parameters
----------
buffer :
- Specify device buffer containing the complete IPC message
+ Device buffer containing the complete IPC message
schema : Schema
- Specify schema for the record batch
- pool : {MemoryPool, None}
- Specify pool to use for allocating space for the metadata
+ The schema for the record batch
+ pool : MemoryPool (optional)
+ Pool to allocate metadata from
Returns
-------
batch : RecordBatch
- reconstructed record batch, with device pointers
+ Reconstructed record batch, with device pointers
"""
cdef shared_ptr[CSchema] schema_ = pyarrow_unwrap_schema(schema)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 54d0e92..41a3b97 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -392,6 +392,9 @@ cdef class _PandasConvertible:
cdef class Array(_PandasConvertible):
+ """
+ The base class for all Arrow arrays.
+ """
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use one of "
@@ -616,11 +619,18 @@ cdef class Array(_PandasConvertible):
def isnull(self):
raise NotImplemented
- def __getitem__(self, key):
- if PySlice_Check(key):
- return _normalize_slice(self, key)
+ def __getitem__(self, index):
+ """
+ Return the value at the given index.
- return self.getitem(_normalize_index(key, self.length()))
+ Returns
+ -------
+ value : Scalar
+ """
+ if PySlice_Check(index):
+ return _normalize_slice(self, index)
+
+ return self.getitem(_normalize_index(index, self.length()))
cdef getitem(self, int64_t i):
return box_scalar(self.type, self.sp_array, i)
@@ -736,6 +746,9 @@ cdef class Array(_PandasConvertible):
cdef class Tensor:
+ """
+ A n-dimensional array a.k.a Tensor.
+ """
def __init__(self):
raise TypeError("Do not call Tensor's constructor directly, use one "
@@ -842,98 +855,147 @@ cdef wrap_array_output(PyObject* output):
cdef class NullArray(Array):
- pass
+ """
+ Concrete class for Arrow arrays of null data type.
+ """
cdef class BooleanArray(Array):
- pass
+ """
+ Concrete class for Arrow arrays of boolean data type.
+ """
cdef class NumericArray(Array):
- pass
+ """
+ A base class for Arrow numeric arrays.
+ """
cdef class IntegerArray(NumericArray):
- pass
+ """
+ A base class for Arrow integer arrays.
+ """
cdef class FloatingPointArray(NumericArray):
- pass
+ """
+ A base class for Arrow floating-point arrays.
+ """
cdef class Int8Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of int8 data type.
+ """
cdef class UInt8Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of uint8 data type.
+ """
cdef class Int16Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of int16 data type.
+ """
cdef class UInt16Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of uint16 data type.
+ """
cdef class Int32Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of int32 data type.
+ """
cdef class UInt32Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of uint32 data type.
+ """
cdef class Int64Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of int64 data type.
+ """
cdef class UInt64Array(IntegerArray):
- pass
+ """
+ Concrete class for Arrow arrays of uint64 data type.
+ """
cdef class Date32Array(NumericArray):
- pass
+ """
+ Concrete class for Arrow arrays of date32 data type.
+ """
cdef class Date64Array(NumericArray):
- pass
+ """
+ Concrete class for Arrow arrays of date64 data type.
+ """
cdef class TimestampArray(NumericArray):
- pass
+ """
+ Concrete class for Arrow arrays of timestamp data type.
+ """
cdef class Time32Array(NumericArray):
- pass
+ """
+ Concrete class for Arrow arrays of time32 data type.
+ """
cdef class Time64Array(NumericArray):
- pass
+ """
+ Concrete class for Arrow arrays of time64 data type.
+ """
cdef class HalfFloatArray(FloatingPointArray):
- pass
+ """
+ Concrete class for Arrow arrays of float16 data type.
+ """
cdef class FloatArray(FloatingPointArray):
- pass
+ """
+ Concrete class for Arrow arrays of float32 data type.
+ """
cdef class DoubleArray(FloatingPointArray):
- pass
+ """
+ Concrete class for Arrow arrays of float64 data type.
+ """
cdef class FixedSizeBinaryArray(Array):
- pass
+ """
+ Concrete class for Arrow arrays of a fixed-size binary data type.
+ """
cdef class Decimal128Array(FixedSizeBinaryArray):
- pass
+ """
+ Concrete class for Arrow arrays of decimal128 data type.
+ """
cdef class ListArray(Array):
+ """
+ Concrete class for Arrow arrays of a list data type.
+ """
@staticmethod
def from_arrays(offsets, values, MemoryPool pool=None):
@@ -975,6 +1037,9 @@ cdef class ListArray(Array):
cdef class UnionArray(Array):
+ """
+ Concrete class for Arrow arrays of a Union data type.
+ """
@staticmethod
def from_dense(Array types, Array value_offsets, list children):
@@ -1028,6 +1093,9 @@ cdef class UnionArray(Array):
cdef class StringArray(Array):
+ """
+ Concrete class for Arrow arrays of string (or utf8) data type.
+ """
@staticmethod
def from_buffers(int length, Buffer value_offsets, Buffer data,
@@ -1066,10 +1134,15 @@ cdef class StringArray(Array):
cdef class BinaryArray(Array):
- pass
+ """
+ Concrete class for Arrow arrays of variable-sized binary data type.
+ """
cdef class DictionaryArray(Array):
+ """
+ Concrete class for dictionary-encoded Arrow arrays.
+ """
def dictionary_encode(self):
return self
@@ -1163,6 +1236,9 @@ cdef class DictionaryArray(Array):
cdef class StructArray(Array):
+ """
+ Concrete class for Arrow arrays of a struct data type.
+ """
def field(self, index):
"""
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 5212274..8edffbe 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -44,6 +44,16 @@ cdef extern from "Python.h":
cdef class NativeFile:
+ """
+ The base class for all Arrow streams.
+
+ Streams are either readable, writable, or both.
+ They optionally support seeking.
+
+ While this class exposes methods to read or write data from Python, the
+ primary intent of using a Arrow stream is to pass it to other Arrow
+ facilities that will make use of it, such as Arrow IPC routines.
+ """
def __cinit__(self):
self.own_file = False
@@ -559,6 +569,16 @@ BufferedIOBase.register(NativeFile)
cdef class PythonFile(NativeFile):
+ """
+ A stream backed by a Python file object.
+
+ This class allows using Python file objects with arbitrary Arrow
+ functions, including functions written in another language than Python.
+
+ As a downside, there is a non-zero redirection cost in translating
+ Arrow stream calls to Python method calls. Furthermore, Python's
+ Global Interpreter Lock may limit parallelism in some situations.
+ """
cdef:
object handle
@@ -628,7 +648,9 @@ cdef class PythonFile(NativeFile):
cdef class MemoryMappedFile(NativeFile):
"""
- Supports 'r', 'r+w', 'w' modes
+ A stream that represents a memory-mapped file.
+
+ Supports 'r', 'r+', 'w' modes.
"""
cdef:
shared_ptr[CMemoryMappedFile] handle
@@ -704,7 +726,9 @@ def memory_map(path, mode='r'):
Parameters
----------
path : string
- mode : {'r', 'w'}, default 'r'
+ mode : {'r', 'r+', 'w'}, default 'r'
+ Whether the file is opened for reading ('r+'), writing ('w')
+ or both ('r+').
Returns
-------
@@ -717,13 +741,14 @@ def memory_map(path, mode='r'):
def create_memory_map(path, size):
"""
- Create memory map at indicated path of the given size, return open
- writable file object
+ Create a file of the given size and memory-map it.
Parameters
----------
path : string
+ The file path to create, on the local filesystem.
size : int
+ The file size to create.
Returns
-------
@@ -734,7 +759,7 @@ def create_memory_map(path, size):
cdef class OSFile(NativeFile):
"""
- Supports 'r', 'w' modes
+ A stream backed by a regular file descriptor.
"""
cdef:
object path
@@ -774,6 +799,9 @@ cdef class OSFile(NativeFile):
cdef class FixedSizeBufferWriter(NativeFile):
+ """
+ A stream writing to a Arrow buffer.
+ """
def __cinit__(self, Buffer buffer):
self.output_stream.reset(new CFixedSizeBufferWriter(buffer.buffer))
@@ -800,6 +828,12 @@ cdef class FixedSizeBufferWriter(NativeFile):
cdef class Buffer:
+ """
+ The base class for all Arrow buffers.
+
+ A buffer represents a contiguous memory area. Many buffers will own
+ their memory, though not all of them do.
+ """
def __cinit__(self):
pass
@@ -818,14 +852,23 @@ cdef class Buffer:
@property
def size(self):
+ """
+ The buffer size in bytes.
+ """
return self.buffer.get().size()
@property
def address(self):
+ """
+ The buffer's address, as an integer.
+ """
return <uintptr_t> self.buffer.get().data()
@property
def is_mutable(self):
+ """
+ Whether the buffer is mutable.
+ """
return self.buffer.get().is_mutable()
@property
@@ -848,7 +891,9 @@ cdef class Buffer:
def slice(self, offset=0, length=None):
"""
- Compute slice of this buffer
+ Slice this buffer. Memory is not copied.
+
+ You can also use the Python slice notation ``buffer[start:stop]``.
Parameters
----------
@@ -861,6 +906,7 @@ cdef class Buffer:
Returns
-------
sliced : Buffer
+ A logical view over this buffer.
"""
cdef shared_ptr[CBuffer] result
@@ -876,7 +922,7 @@ cdef class Buffer:
def equals(self, Buffer other):
"""
- Determine if two buffers contain exactly the same data
+ Determine if two buffers contain exactly the same data.
Parameters
----------
@@ -904,6 +950,9 @@ cdef class Buffer:
return py_buffer, (self.to_pybytes(),)
def to_pybytes(self):
+ """
+ Return this buffer as a Python bytes object. Memory is copied.
+ """
return cp.PyBytes_FromStringAndSize(
<const char*>self.buffer.get().data(),
self.buffer.get().size())
@@ -950,21 +999,25 @@ cdef class Buffer:
cdef class ResizableBuffer(Buffer):
+ """
+ A base class for buffers that can be resized.
+ """
cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer):
self.init(<shared_ptr[CBuffer]> buffer)
def resize(self, int64_t new_size, shrink_to_fit=False):
"""
- Resize buffer to indicated size
+ Resize buffer to indicated size.
Parameters
----------
- new_size : int64_t
+ new_size : int
New size of buffer (padding may be added internally)
shrink_to_fit : boolean, default False
- If new_size is less than the current size, shrink internal
- capacity, otherwise leave at current capacity
+ If this is true, the buffer is shrunk when new_size is less
+ than the current size.
+ If this is false, the buffer is never shrunk.
"""
cdef c_bool c_shrink_to_fit = shrink_to_fit
with nogil:
@@ -982,15 +1035,17 @@ cdef shared_ptr[CResizableBuffer] _allocate_buffer(CMemoryPool* pool):
def allocate_buffer(int64_t size, MemoryPool memory_pool=None,
resizable=False):
"""
- Allocate mutable fixed-size buffer
+ Allocate a mutable buffer.
Parameters
----------
size : int
Number of bytes to allocate (plus internal padding)
memory_pool : MemoryPool, optional
- Uses default memory pool if not provided
+ The pool to allocate memory from.
+ If not given, the default memory pool is used.
resizable : boolean, default False
+ If true, the returned buffer is resizable.
Returns
-------
@@ -1305,8 +1360,7 @@ def _detect_compression(path):
def compress(object buf, codec='lz4', asbytes=False, memory_pool=None):
"""
- Compress pyarrow.Buffer or Python object supporting the buffer (memoryview)
- protocol
+ Compress data from buffer-like object.
Parameters
----------
@@ -1367,7 +1421,7 @@ def compress(object buf, codec='lz4', asbytes=False, memory_pool=None):
def decompress(object buf, decompressed_size=None, codec='lz4',
asbytes=False, memory_pool=None):
"""
- Decompress data from buffer-like object
+ Decompress data from buffer-like object.
Parameters
----------
diff --git a/python/pyarrow/memory.pxi b/python/pyarrow/memory.pxi
index 7fa6d79..047e70d 100644
--- a/python/pyarrow/memory.pxi
+++ b/python/pyarrow/memory.pxi
@@ -21,6 +21,12 @@
cdef class MemoryPool:
+ """
+ Base class for memory allocation.
+
+ Besides tracking its number of allocated bytes, a memory pool also
+ takes care of the required 64-byte alignment for Arrow data.
+ """
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, "
@@ -68,8 +74,9 @@ cdef class LoggingMemoryPool(MemoryPool):
cdef class ProxyMemoryPool(MemoryPool):
"""
- Derived MemoryPool class that tracks the number of bytes and
- maximum memory allocated through its direct calls.
+ Memory pool implementation that tracks the number of bytes and
+ maximum memory allocated through its direct calls, while redirecting
+ to another memory pool.
"""
cdef:
unique_ptr[CProxyMemoryPool] proxy_pool
@@ -81,6 +88,9 @@ cdef class ProxyMemoryPool(MemoryPool):
def default_memory_pool():
+ """
+ Return the process-global memory pool.
+ """
cdef:
MemoryPool pool = MemoryPool.__new__(MemoryPool)
pool.init(c_get_memory_pool())
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index fd3f580..e2c1481 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -19,10 +19,17 @@
_NULL = NA = None
+cdef class Scalar:
+ """
+ The base class for all array elements.
+ """
+
+
cdef class NullType(Scalar):
"""
- Null (NA) value singleton
+ Singleton for null array elements.
"""
+ # TODO rename this NullValue?
def __cinit__(self):
global NA
if NA is not None:
@@ -44,6 +51,9 @@ _NULL = NA = NullType()
cdef class ArrayValue(Scalar):
+ """
+ The base class for non-null array elements.
+ """
def __init__(self):
raise TypeError("Do not call {}'s constructor directly, use array "
@@ -85,6 +95,9 @@ cdef class ArrayValue(Scalar):
cdef class BooleanValue(ArrayValue):
+ """
+ Concrete class for boolean array elements.
+ """
def as_py(self):
"""
@@ -95,6 +108,9 @@ cdef class BooleanValue(ArrayValue):
cdef class Int8Value(ArrayValue):
+ """
+ Concrete class for int8 array elements.
+ """
def as_py(self):
"""
@@ -105,6 +121,9 @@ cdef class Int8Value(ArrayValue):
cdef class UInt8Value(ArrayValue):
+ """
+ Concrete class for uint8 array elements.
+ """
def as_py(self):
"""
@@ -115,6 +134,9 @@ cdef class UInt8Value(ArrayValue):
cdef class Int16Value(ArrayValue):
+ """
+ Concrete class for int16 array elements.
+ """
def as_py(self):
"""
@@ -125,6 +147,9 @@ cdef class Int16Value(ArrayValue):
cdef class UInt16Value(ArrayValue):
+ """
+ Concrete class for uint16 array elements.
+ """
def as_py(self):
"""
@@ -135,6 +160,9 @@ cdef class UInt16Value(ArrayValue):
cdef class Int32Value(ArrayValue):
+ """
+ Concrete class for int32 array elements.
+ """
def as_py(self):
"""
@@ -145,6 +173,9 @@ cdef class Int32Value(ArrayValue):
cdef class UInt32Value(ArrayValue):
+ """
+ Concrete class for uint32 array elements.
+ """
def as_py(self):
"""
@@ -155,6 +186,9 @@ cdef class UInt32Value(ArrayValue):
cdef class Int64Value(ArrayValue):
+ """
+ Concrete class for int64 array elements.
+ """
def as_py(self):
"""
@@ -165,6 +199,9 @@ cdef class Int64Value(ArrayValue):
cdef class UInt64Value(ArrayValue):
+ """
+ Concrete class for uint64 array elements.
+ """
def as_py(self):
"""
@@ -175,6 +212,9 @@ cdef class UInt64Value(ArrayValue):
cdef class Date32Value(ArrayValue):
+ """
+ Concrete class for date32 array elements.
+ """
def as_py(self):
"""
@@ -188,6 +228,9 @@ cdef class Date32Value(ArrayValue):
cdef class Date64Value(ArrayValue):
+ """
+ Concrete class for date64 array elements.
+ """
def as_py(self):
"""
@@ -199,6 +242,9 @@ cdef class Date64Value(ArrayValue):
cdef class Time32Value(ArrayValue):
+ """
+ Concrete class for time32 array elements.
+ """
def as_py(self):
"""
@@ -217,6 +263,9 @@ cdef class Time32Value(ArrayValue):
cdef class Time64Value(ArrayValue):
+ """
+ Concrete class for time64 array elements.
+ """
def as_py(self):
"""
@@ -269,6 +318,9 @@ else:
cdef class TimestampValue(ArrayValue):
+ """
+ Concrete class for timestamp array elements.
+ """
@property
def value(self):
@@ -301,6 +353,9 @@ cdef class TimestampValue(ArrayValue):
cdef class HalfFloatValue(ArrayValue):
+ """
+ Concrete class for float16 array elements.
+ """
def as_py(self):
"""
@@ -311,6 +366,9 @@ cdef class HalfFloatValue(ArrayValue):
cdef class FloatValue(ArrayValue):
+ """
+ Concrete class for float32 array elements.
+ """
def as_py(self):
"""
@@ -321,6 +379,9 @@ cdef class FloatValue(ArrayValue):
cdef class DoubleValue(ArrayValue):
+ """
+ Concrete class for float64 array elements.
+ """
def as_py(self):
"""
@@ -331,6 +392,9 @@ cdef class DoubleValue(ArrayValue):
cdef class DecimalValue(ArrayValue):
+ """
+ Concrete class for decimal128 array elements.
+ """
def as_py(self):
"""
@@ -343,6 +407,9 @@ cdef class DecimalValue(ArrayValue):
cdef class StringValue(ArrayValue):
+ """
+ Concrete class for string (utf8) array elements.
+ """
def as_py(self):
"""
@@ -353,6 +420,9 @@ cdef class StringValue(ArrayValue):
cdef class BinaryValue(ArrayValue):
+ """
+ Concrete class for variable-sized binary array elements.
+ """
def as_py(self):
"""
@@ -380,14 +450,26 @@ cdef class BinaryValue(ArrayValue):
cdef class ListValue(ArrayValue):
+ """
+ Concrete class for list array elements.
+ """
def __len__(self):
+ """
+ Return the number of values.
+ """
return self.length()
def __getitem__(self, i):
+ """
+ Return the value at the given index.
+ """
return self.getitem(_normalize_index(i, self.length()))
def __iter__(self):
+ """
+ Iterate over this element's values.
+ """
for i in range(len(self)):
yield self.getitem(i)
raise StopIteration
@@ -419,6 +501,9 @@ cdef class ListValue(ArrayValue):
cdef class UnionValue(ArrayValue):
+ """
+ Concrete class for union array elements.
+ """
cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
self.sp_array = sp_array
@@ -436,11 +521,16 @@ cdef class UnionValue(ArrayValue):
def as_py(self):
"""
Return this value as a Python object.
+
+ The exact type depends on the underlying union member.
"""
return self.getitem(self.index).as_py()
cdef class FixedSizeBinaryValue(ArrayValue):
+ """
+ Concrete class for fixed-size binary array elements.
+ """
def as_py(self):
"""
@@ -459,12 +549,18 @@ cdef class FixedSizeBinaryValue(ArrayValue):
cdef class StructValue(ArrayValue):
+ """
+ Concrete class for struct array elements.
+ """
cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
self.sp_array = sp_array
self.ap = <CStructArray*> sp_array.get()
def __getitem__(self, key):
+ """
+ Return the child value for the given field name.
+ """
cdef:
CStructType* type
int index
@@ -496,17 +592,23 @@ cdef class StructValue(ArrayValue):
cdef class DictionaryValue(ArrayValue):
+ """
+ Concrete class for dictionary-encoded array elements.
+ """
def as_py(self):
"""
Return this value as a Python object.
+
+ The exact type depends on the dictionary value type.
"""
return self.dictionary_value.as_py()
@property
def index_value(self):
"""
- Return this value's underlying index as a Int32Value.
+ Return this value's underlying index as a ArrayValue of the right
+ signed integer type.
"""
cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
indices = pyarrow_wrap_array(darr.indices())
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 29b2a1e..7c6aec3 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -88,7 +88,9 @@ ctypedef CFixedWidthType* _CFixedWidthTypePtr
cdef class DataType:
"""
- Base type for Apache Arrow data type instances. Wraps C++ arrow::DataType
+ Base class of all Arrow data types.
+
+ Each data type is an *instance* of this class.
"""
def __cinit__(self):
pass
@@ -162,7 +164,7 @@ cdef class DataType:
def to_pandas_dtype(self):
"""
- Return the NumPy dtype that would be used for storing this
+ Return the equivalent NumPy / Pandas dtype.
"""
cdef Type type_id = self.type.id()
if type_id in _pandas_type_map:
@@ -172,6 +174,9 @@ cdef class DataType:
cdef class DictionaryType(DataType):
+ """
+ Concrete class for dictionary data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -182,18 +187,31 @@ cdef class DictionaryType(DataType):
@property
def ordered(self):
+ """
+ Whether the dictionary is ordered, i.e. whether the ordering of values
+ in the dictionary is important.
+ """
return self.dict_type.ordered()
@property
def index_type(self):
+ """
+ The data type of dictionary indices (a signed integer type).
+ """
return pyarrow_wrap_data_type(self.dict_type.index_type())
@property
def dictionary(self):
+ """
+ The dictionary array, mapping dictionary indices to values.
+ """
return pyarrow_wrap_array(self.dict_type.dictionary())
cdef class ListType(DataType):
+ """
+ Concrete class for list data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -204,10 +222,16 @@ cdef class ListType(DataType):
@property
def value_type(self):
+ """
+ The data type of list values.
+ """
return pyarrow_wrap_data_type(self.list_type.value_type())
cdef class StructType(DataType):
+ """
+ Concrete class for struct data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -215,13 +239,13 @@ cdef class StructType(DataType):
cdef Field field(self, int i):
"""
- Alias for child(i)
+ Return a child field by its index.
"""
return self.child(i)
cdef Field field_by_name(self, name):
"""
- Access a child field by its name rather than the column index.
+ Return a child field by its name rather than its index.
"""
cdef shared_ptr[CField] field
@@ -232,13 +256,22 @@ cdef class StructType(DataType):
return pyarrow_wrap_field(field)
def __len__(self):
+ """
+ Like num_children().
+ """
return self.type.num_children()
def __iter__(self):
+ """
+ Iterate over struct fields, in order.
+ """
for i in range(len(self)):
yield self[i]
def __getitem__(self, i):
+ """
+ Return the struct field with the given index or name.
+ """
if isinstance(i, six.string_types):
return self.field_by_name(i)
elif isinstance(i, six.integer_types):
@@ -251,20 +284,32 @@ cdef class StructType(DataType):
@property
def num_children(self):
+ """
+ The number of struct fields.
+ """
return self.type.num_children()
cdef class UnionType(DataType):
+ """
+ Concrete class for struct data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@property
def num_children(self):
+ """
+ The number of union members.
+ """
return self.type.num_children()
@property
def mode(self):
+ """
+ The mode of the union ("dense" or "sparse").
+ """
cdef CUnionType* type = <CUnionType*> self.sp_type.get()
cdef int mode = type.mode()
if mode == _UnionMode_DENSE:
@@ -274,13 +319,22 @@ cdef class UnionType(DataType):
assert 0
def __len__(self):
+ """
+ Like num_children()
+ """
return self.type.num_children()
def __iter__(self):
+ """
+ Iterate over union members, in order.
+ """
for i in range(len(self)):
yield self[i]
def __getitem__(self, i):
+ """
+ Return a child member by its index.
+ """
return self.child(i)
def __reduce__(self):
@@ -288,6 +342,9 @@ cdef class UnionType(DataType):
cdef class TimestampType(DataType):
+ """
+ Concrete class for timestamp data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -295,10 +352,16 @@ cdef class TimestampType(DataType):
@property
def unit(self):
+ """
+ The timestamp unit ('s', 'ms', 'us' or 'ns').
+ """
return timeunit_to_string(self.ts_type.unit())
@property
def tz(self):
+ """
+ The timestamp time zone, if any, or None.
+ """
if self.ts_type.timezone().size() > 0:
return frombytes(self.ts_type.timezone())
else:
@@ -306,7 +369,7 @@ cdef class TimestampType(DataType):
def to_pandas_dtype(self):
"""
- Return the NumPy dtype that would be used for storing this
+ Return the equivalent NumPy / Pandas dtype.
"""
if self.tz is None:
return _pandas_type_map[_Type_TIMESTAMP]
@@ -319,6 +382,9 @@ cdef class TimestampType(DataType):
cdef class Time32Type(DataType):
+ """
+ Concrete class for time32 data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -326,10 +392,16 @@ cdef class Time32Type(DataType):
@property
def unit(self):
+ """
+ The time unit ('s', 'ms', 'us' or 'ns').
+ """
return timeunit_to_string(self.time_type.unit())
cdef class Time64Type(DataType):
+ """
+ Concrete class for time64 data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -337,10 +409,16 @@ cdef class Time64Type(DataType):
@property
def unit(self):
+ """
+ The time unit ('s', 'ms', 'us' or 'ns').
+ """
return timeunit_to_string(self.time_type.unit())
cdef class FixedSizeBinaryType(DataType):
+ """
+ Concrete class for fixed-size binary data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
DataType.init(self, type)
@@ -352,10 +430,16 @@ cdef class FixedSizeBinaryType(DataType):
@property
def byte_width(self):
+ """
+ The binary size in bytes.
+ """
return self.fixed_size_binary_type.byte_width()
cdef class Decimal128Type(FixedSizeBinaryType):
+ """
+ Concrete class for decimal128 data types.
+ """
cdef void init(self, const shared_ptr[CDataType]& type):
FixedSizeBinaryType.init(self, type)
@@ -366,17 +450,22 @@ cdef class Decimal128Type(FixedSizeBinaryType):
@property
def precision(self):
+ """
+ The decimal precision, in number of decimal digits (an integer).
+ """
return self.decimal128_type.precision()
@property
def scale(self):
+ """
+ The decimal scale (an integer).
+ """
return self.decimal128_type.scale()
cdef class Field:
"""
- Represents a named field, with a data type, nullability, and optional
- metadata
+ A named field, with a data type, nullability, and optional metadata.
Notes
-----