You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/03/19 16:59:41 UTC
[arrow] branch master updated: ARROW-4637: [Python] Conditionally
import pandas symbols if they are used. Do not require pandas as a test
dependency
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 286bf7c ARROW-4637: [Python] Conditionally import pandas symbols if they are used. Do not require pandas as a test dependency
286bf7c is described below
commit 286bf7c9d343cb972691c32ea8128390aed39119
Author: Wes McKinney <we...@apache.org>
AuthorDate: Tue Mar 19 11:59:30 2019 -0500
ARROW-4637: [Python] Conditionally import pandas symbols if they are used. Do not require pandas as a test dependency
Warning: hold your nose for this one =) I think this can be made cleaner, but I just wanted to triage all the problems and make sure we don't introduce a hard dependency on pandas again
Also resolves ARROW-4794: make pandas an optional dependency
Author: Wes McKinney <we...@apache.org>
Closes #3893 from wesm/ARROW-4637 and squashes the following commits:
3c353b6b <Wes McKinney> do not override orc global mark in test_orc.py
59fdf8ab <Wes McKinney> Tweak pyarrow._orc import to see if it fixes MSVC
747c8937 <Wes McKinney> Address Python 2.7 unicode interaction issue with Cython
4d94b2e1 <Wes McKinney> Add benchmark from ARROW-4629
f7bf7741 <Wes McKinney> Cythonize pandas API shim for better performance
4a2549aa <Wes McKinney> Remove TF testing from travis_script_manylinux1.sh
5061c6e1 <Wes McKinney> Do not require pandas to run unit tests
385cfe55 <Wes McKinney> Finish pandas API shim; do not eagerly import pandas, add to CI
804587a0 <Wes McKinney> add import test script
bb0240e2 <Wes McKinney> Begin to refactor to make references to pandas more centralized and lazy
---
ci/travis_script_manylinux.sh | 3 +-
ci/travis_script_python.sh | 3 +
docs/source/developers/python.rst | 6 +-
python/benchmarks/convert_pandas.py | 14 ++
python/pyarrow/array.pxi | 32 ++--
python/pyarrow/compat.py | 33 ----
python/pyarrow/feather.py | 20 +--
python/pyarrow/lib.pyx | 7 +-
python/pyarrow/orc.py | 2 +-
python/pyarrow/pandas-shim.pxi | 172 +++++++++++++++++++
python/pyarrow/pandas_compat.py | 68 ++++----
python/pyarrow/scalar.pxi | 74 ++++----
python/pyarrow/serialization.pxi | 14 +-
python/pyarrow/serialization.py | 6 +-
python/pyarrow/table.pxi | 29 ++--
python/pyarrow/tests/conftest.py | 10 ++
python/pyarrow/tests/test_array.py | 119 -------------
python/pyarrow/tests/test_convert_builtin.py | 45 +++--
python/pyarrow/tests/test_feather.py | 16 +-
python/pyarrow/tests/test_gandiva.py | 18 +-
python/pyarrow/tests/test_hdfs.py | 28 +--
python/pyarrow/tests/test_io.py | 7 +-
python/pyarrow/tests/test_ipc.py | 17 +-
python/pyarrow/tests/test_orc.py | 11 +-
.../{test_convert_pandas.py => test_pandas.py} | 189 ++++++++++++++++++++-
python/pyarrow/tests/test_parquet.py | 91 +++++++++-
python/pyarrow/tests/test_plasma.py | 3 +-
python/pyarrow/tests/test_scalars.py | 5 +-
python/pyarrow/tests/test_schema.py | 35 ++--
python/pyarrow/tests/test_table.py | 59 ++-----
python/pyarrow/types.pxi | 6 +-
python/requirements-test.txt | 1 +
python/scripts/test_imports.py | 21 +++
33 files changed, 756 insertions(+), 408 deletions(-)
diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh
index fe6c108..6fbaa0f 100755
--- a/ci/travis_script_manylinux.sh
+++ b/ci/travis_script_manylinux.sh
@@ -59,10 +59,9 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
conda activate $CONDA_ENV_DIR
# install the produced wheels
- pip install tensorflow
pip install dist/*.whl
- # Test optional dependencies and the presence of tensorflow
+ # Test optional dependencies
python check_imports.py
# Install test dependencies and run pyarrow tests
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 5e4aaac..1e049c4 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -166,6 +166,9 @@ python -c "import pyarrow.parquet"
python -c "import pyarrow.plasma"
python -c "import pyarrow.orc"
+# Ensure we do eagerly import pandas (or other expensive imports)
+python < scripts/test_imports.py
+
echo "PLASMA_VALGRIND: $PLASMA_VALGRIND"
# Set up huge pages for plasma test
diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst
index 7a0b7f7..0ebe8f1 100644
--- a/docs/source/developers/python.rst
+++ b/docs/source/developers/python.rst
@@ -360,8 +360,6 @@ Now, we build and install Arrow C++ libraries
set ARROW_HOME=C:\thirdparty
cmake -G "Visual Studio 14 2015 Win64" ^
-DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
- -DCMAKE_BUILD_TYPE=Release ^
- -DARROW_BUILD_TESTS=on ^
-DARROW_CXXFLAGS="/WX /MP" ^
-DARROW_GANDIVA=on ^
-DARROW_PARQUET=on ^
@@ -380,7 +378,9 @@ Now, we can build pyarrow:
.. code-block:: shell
cd python
- python setup.py build_ext --inplace --with-parquet
+ set PYARROW_WITH_GANDIVA=1
+ set PYARROW_WITH_PARQUET=1
+ python setup.py build_ext --inplace
Then run the unit tests with:
diff --git a/python/benchmarks/convert_pandas.py b/python/benchmarks/convert_pandas.py
index 57a0518..9cf6bde 100644
--- a/python/benchmarks/convert_pandas.py
+++ b/python/benchmarks/convert_pandas.py
@@ -105,3 +105,17 @@ class SerializeDeserializePandas(object):
def time_deserialize_pandas(self):
pa.deserialize_pandas(self.serialized)
+
+
+class TableFromPandasMicroperformance(object):
+ # ARROW-4629
+
+ def setup(self):
+ ser = pd.Series(range(10000))
+ df = pd.DataFrame({col: ser.copy(deep=True) for col in range(100)})
+ # Simulate a real dataset by converting some columns to strings
+ self.df = df.astype({col: str for col in range(50)})
+
+ def time_Table_from_pandas(self):
+ for _ in range(50):
+ pa.Table.from_pandas(self.df, nthreads=1)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 324c140..4824db6 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -15,11 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-from pyarrow.compat import HAVE_PANDAS
-
-if HAVE_PANDAS:
- import pyarrow.pandas_compat as pdcompat
-
cdef _sequence_to_array(object sequence, object mask, object size,
DataType type, CMemoryPool* pool, c_bool from_pandas):
@@ -46,12 +41,10 @@ cdef _sequence_to_array(object sequence, object mask, object size,
return pyarrow_wrap_chunked_array(out)
-cdef _is_array_like(obj):
- try:
- import pandas
- return isinstance(obj, (np.ndarray, pd.Series, pd.Index, Categorical))
- except ImportError:
- return isinstance(obj, np.ndarray)
+cdef inline _is_array_like(obj):
+ if isinstance(obj, np.ndarray):
+ return True
+ return pandas_api._have_pandas_internal() and pandas_api.is_array_like(obj)
def _ndarray_to_arrow_type(object values, DataType type):
@@ -163,15 +156,15 @@ def array(object obj, type=None, mask=None, size=None, bint from_pandas=False,
values = get_series_values(obj)
- if isinstance(values, Categorical):
+ if pandas_api.is_categorical(values):
return DictionaryArray.from_arrays(
values.codes, values.categories.values,
mask=mask, ordered=values.ordered,
from_pandas=True, safe=safe,
memory_pool=memory_pool)
else:
- if HAVE_PANDAS:
- values, type = pdcompat.get_datetimetz_type(
+ if pandas_api.have_pandas:
+ values, type = pandas_api.compat.get_datetimetz_type(
values, obj.dtype, type)
return _ndarray_to_array(values, mask, type, from_pandas, safe,
pool)
@@ -852,9 +845,10 @@ cdef wrap_array_output(PyObject* output):
cdef object obj = PyObject_to_object(output)
if isinstance(obj, dict):
- return Categorical(obj['indices'],
- categories=obj['dictionary'],
- ordered=obj['ordered'], fastpath=True)
+ return pandas_api.categorical_type(obj['indices'],
+ categories=obj['dictionary'],
+ ordered=obj['ordered'],
+ fastpath=True)
else:
return obj
@@ -1385,11 +1379,11 @@ cdef dict _array_classes = {
cdef object get_series_values(object obj):
- if isinstance(obj, PandasSeries):
+ if pandas_api.is_series(obj):
result = obj.values
elif isinstance(obj, np.ndarray):
result = obj
else:
- result = PandasSeries(obj).values
+ result = pandas_api.make_series(obj).values
return result
diff --git a/python/pyarrow/compat.py b/python/pyarrow/compat.py
index e41de73..7e11b2c 100644
--- a/python/pyarrow/compat.py
+++ b/python/pyarrow/compat.py
@@ -17,7 +17,6 @@
# flake8: noqa
-from distutils.version import LooseVersion
import itertools
import numpy as np
@@ -31,38 +30,6 @@ import socket
PY26 = sys.version_info[:2] == (2, 6)
PY2 = sys.version_info[0] == 2
-try:
- import pandas as pd
- pdver = LooseVersion(pd.__version__)
- if pdver >= '0.20.0':
- from pandas.api.types import DatetimeTZDtype
- pdapi = pd.api.types
- elif pdver >= '0.19.0':
- from pandas.types.dtypes import DatetimeTZDtype
- pdapi = pd.api.types
- else:
- from pandas.types.dtypes import DatetimeTZDtype
- pdapi = pd.core.common
-
- PandasSeries = pd.Series
- Categorical = pd.Categorical
- HAVE_PANDAS = True
-except:
- HAVE_PANDAS = False
- class DatetimeTZDtype(object):
- pass
-
- class ClassPlaceholder(object):
-
- def __init__(self, *args, **kwargs):
- raise NotImplementedError
-
- class PandasSeries(ClassPlaceholder):
- pass
-
- class Categorical(ClassPlaceholder):
- pass
-
if PY26:
import unittest2 as unittest
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 3713c1f..93bcada 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -15,31 +15,25 @@
# specific language governing permissions and limitations
# under the License.
-from distutils.version import LooseVersion
import os
import six
-import pandas as pd
-from pyarrow.compat import pdapi
+from pyarrow.pandas_compat import _pandas_api # noqa
from pyarrow.lib import FeatherError # noqa
from pyarrow.lib import Table, concat_tables
import pyarrow.lib as ext
-try:
- infer_dtype = pdapi.infer_dtype
-except AttributeError:
- infer_dtype = pd.lib.infer_dtype
-
-
-if LooseVersion(pd.__version__) < '0.17.0':
- raise ImportError("feather requires pandas >= 0.17.0")
+def _check_pandas_version():
+ if _pandas_api.loose_version < '0.17.0':
+ raise ImportError("feather requires pandas >= 0.17.0")
class FeatherReader(ext.FeatherReader):
def __init__(self, source):
+ _check_pandas_version()
self.source = source
self.open(source)
@@ -80,12 +74,13 @@ def check_chunked_overflow(col):
class FeatherWriter(object):
def __init__(self, dest):
+ _check_pandas_version()
self.dest = dest
self.writer = ext.FeatherWriter()
self.writer.open(dest)
def write(self, df):
- if isinstance(df, pd.SparseDataFrame):
+ if isinstance(df, _pandas_api.pd.SparseDataFrame):
df = df.to_dense()
if not df.columns.is_unique:
@@ -114,6 +109,7 @@ class FeatherDataset(object):
Check that individual file schemas are all the same / compatible
"""
def __init__(self, path_or_paths, validate_schema=True):
+ _check_pandas_version()
self.paths = path_or_paths
self.validate_schema = validate_schema
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 3fe879a..b7c2a93 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -19,13 +19,15 @@
# distutils: language = c++
# cython: embedsignature = True
+from collections import OrderedDict
import datetime
import decimal as _pydecimal
+import json
import multiprocessing
import numpy as np
import os
import six
-from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
+from pyarrow.compat import frombytes, tobytes
from cython.operator cimport dereference as deref
from pyarrow.includes.libarrow cimport *
@@ -90,6 +92,9 @@ Type_MAP = _Type_MAP
UnionMode_SPARSE = _UnionMode_SPARSE
UnionMode_DENSE = _UnionMode_DENSE
+# pandas API shim
+include "pandas-shim.pxi"
+
# Exception types
include "error.pxi"
diff --git a/python/pyarrow/orc.py b/python/pyarrow/orc.py
index 22451d5..39111e5 100644
--- a/python/pyarrow/orc.py
+++ b/python/pyarrow/orc.py
@@ -18,9 +18,9 @@
from itertools import count
from numbers import Integral
-from pyarrow import _orc
from pyarrow import types
from pyarrow.lib import Schema
+import pyarrow._orc as _orc
def _is_map(typ):
diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi
new file mode 100644
index 0000000..b0c5329
--- /dev/null
+++ b/python/pyarrow/pandas-shim.pxi
@@ -0,0 +1,172 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pandas lazy-loading API shim that reduces API call and import overhead
+
+cdef class _PandasAPIShim(object):
+ """
+ Lazy pandas importer that isolates usages of pandas APIs and avoids
+ importing pandas until it's actually needed
+ """
+ cdef:
+ bint _tried_importing_pandas
+ bint _have_pandas
+
+ cdef readonly:
+ object _loose_version, _version
+ object _pd, _types_api, _compat_module
+ object _data_frame, _index, _series, _categorical_type
+ object _datetimetz_type
+ object _array_like_types
+
+ def __init__(self):
+ self._tried_importing_pandas = False
+ self._have_pandas = 0
+
+ cdef _import_pandas(self, bint raise_):
+ try:
+ import pandas as pd
+ import pyarrow.pandas_compat as pdcompat
+ except ImportError:
+ self._have_pandas = False
+ if raise_:
+ raise
+ else:
+ return
+
+ self._pd = pd
+ self._compat_module = pdcompat
+ self._data_frame = pd.DataFrame
+ self._index = pd.Index
+ self._categorical_type = pd.Categorical
+ self._series = pd.Series
+
+ self._array_like_types = (self._series, self._index,
+ self._categorical_type)
+
+ from distutils.version import LooseVersion
+ self._loose_version = LooseVersion(pd.__version__)
+ if self._loose_version >= LooseVersion('0.20.0'):
+ from pandas.api.types import DatetimeTZDtype
+ self._types_api = pd.api.types
+ elif self._loose_version >= LooseVersion('0.19.0'):
+ from pandas.types.dtypes import DatetimeTZDtype
+ self._types_api = pd.api.types
+ else:
+ from pandas.types.dtypes import DatetimeTZDtype
+ self._types_api = pd.core.common
+
+ self._datetimetz_type = DatetimeTZDtype
+ self._have_pandas = True
+
+ cdef inline _check_import(self, bint raise_=True):
+ if self._tried_importing_pandas:
+ if not self._have_pandas and raise_:
+ self._import_pandas(raise_)
+ return
+
+ self._tried_importing_pandas = True
+ self._import_pandas(raise_)
+
+ def make_series(self, *args, **kwargs):
+ self._check_import()
+ return self._series(*args, **kwargs)
+
+ def data_frame(self, *args, **kwargs):
+ self._check_import()
+ return self._data_frame(*args, **kwargs)
+
+ cdef inline bint _have_pandas_internal(self):
+ if not self._tried_importing_pandas:
+ self._check_import(raise_=False)
+ return self._have_pandas
+
+ @property
+ def have_pandas(self):
+ return self._have_pandas_internal()
+
+ @property
+ def compat(self):
+ self._check_import()
+ return self._compat_module
+
+ @property
+ def pd(self):
+ self._check_import()
+ return self._pd
+
+ cpdef infer_dtype(self, obj):
+ self._check_import()
+ try:
+ return self._types_api.infer_dtype(obj, skipna=False)
+ except AttributeError:
+ return self._pd.lib.infer_dtype(obj)
+
+ @property
+ def loose_version(self):
+ self._check_import()
+ return self._loose_version
+
+ @property
+ def version(self):
+ self._check_import()
+ return self._version
+
+ @property
+ def categorical_type(self):
+ self._check_import()
+ return self._categorical_type
+
+ @property
+ def datetimetz_type(self):
+ return self._datetimetz_type
+
+ cpdef is_array_like(self, obj):
+ self._check_import()
+ return isinstance(obj, self._array_like_types)
+
+ cpdef is_categorical(self, obj):
+ if self._have_pandas_internal():
+ return isinstance(obj, self._categorical_type)
+ else:
+ return False
+
+ cpdef is_datetimetz(self, obj):
+ if self._have_pandas_internal():
+ return isinstance(obj, self._datetimetz_type)
+ else:
+ return False
+
+ cpdef is_data_frame(self, obj):
+ if self._have_pandas_internal():
+ return isinstance(obj, self._data_frame)
+ else:
+ return False
+
+ cpdef is_series(self, obj):
+ if self._have_pandas_internal():
+ return isinstance(obj, self._series)
+ else:
+ return False
+
+ def assert_frame_equal(self, *args, **kwargs):
+ self._check_import()
+ return self._pd.util.testing.assert_frame_equal
+
+
+cdef _PandasAPIShim pandas_api = _PandasAPIShim()
+_pandas_api = pandas_api
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 90a0ad6..873a4da 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -21,22 +21,13 @@ import json
import operator
import re
-import pandas.core.internals as _int
import numpy as np
-import pandas as pd
import six
import pyarrow as pa
-from pyarrow.compat import (builtin_pickle, DatetimeTZDtype, # noqa
- PY2, zip_longest)
-
-
-def infer_dtype(column):
- try:
- return pd.api.types.infer_dtype(column, skipna=False)
- except AttributeError:
- return pd.lib.infer_dtype(column)
+from pyarrow.lib import _pandas_api
+from pyarrow.compat import (builtin_pickle, PY2, zip_longest) # noqa
_logical_type_map = {}
@@ -115,8 +106,7 @@ def get_logical_type_from_numpy(pandas_collection):
# See https://github.com/pandas-dev/pandas/issues/24739
if str(pandas_collection.dtype) == 'datetime64[ns]':
return 'datetime64[ns]'
- result = infer_dtype(pandas_collection)
-
+ result = _pandas_api.infer_dtype(pandas_collection)
if result == 'string':
return 'bytes' if PY2 else 'unicode'
return result
@@ -249,7 +239,7 @@ def construct_metadata(df, column_names, index_levels, index_descriptors,
'library': 'pyarrow',
'version': pa.__version__
},
- 'pandas_version': pd.__version__
+ 'pandas_version': _pandas_api.version
}).encode('utf8')
}
@@ -359,7 +349,7 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
index_column_names = []
for i, index_level in enumerate(index_levels):
name = _index_level_name(index_level, i, column_names)
- if isinstance(index_level, pd.RangeIndex):
+ if isinstance(index_level, _pandas_api.pd.RangeIndex):
descr = _get_range_index_descriptor(index_level)
else:
columns_to_convert.append(index_level)
@@ -430,7 +420,7 @@ def dataframe_to_types(df, preserve_index, columns=None):
# If pandas knows type, skip conversion
for c in columns_to_convert:
values = c.values
- if isinstance(values, pd.Categorical):
+ if _pandas_api.is_categorical(values):
type_ = pa.array(c, from_pandas=True).type
else:
values, type_ = get_datetimetz_type(values, c.dtype, None)
@@ -498,7 +488,7 @@ def get_datetimetz_type(values, dtype, type_):
if values.dtype.type != np.datetime64:
return values, type_
- if isinstance(dtype, DatetimeTZDtype) and type_ is None:
+ if _pandas_api.is_datetimetz(dtype) and type_ is None:
# If no user type passed, construct a tz-aware timestamp type
tz = dtype.tz
unit = dtype.unit
@@ -515,6 +505,7 @@ def get_datetimetz_type(values, dtype, type_):
def dataframe_to_serialized_dict(frame):
+ import pandas.core.internals as _int
block_manager = frame._data
blocks = []
@@ -554,14 +545,16 @@ def dataframe_to_serialized_dict(frame):
def serialized_dict_to_dataframe(data):
+ import pandas.core.internals as _int
reconstructed_blocks = [_reconstruct_block(block)
for block in data['blocks']]
block_mgr = _int.BlockManager(reconstructed_blocks, data['axes'])
- return pd.DataFrame(block_mgr)
+ return _pandas_api.data_frame(block_mgr)
def _reconstruct_block(item):
+ import pandas.core.internals as _int
# Construct the individual blocks converting dictionary types to pandas
# categorical types and Timestamps-with-timezones types to the proper
# pandas Blocks
@@ -569,13 +562,13 @@ def _reconstruct_block(item):
block_arr = item['block']
placement = item['placement']
if 'dictionary' in item:
- cat = pd.Categorical.from_codes(block_arr,
- categories=item['dictionary'],
- ordered=item['ordered'])
+ cat = _pandas_api.categorical_type.from_codes(
+ block_arr, categories=item['dictionary'],
+ ordered=item['ordered'])
block = _int.make_block(cat, placement=placement,
klass=_int.CategoricalBlock)
elif 'timezone' in item:
- dtype = _make_datetimetz(item['timezone'])
+ dtype = make_datetimetz(item['timezone'])
block = _int.make_block(block_arr, placement=placement,
klass=_int.DatetimeTZBlock,
dtype=dtype)
@@ -588,9 +581,9 @@ def _reconstruct_block(item):
return block
-def _make_datetimetz(tz):
+def make_datetimetz(tz):
tz = pa.lib.string_to_tzinfo(tz)
- return DatetimeTZDtype('ns', tz=tz)
+ return _pandas_api.datetimetz_type('ns', tz=tz)
# ----------------------------------------------------------------------
@@ -599,6 +592,8 @@ def _make_datetimetz(tz):
def table_to_blockmanager(options, table, categories=None,
ignore_metadata=False):
+ from pandas.core.internals import BlockManager
+
all_columns = []
column_indexes = []
pandas_metadata = table.schema.pandas_metadata
@@ -611,7 +606,7 @@ def table_to_blockmanager(options, table, categories=None,
table, index = _reconstruct_index(table, index_descriptors,
all_columns)
else:
- index = pd.RangeIndex(table.num_rows)
+ index = _pandas_api.pd.RangeIndex(table.num_rows)
_check_data_column_metadata_consistency(all_columns)
blocks = _table_to_blocks(options, table, pa.default_memory_pool(),
@@ -619,7 +614,7 @@ def table_to_blockmanager(options, table, categories=None,
columns = _deserialize_column_index(table, all_columns, column_indexes)
axes = [columns, index]
- return _int.BlockManager(blocks, axes)
+ return BlockManager(blocks, axes)
def _check_data_column_metadata_consistency(all_columns):
@@ -655,9 +650,9 @@ def _deserialize_column_index(block_table, all_columns, column_indexes):
# Construct the base index
if not columns_values:
- columns = pd.Index(columns_values)
+ columns = _pandas_api.pd.Index(columns_values)
else:
- columns = pd.MultiIndex.from_tuples(
+ columns = _pandas_api.pd.MultiIndex.from_tuples(
list(map(to_pair, columns_values)),
names=[col_index['name'] for col_index in column_indexes] or None,
)
@@ -709,8 +704,10 @@ def _reconstruct_index(table, index_descriptors, all_columns):
continue
elif descr['kind'] == 'range':
index_name = descr['name']
- index_level = pd.RangeIndex(descr['start'], descr['stop'],
- step=descr['step'], name=index_name)
+ index_level = _pandas_api.pd.RangeIndex(descr['start'],
+ descr['stop'],
+ step=descr['step'],
+ name=index_name)
if len(index_level) != len(table):
# Possibly the result of munged metadata
continue
@@ -720,6 +717,8 @@ def _reconstruct_index(table, index_descriptors, all_columns):
index_arrays.append(index_level)
index_names.append(index_name)
+ pd = _pandas_api.pd
+
# Reconstruct the row index
if len(index_arrays) > 1:
index = pd.MultiIndex.from_arrays(index_arrays, names=index_names)
@@ -753,7 +752,9 @@ def _extract_index_level(table, result_table, descr,
# non-writeable arrays when calling MultiIndex.from_arrays
values = values.copy()
- if isinstance(col_pandas.dtype, DatetimeTZDtype):
+ pd = _pandas_api.pd
+
+ if _pandas_api.is_datetimetz(col_pandas.dtype):
index_level = (pd.Series(values).dt.tz_localize('utc')
.dt.tz_convert(col_pandas.dtype.tz))
else:
@@ -849,7 +850,7 @@ def _pandas_type_to_numpy_type(pandas_type):
def _get_multiindex_codes(mi):
# compat for pandas < 0.24 (MI labels renamed to codes).
- if isinstance(mi, pd.MultiIndex):
+ if isinstance(mi, _pandas_api.pd.MultiIndex):
return mi.codes if hasattr(mi, 'codes') else mi.labels
else:
return None
@@ -877,7 +878,7 @@ def _reconstruct_columns_from_metadata(columns, column_indexes):
-----
* Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
"""
-
+ pd = _pandas_api.pd
# Get levels and labels, and provide sane defaults if the index has a
# single level to avoid if/else spaghetti.
levels = getattr(columns, 'levels', None) or [columns]
@@ -923,6 +924,7 @@ def _table_to_blocks(options, block_table, memory_pool, categories):
def _flatten_single_level_multiindex(index):
+ pd = _pandas_api.pd
if isinstance(index, pd.MultiIndex) and index.nlevels == 1:
levels, = index.levels
labels, = _get_multiindex_codes(index)
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index e2c1481..1e62593 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -284,37 +284,47 @@ cdef class Time64Value(ArrayValue):
datetime.timedelta(microseconds=val / 1000)).time()
-cdef dict DATETIME_CONVERSION_FUNCTIONS
-
-try:
- import pandas as pd
-except ImportError:
- DATETIME_CONVERSION_FUNCTIONS = {
- TimeUnit_SECOND: lambda x, tzinfo: (
- datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
- ),
- TimeUnit_MILLI: lambda x, tzinfo: (
- datetime.datetime.utcfromtimestamp(x / 1e3).replace(tzinfo=tzinfo)
- ),
- TimeUnit_MICRO: lambda x, tzinfo: (
- datetime.datetime.utcfromtimestamp(x / 1e6).replace(tzinfo=tzinfo)
- ),
- }
-else:
- DATETIME_CONVERSION_FUNCTIONS = {
- TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
- x * 1000000000, tz=tzinfo, unit='ns',
- ),
- TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
- x * 1000000, tz=tzinfo, unit='ns',
- ),
- TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
- x * 1000, tz=tzinfo, unit='ns',
- ),
- TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- }
+cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
+cdef c_bool _datetime_conversion_initialized = False
+
+cdef _datetime_conversion_functions():
+ global _datetime_conversion_initialized
+ if _datetime_conversion_initialized:
+ return _DATETIME_CONVERSION_FUNCTIONS
+
+ try:
+ import pandas as pd
+ except ImportError:
+ _DATETIME_CONVERSION_FUNCTIONS.update({
+ TimeUnit_SECOND: lambda x, tzinfo: (
+ datetime.datetime.utcfromtimestamp(x).replace(tzinfo=tzinfo)
+ ),
+ TimeUnit_MILLI: lambda x, tzinfo: (
+ (datetime.datetime.utcfromtimestamp(x / 1e3)
+ .replace(tzinfo=tzinfo))
+ ),
+ TimeUnit_MICRO: lambda x, tzinfo: (
+ (datetime.datetime.utcfromtimestamp(x / 1e6)
+ .replace(tzinfo=tzinfo))
+ ),
+ })
+ else:
+ _DATETIME_CONVERSION_FUNCTIONS.update({
+ TimeUnit_SECOND: lambda x, tzinfo: pd.Timestamp(
+ x * 1000000000, tz=tzinfo, unit='ns',
+ ),
+ TimeUnit_MILLI: lambda x, tzinfo: pd.Timestamp(
+ x * 1000000, tz=tzinfo, unit='ns',
+ ),
+ TimeUnit_MICRO: lambda x, tzinfo: pd.Timestamp(
+ x * 1000, tz=tzinfo, unit='ns',
+ ),
+ TimeUnit_NANO: lambda x, tzinfo: pd.Timestamp(
+ x, tz=tzinfo, unit='ns',
+ )
+ })
+ _datetime_conversion_initialized = True
+ return _DATETIME_CONVERSION_FUNCTIONS
cdef class TimestampValue(ArrayValue):
@@ -344,7 +354,7 @@ cdef class TimestampValue(ArrayValue):
tzinfo = None
try:
- converter = DATETIME_CONVERSION_FUNCTIONS[dtype.unit()]
+ converter = _datetime_conversion_functions()[dtype.unit()]
except KeyError:
raise ValueError(
'Cannot convert nanosecond timestamps without pandas'
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index 7fae394..bce4cfd 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -217,6 +217,16 @@ cdef class SerializationContext:
_default_serialization_context = SerializationContext()
+_default_context_initialized = False
+
+
+def _get_default_context():
+ global _default_context_initialized
+ from pyarrow.serialization import register_default_serialization_handlers
+ if not _default_context_initialized:
+ register_default_serialization_handlers(_default_serialization_context)
+ _default_context_initialized = True
+ return _default_serialization_context
cdef class SerializedPyObject:
@@ -256,7 +266,7 @@ cdef class SerializedPyObject:
cdef PyObject* result
if context is None:
- context = _default_serialization_context
+ context = _get_default_context()
with nogil:
check_status(DeserializeObject(context, self.data,
@@ -339,7 +349,7 @@ def serialize(object value, SerializationContext context=None):
wrapped_value = [value]
if context is None:
- context = _default_serialization_context
+ context = _get_default_context()
with nogil:
check_status(SerializeObject(context, wrapped_value, &serialized.data))
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index 96785de..3eed68d 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -23,8 +23,7 @@ import numpy as np
import pyarrow
from pyarrow.compat import builtin_pickle
-from pyarrow.lib import (SerializationContext, _default_serialization_context,
- py_buffer)
+from pyarrow.lib import SerializationContext, py_buffer
try:
import cloudpickle
@@ -326,6 +325,3 @@ def default_serialization_context():
context = SerializationContext()
register_default_serialization_handlers(context)
return context
-
-
-register_default_serialization_handlers(_default_serialization_context)
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index f0e7de9..53678f2 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -15,19 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-import json
-
-from collections import OrderedDict
-
-try:
- import pandas as pd
-except ImportError:
- # The pure-Python based API works without a pandas installation
- pass
-else:
- import pyarrow.pandas_compat as pdcompat
-
-
cdef class ChunkedArray(_PandasConvertible):
"""
Array backed via one or more memory chunks.
@@ -473,7 +460,7 @@ cdef class Column(_PandasConvertible):
def _to_pandas(self, options, **kwargs):
values = self.data._to_pandas(options)
- result = pd.Series(values, name=self.name)
+ result = pandas_api.make_series(values, name=self.name)
if isinstance(self.type, TimestampType):
tz = self.type.tz
@@ -867,7 +854,8 @@ cdef class RecordBatch(_PandasConvertible):
-------
pyarrow.RecordBatch
"""
- names, arrays, metadata = pdcompat.dataframe_to_arrays(
+ from pyarrow.pandas_compat import dataframe_to_arrays
+ names, arrays, metadata = dataframe_to_arrays(
df, schema, preserve_index, nthreads=nthreads, columns=columns
)
return cls.from_arrays(arrays, names, metadata)
@@ -1147,7 +1135,8 @@ cdef class Table(_PandasConvertible):
>>> pa.Table.from_pandas(df)
<pyarrow.lib.Table object at 0x7f05d1fb1b40>
"""
- names, arrays, metadata = pdcompat.dataframe_to_arrays(
+ from pyarrow.pandas_compat import dataframe_to_arrays
+ names, arrays, metadata = dataframe_to_arrays(
df,
schema=schema,
preserve_index=preserve_index,
@@ -1303,9 +1292,11 @@ cdef class Table(_PandasConvertible):
return result
def _to_pandas(self, options, categories=None, ignore_metadata=False):
- mgr = pdcompat.table_to_blockmanager(options, self, categories,
- ignore_metadata=ignore_metadata)
- return pd.DataFrame(mgr)
+ from pyarrow.pandas_compat import table_to_blockmanager
+ mgr = table_to_blockmanager(
+ options, self, categories,
+ ignore_metadata=ignore_metadata)
+ return pandas_api.data_frame(mgr)
def to_pydict(self):
"""
diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py
index b02fd3e..8a6304e 100644
--- a/python/pyarrow/tests/conftest.py
+++ b/python/pyarrow/tests/conftest.py
@@ -43,6 +43,7 @@ groups = [
'hdfs',
'large_memory',
'orc',
+ 'pandas',
'parquet',
'plasma',
's3',
@@ -56,6 +57,7 @@ defaults = {
'hdfs': False,
'large_memory': False,
'orc': False,
+ 'pandas': False,
'parquet': False,
'plasma': False,
's3': False,
@@ -74,6 +76,14 @@ try:
except ImportError:
pass
+
+try:
+ import pandas # noqa
+ defaults['pandas'] = True
+except ImportError:
+ pass
+
+
try:
import pyarrow.parquet # noqa
defaults['parquet'] = True
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index cef9822..8c0143c 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -27,8 +27,6 @@ import struct
import sys
import numpy as np
-import pandas as pd
-import pandas.util.testing as tm
try:
import pickle5
except ImportError:
@@ -36,7 +34,6 @@ except ImportError:
import pyarrow as pa
import pyarrow.tests.strategies as past
-from pyarrow.pandas_compat import get_logical_type
def test_total_bytes_allocated():
@@ -376,26 +373,6 @@ def test_dictionary_from_arrays_boundscheck():
pa.DictionaryArray.from_arrays(indices2, dictionary, safe=False)
-def test_dictionary_with_pandas():
- indices = np.repeat([0, 1, 2], 2)
- dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
- mask = np.array([False, False, True, False, False, False])
-
- d1 = pa.DictionaryArray.from_arrays(indices, dictionary)
- d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)
-
- pandas1 = d1.to_pandas()
- ex_pandas1 = pd.Categorical.from_codes(indices, categories=dictionary)
-
- tm.assert_series_equal(pd.Series(pandas1), pd.Series(ex_pandas1))
-
- pandas2 = d2.to_pandas()
- ex_pandas2 = pd.Categorical.from_codes(np.where(mask, -1, indices),
- categories=dictionary)
-
- tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))
-
-
def test_list_from_arrays():
offsets_arr = np.array([0, 2, 5, 8], dtype='i4')
offsets = pa.array(offsets_arr, type='int32')
@@ -619,52 +596,6 @@ def test_safe_cast_nan_to_int_raises():
arr.cast(pa.int64(), safe=True)
-def test_cast_timestamp_unit():
- # ARROW-1680
- val = datetime.datetime.now()
- s = pd.Series([val])
- s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
-
- us_with_tz = pa.timestamp('us', tz='America/New_York')
-
- arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
-
- # ARROW-1906
- assert arr.type == us_with_tz
-
- arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
-
- assert arr[0].as_py() == s_nyc[0]
- assert arr2[0].as_py() == s[0]
-
- # Disallow truncation
- arr = pa.array([123123], type='int64').cast(pa.timestamp('ms'))
- expected = pa.array([123], type='int64').cast(pa.timestamp('s'))
-
- target = pa.timestamp('s')
- with pytest.raises(ValueError):
- arr.cast(target)
-
- result = arr.cast(target, safe=False)
- assert result.equals(expected)
-
- # ARROW-1949
- series = pd.Series([pd.Timestamp(1), pd.Timestamp(10), pd.Timestamp(1000)])
- expected = pa.array([0, 0, 1], type=pa.timestamp('us'))
-
- with pytest.raises(ValueError):
- pa.array(series, type=pa.timestamp('us'))
-
- with pytest.raises(ValueError):
- pa.Array.from_pandas(series, type=pa.timestamp('us'))
-
- result = pa.Array.from_pandas(series, type=pa.timestamp('us'), safe=False)
- assert result.equals(expected)
-
- result = pa.array(series, type=pa.timestamp('us'), safe=False)
- assert result.equals(expected)
-
-
def test_cast_signed_to_unsigned():
safe_cases = [
(np.array([0, 1, 2, 3], dtype='i1'), pa.uint8(),
@@ -891,39 +822,6 @@ def test_to_numpy_roundtrip(narr):
np.testing.assert_array_equal(narr[2:6], arr[2:6].to_numpy())
-@pytest.mark.parametrize(
- ('type', 'expected'),
- [
- (pa.null(), 'empty'),
- (pa.bool_(), 'bool'),
- (pa.int8(), 'int8'),
- (pa.int16(), 'int16'),
- (pa.int32(), 'int32'),
- (pa.int64(), 'int64'),
- (pa.uint8(), 'uint8'),
- (pa.uint16(), 'uint16'),
- (pa.uint32(), 'uint32'),
- (pa.uint64(), 'uint64'),
- (pa.float16(), 'float16'),
- (pa.float32(), 'float32'),
- (pa.float64(), 'float64'),
- (pa.date32(), 'date'),
- (pa.date64(), 'date'),
- (pa.binary(), 'bytes'),
- (pa.binary(length=4), 'bytes'),
- (pa.string(), 'unicode'),
- (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
- (pa.decimal128(18, 3), 'decimal'),
- (pa.timestamp('ms'), 'datetime'),
- (pa.timestamp('us', 'UTC'), 'datetimetz'),
- (pa.time32('s'), 'time'),
- (pa.time64('us'), 'time')
- ]
-)
-def test_logical_type(type, expected):
- assert get_logical_type(type) == expected
-
-
def test_array_uint64_from_py_over_range():
arr = pa.array([2 ** 63], type=pa.uint64())
expected = pa.array(np.array([2 ** 63], dtype='u8'))
@@ -1013,23 +911,6 @@ def test_array_from_timestamp_with_generic_unit():
pa.array([n, x, y])
-def test_array_from_py_float32():
- data = [[1.2, 3.4], [9.0, 42.0]]
-
- t = pa.float32()
-
- arr1 = pa.array(data[0], type=t)
- arr2 = pa.array(data, type=pa.list_(t))
-
- expected1 = np.array(data[0], dtype=np.float32)
- expected2 = pd.Series([np.array(data[0], dtype=np.float32),
- np.array(data[1], dtype=np.float32)])
-
- assert arr1.type == t
- assert arr1.equals(pa.array(expected1))
- assert arr2.equals(pa.array(expected2))
-
-
def test_array_from_numpy_ascii():
arr = np.array(['abcde', 'abc', ''], dtype='|S5')
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index a3d29be..17ae1e3 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -19,6 +19,7 @@
import pytest
from pyarrow.compat import unittest, u # noqa
+from pyarrow.pandas_compat import _pandas_api # noqa
import pyarrow as pa
import collections
@@ -643,7 +644,6 @@ def test_sequence_timestamp_with_unit():
s = pa.timestamp('s')
ms = pa.timestamp('ms')
us = pa.timestamp('us')
- ns = pa.timestamp('ns')
arr_s = pa.array(data, type=s)
assert len(arr_s) == 1
@@ -663,16 +663,16 @@ def test_sequence_timestamp_with_unit():
assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
23, 34, 123456)
- arr_ns = pa.array(data, type=ns)
- assert len(arr_ns) == 1
- assert arr_ns.type == ns
- assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
- 23, 34, 123456)
+
+class MyDate(datetime.date):
+ pass
+
+
+class MyDatetime(datetime.datetime):
+ pass
def test_datetime_subclassing():
- class MyDate(datetime.date):
- pass
data = [
MyDate(2007, 7, 13),
]
@@ -682,9 +682,6 @@ def test_datetime_subclassing():
assert arr_date.type == date_type
assert arr_date[0].as_py() == datetime.date(2007, 7, 13)
- class MyDatetime(datetime.datetime):
- pass
-
data = [
MyDatetime(2007, 7, 13, 1, 23, 34, 123456),
]
@@ -692,7 +689,6 @@ def test_datetime_subclassing():
s = pa.timestamp('s')
ms = pa.timestamp('ms')
us = pa.timestamp('us')
- ns = pa.timestamp('ns')
arr_s = pa.array(data, type=s)
assert len(arr_s) == 1
@@ -712,14 +708,29 @@ def test_datetime_subclassing():
assert arr_us[0].as_py() == datetime.datetime(2007, 7, 13, 1,
23, 34, 123456)
- arr_ns = pa.array(data, type=ns)
- assert len(arr_ns) == 1
- assert arr_ns.type == ns
- assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
- 23, 34, 123456)
+@pytest.mark.xfail(not _pandas_api.have_pandas,
+ reason="pandas required for nanosecond conversion")
+def test_sequence_timestamp_nanoseconds():
+ inputs = [
+ [datetime.datetime(2007, 7, 13, 1, 23, 34, 123456)],
+ [MyDatetime(2007, 7, 13, 1, 23, 34, 123456)]
+ ]
+
+ for data in inputs:
+ ns = pa.timestamp('ns')
+ arr_ns = pa.array(data, type=ns)
+ assert len(arr_ns) == 1
+ assert arr_ns.type == ns
+ assert arr_ns[0].as_py() == datetime.datetime(2007, 7, 13, 1,
+ 23, 34, 123456)
+
+@pytest.mark.pandas
def test_sequence_timestamp_from_int_with_unit():
+ # TODO(wesm): This test might be rewritten to assert the actual behavior
+ # when pandas is not installed
+
data = [1]
s = pa.timestamp('s')
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index d144f98..f26f7ca 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -25,15 +25,25 @@ import pytest
from numpy.testing import assert_array_equal
import numpy as np
-from pandas.util.testing import assert_frame_equal
-import pandas as pd
-
import pyarrow as pa
from pyarrow.feather import (read_feather, write_feather,
read_table, FeatherReader, FeatherDataset)
from pyarrow.lib import FeatherWriter
+try:
+ from pandas.util.testing import assert_frame_equal
+ import pandas as pd
+except ImportError:
+ pass
+
+
+# TODO(wesm): The Feather tests currently are tangled with pandas
+# dependency. We should isolate the pandas-depending parts and mark those with
+# pytest.mark.pandas
+pytestmark = pytest.mark.pandas
+
+
def random_path(prefix='feather_'):
return tempfile.mktemp(prefix=prefix)
diff --git a/python/pyarrow/tests/test_gandiva.py b/python/pyarrow/tests/test_gandiva.py
index fa99b99..34b3cb7 100644
--- a/python/pyarrow/tests/test_gandiva.py
+++ b/python/pyarrow/tests/test_gandiva.py
@@ -19,7 +19,6 @@ import datetime
import pytest
import pyarrow as pa
-import pandas as pd
@pytest.mark.gandiva
@@ -60,8 +59,8 @@ def test_tree_exp_builder():
def test_table():
import pyarrow.gandiva as gandiva
- df = pd.DataFrame({"a": [1.0, 2.0], "b": [3.0, 4.0]})
- table = pa.Table.from_pandas(df)
+ table = pa.Table.from_arrays([pa.array([1.0, 2.0]), pa.array([3.0, 4.0])],
+ ['a', 'b'])
builder = gandiva.TreeExprBuilder()
node_a = builder.make_field(table.schema.field_by_name("a"))
@@ -79,7 +78,7 @@ def test_table():
# RecordBatches
r, = projector.evaluate(table.to_batches()[0])
- e = pa.Array.from_pandas(df["a"] + df["b"])
+ e = pa.array([4.0, 6.0])
assert r.equals(e)
@@ -87,8 +86,8 @@ def test_table():
def test_filter():
import pyarrow.gandiva as gandiva
- df = pd.DataFrame({"a": [1.0 * i for i in range(10000)]})
- table = pa.Table.from_pandas(df)
+ table = pa.Table.from_arrays([pa.array([1.0 * i for i in range(10000)])],
+ ['a'])
builder = gandiva.TreeExprBuilder()
node_a = builder.make_field(table.schema.field_by_name("a"))
@@ -216,9 +215,10 @@ def test_in_expr_todo():
def test_boolean():
import pyarrow.gandiva as gandiva
- df = pd.DataFrame({"a": [1., 31., 46., 3., 57., 44., 22.],
- "b": [5., 45., 36., 73., 83., 23., 76.]})
- table = pa.Table.from_pandas(df)
+ table = pa.Table.from_arrays([pa.array([1., 31., 46., 3., 57., 44., 22.]),
+ pa.array([5., 45., 36., 73.,
+ 83., 23., 76.])],
+ ['a', 'b'])
builder = gandiva.TreeExprBuilder()
node_a = builder.make_field(table.schema.field_by_name("a"))
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 1af841f..f2430e5 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -20,7 +20,6 @@ import pickle
import pytest
import random
import unittest
-import pandas.util.testing as pdt
from io import BytesIO
from os.path import join as pjoin
@@ -30,6 +29,7 @@ import pyarrow as pa
import pyarrow.tests.test_parquet as test_parquet
from pyarrow.compat import guid
+from pyarrow.pandas_compat import _pandas_api
# ----------------------------------------------------------------------
@@ -297,6 +297,7 @@ class HdfsTestCases(object):
expected = pa.concat_tables(test_data)
return expected
+ @pytest.mark.pandas
@pytest.mark.parquet
def test_read_multiple_parquet_files(self):
@@ -307,10 +308,12 @@ class HdfsTestCases(object):
expected = self._write_multiple_hdfs_pq_files(tmpdir)
result = self.hdfs.read_parquet(tmpdir)
- pdt.assert_frame_equal(result.to_pandas()
- .sort_values(by='index').reset_index(drop=True),
- expected.to_pandas())
+ _pandas_api.assert_frame_equal(result.to_pandas()
+ .sort_values(by='index')
+ .reset_index(drop=True),
+ expected.to_pandas())
+ @pytest.mark.pandas
@pytest.mark.parquet
def test_read_multiple_parquet_files_with_uri(self):
import pyarrow.parquet as pq
@@ -323,10 +326,12 @@ class HdfsTestCases(object):
path = _get_hdfs_uri(tmpdir)
result = pq.read_table(path)
- pdt.assert_frame_equal(result.to_pandas()
- .sort_values(by='index').reset_index(drop=True),
- expected.to_pandas())
+ _pandas_api.assert_frame_equal(result.to_pandas()
+ .sort_values(by='index')
+ .reset_index(drop=True),
+ expected.to_pandas())
+ @pytest.mark.pandas
@pytest.mark.parquet
def test_read_write_parquet_files_with_uri(self):
import pyarrow.parquet as pq
@@ -345,7 +350,7 @@ class HdfsTestCases(object):
result = pq.read_table(path, filesystem=self.hdfs).to_pandas()
- pdt.assert_frame_equal(result, df)
+ _pandas_api.assert_frame_equal(result, df)
@pytest.mark.parquet
def test_read_common_metadata_files(self):
@@ -409,10 +414,13 @@ def _get_hdfs_uri(path):
return uri
+@pytest.mark.pandas
@pytest.mark.parquet
@pytest.mark.fastparquet
@pytest.mark.parametrize('client', ['libhdfs', 'libhdfs3'])
def test_fastparquet_read_with_hdfs(client):
+ from pandas.util.testing import assert_frame_equal, makeDataFrame
+
try:
import snappy # noqa
except ImportError:
@@ -423,7 +431,7 @@ def test_fastparquet_read_with_hdfs(client):
fs = hdfs_test_client(client)
- df = pdt.makeDataFrame()
+ df = makeDataFrame()
table = pa.Table.from_pandas(df)
path = '/tmp/testing.parquet'
@@ -433,4 +441,4 @@ def test_fastparquet_read_with_hdfs(client):
parquet_file = fastparquet.ParquetFile(path, open_with=fs.open)
result = parquet_file.to_pandas()
- pdt.assert_frame_equal(result, df)
+ assert_frame_equal(result, df)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index 77ed70c..8eb76ed 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -33,8 +33,6 @@ except ImportError:
import numpy as np
-import pandas as pd
-
from pyarrow.compat import u, guid, PY2
import pyarrow as pa
@@ -628,9 +626,8 @@ def test_mock_output_stream():
assert f1.size() == len(f2.getvalue())
- # Do the same test with a pandas DataFrame
- val = pd.DataFrame({'a': [1, 2, 3]})
- record_batch = pa.RecordBatch.from_pandas(val)
+ # Do the same test with a table
+ record_batch = pa.RecordBatch.from_arrays([pa.array([1, 2, 3])], ['a'])
f1 = pa.MockOutputStream()
f2 = pa.BufferOutputStream()
diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py
index 7c0bb2f..467e26d 100644
--- a/python/pyarrow/tests/test_ipc.py
+++ b/python/pyarrow/tests/test_ipc.py
@@ -23,13 +23,22 @@ import threading
import numpy as np
-from pandas.util.testing import (assert_frame_equal,
- assert_series_equal)
-import pandas as pd
-
import pyarrow as pa
+try:
+ from pandas.util.testing import (assert_frame_equal,
+ assert_series_equal)
+ import pandas as pd
+except ImportError:
+ pass
+
+
+# TODO(wesm): The IPC tests depend a lot on pandas currently, so all excluded
+# when it is not installed
+pytestmark = pytest.mark.pandas
+
+
class IpcFixture(object):
def __init__(self, sink_factory=lambda: io.BytesIO()):
diff --git a/python/pyarrow/tests/test_orc.py b/python/pyarrow/tests/test_orc.py
index ca0406f..2a274c9 100644
--- a/python/pyarrow/tests/test_orc.py
+++ b/python/pyarrow/tests/test_orc.py
@@ -19,16 +19,20 @@ import pytest
import decimal
import datetime
-import pandas as pd
import pyarrow as pa
-from pandas.util.testing import assert_frame_equal
-
# Marks all of the tests in this module
# Ignore these with pytest ... -m 'not orc'
pytestmark = pytest.mark.orc
+try:
+ from pandas.util.testing import assert_frame_equal
+ import pandas as pd
+except ImportError:
+ pass
+
+
@pytest.fixture(scope='module')
def datadir(datadir):
return datadir / 'orc'
@@ -110,6 +114,7 @@ def check_example_file(orc_path, expected_df, need_fix=False):
assert json_pos == orc_file.nrows
+@pytest.mark.pandas
@pytest.mark.parametrize('filename', [
'TestOrcFile.test1.orc',
'TestOrcFile.testDate1900.orc',
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_pandas.py
similarity index 94%
rename from python/pyarrow/tests/test_convert_pandas.py
rename to python/pyarrow/tests/test_pandas.py
index 0aab444..763c0e2 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_pandas.py
@@ -30,13 +30,22 @@ import hypothesis.extra.pytz as tzst
import hypothesis.strategies as st
import numpy as np
import numpy.testing as npt
-import pandas as pd
-import pandas.util.testing as tm
import pytest
import pytz
+from pyarrow.pandas_compat import get_logical_type
import pyarrow as pa
-from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+
+try:
+ import pandas as pd
+ import pandas.util.testing as tm
+ from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+except ImportError:
+ pass
+
+
+# Marks all of the tests in this module
+pytestmark = pytest.mark.pandas
def _alltypes_example(size=100):
@@ -2532,6 +2541,180 @@ def test_table_from_pandas_columns_and_schema_are_mutually_exclusive():
# ----------------------------------------------------------------------
+# RecordBatch, Table
+
+
+def test_recordbatch_from_to_pandas():
+ data = pd.DataFrame({
+ 'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
+ 'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
+ 'c3': np.random.randn(5),
+ 'c4': ['foo', 'bar', None, 'baz', 'qux'],
+ 'c5': [False, True, False, True, False]
+ })
+
+ batch = pa.RecordBatch.from_pandas(data)
+ result = batch.to_pandas()
+ tm.assert_frame_equal(data, result)
+
+
+def test_recordbatchlist_to_pandas():
+ data1 = pd.DataFrame({
+ 'c1': np.array([1, 1, 2], dtype='uint32'),
+ 'c2': np.array([1.0, 2.0, 3.0], dtype='float64'),
+ 'c3': [True, None, False],
+ 'c4': ['foo', 'bar', None]
+ })
+
+ data2 = pd.DataFrame({
+ 'c1': np.array([3, 5], dtype='uint32'),
+ 'c2': np.array([4.0, 5.0], dtype='float64'),
+ 'c3': [True, True],
+ 'c4': ['baz', 'qux']
+ })
+
+ batch1 = pa.RecordBatch.from_pandas(data1)
+ batch2 = pa.RecordBatch.from_pandas(data2)
+
+ table = pa.Table.from_batches([batch1, batch2])
+ result = table.to_pandas()
+ data = pd.concat([data1, data2]).reset_index(drop=True)
+ tm.assert_frame_equal(data, result)
+
+
+# ----------------------------------------------------------------------
+# Metadata serialization
+
+
+@pytest.mark.parametrize(
+ ('type', 'expected'),
+ [
+ (pa.null(), 'empty'),
+ (pa.bool_(), 'bool'),
+ (pa.int8(), 'int8'),
+ (pa.int16(), 'int16'),
+ (pa.int32(), 'int32'),
+ (pa.int64(), 'int64'),
+ (pa.uint8(), 'uint8'),
+ (pa.uint16(), 'uint16'),
+ (pa.uint32(), 'uint32'),
+ (pa.uint64(), 'uint64'),
+ (pa.float16(), 'float16'),
+ (pa.float32(), 'float32'),
+ (pa.float64(), 'float64'),
+ (pa.date32(), 'date'),
+ (pa.date64(), 'date'),
+ (pa.binary(), 'bytes'),
+ (pa.binary(length=4), 'bytes'),
+ (pa.string(), 'unicode'),
+ (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
+ (pa.decimal128(18, 3), 'decimal'),
+ (pa.timestamp('ms'), 'datetime'),
+ (pa.timestamp('us', 'UTC'), 'datetimetz'),
+ (pa.time32('s'), 'time'),
+ (pa.time64('us'), 'time')
+ ]
+)
+def test_logical_type(type, expected):
+ assert get_logical_type(type) == expected
+
+
+# ----------------------------------------------------------------------
+# Some nested array tests array tests
+
+
+def test_array_from_py_float32():
+ data = [[1.2, 3.4], [9.0, 42.0]]
+
+ t = pa.float32()
+
+ arr1 = pa.array(data[0], type=t)
+ arr2 = pa.array(data, type=pa.list_(t))
+
+ expected1 = np.array(data[0], dtype=np.float32)
+ expected2 = pd.Series([np.array(data[0], dtype=np.float32),
+ np.array(data[1], dtype=np.float32)])
+
+ assert arr1.type == t
+ assert arr1.equals(pa.array(expected1))
+ assert arr2.equals(pa.array(expected2))
+
+
+# ----------------------------------------------------------------------
+# Timestamp tests
+
+
+def test_cast_timestamp_unit():
+ # ARROW-1680
+ val = datetime.now()
+ s = pd.Series([val])
+ s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
+
+ us_with_tz = pa.timestamp('us', tz='America/New_York')
+
+ arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
+
+ # ARROW-1906
+ assert arr.type == us_with_tz
+
+ arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
+
+ assert arr[0].as_py() == s_nyc[0]
+ assert arr2[0].as_py() == s[0]
+
+ # Disallow truncation
+ arr = pa.array([123123], type='int64').cast(pa.timestamp('ms'))
+ expected = pa.array([123], type='int64').cast(pa.timestamp('s'))
+
+ target = pa.timestamp('s')
+ with pytest.raises(ValueError):
+ arr.cast(target)
+
+ result = arr.cast(target, safe=False)
+ assert result.equals(expected)
+
+ # ARROW-1949
+ series = pd.Series([pd.Timestamp(1), pd.Timestamp(10), pd.Timestamp(1000)])
+ expected = pa.array([0, 0, 1], type=pa.timestamp('us'))
+
+ with pytest.raises(ValueError):
+ pa.array(series, type=pa.timestamp('us'))
+
+ with pytest.raises(ValueError):
+ pa.Array.from_pandas(series, type=pa.timestamp('us'))
+
+ result = pa.Array.from_pandas(series, type=pa.timestamp('us'), safe=False)
+ assert result.equals(expected)
+
+ result = pa.array(series, type=pa.timestamp('us'), safe=False)
+ assert result.equals(expected)
+
+
+# ----------------------------------------------------------------------
+# DictionaryArray tests
+
+
+def test_dictionary_with_pandas():
+ indices = np.repeat([0, 1, 2], 2)
+ dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)
+ mask = np.array([False, False, True, False, False, False])
+
+ d1 = pa.DictionaryArray.from_arrays(indices, dictionary)
+ d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)
+
+ pandas1 = d1.to_pandas()
+ ex_pandas1 = pd.Categorical.from_codes(indices, categories=dictionary)
+
+ tm.assert_series_equal(pd.Series(pandas1), pd.Series(ex_pandas1))
+
+ pandas2 = d2.to_pandas()
+ ex_pandas2 = pd.Categorical.from_codes(np.where(mask, -1, indices),
+ categories=dictionary)
+
+ tm.assert_series_equal(pd.Series(pandas2), pd.Series(ex_pandas2))
+
+
+# ----------------------------------------------------------------------
# Legacy metadata compatibility tests
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index ccc01f4..8406f36 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -25,14 +25,12 @@ import six
import pytest
import numpy as np
-import pandas as pd
-import pandas.util.testing as tm
import pyarrow as pa
from pyarrow.compat import guid, u, BytesIO, unichar, PY2
+from pyarrow.pandas_compat import _pandas_api
from pyarrow.tests import util
from pyarrow.filesystem import LocalFileSystem, FileSystem
-from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
try:
import pyarrow.parquet as pq
@@ -40,6 +38,14 @@ except ImportError:
pq = None
+try:
+ import pandas as pd
+ import pandas.util.testing as tm
+ from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
+except ImportError:
+ pd = tm = None
+
+
# Marks all of the tests in this module
# Ignore these with pytest ... -m 'not parquet'
pytestmark = pytest.mark.parquet
@@ -54,7 +60,7 @@ def _write_table(table, path, **kwargs):
# So we see the ImportError somewhere
import pyarrow.parquet as pq
- if isinstance(table, pd.DataFrame):
+ if _pandas_api.is_data_frame(table):
table = pa.Table.from_pandas(table)
pq.write_table(table, path, **kwargs)
@@ -147,6 +153,7 @@ def alltypes_sample(size=10000, seed=0, categorical=False):
return pd.DataFrame(arrays)
+@pytest.mark.pandas
@pytest.mark.parametrize('chunk_size', [None, 1000])
def test_pandas_parquet_2_0_rountrip(tempdir, chunk_size):
df = alltypes_sample(size=10000, categorical=True)
@@ -166,6 +173,7 @@ def test_pandas_parquet_2_0_rountrip(tempdir, chunk_size):
tm.assert_frame_equal(df, df_read, check_categorical=False)
+@pytest.mark.pandas
def test_chunked_table_write():
# ARROW-232
df = alltypes_sample(size=10)
@@ -183,6 +191,7 @@ def test_chunked_table_write():
_check_roundtrip(table, version='2.0')
+@pytest.mark.pandas
def test_no_memory_map(tempdir):
df = alltypes_sample(size=10)
# The nanosecond->us conversion is a nuisance, so we just avoid it here
@@ -210,6 +219,7 @@ def test_special_chars_filename(tempdir):
assert table_read.equals(table)
+@pytest.mark.pandas
def test_empty_table_roundtrip():
df = alltypes_sample(size=10)
# The nanosecond->us conversion is a nuisance, so we just avoid it here
@@ -233,6 +243,7 @@ def test_empty_lists_table_roundtrip():
_check_roundtrip(table)
+@pytest.mark.pandas
def test_pandas_parquet_datetime_tz():
s = pd.Series([datetime.datetime(2017, 9, 6)])
s = s.dt.tz_localize('utc')
@@ -257,6 +268,7 @@ def test_pandas_parquet_datetime_tz():
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
@pytest.mark.skipif(six.PY2, reason='datetime.timezone is available since '
'python version 3.2')
def test_datetime_timezone_tzinfo():
@@ -267,6 +279,7 @@ def test_datetime_timezone_tzinfo():
_roundtrip_pandas_dataframe(df, write_kwargs={})
+@pytest.mark.pandas
def test_pandas_parquet_custom_metadata(tempdir):
df = alltypes_sample(size=10000)
@@ -286,6 +299,7 @@ def test_pandas_parquet_custom_metadata(tempdir):
'step': 1}]
+@pytest.mark.pandas
def test_pandas_parquet_column_multiindex(tempdir):
df = alltypes_sample(size=10)
df.columns = pd.MultiIndex.from_tuples(
@@ -304,6 +318,7 @@ def test_pandas_parquet_column_multiindex(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tempdir):
df = alltypes_sample(size=10000)
@@ -327,6 +342,7 @@ def test_pandas_parquet_2_0_rountrip_read_pandas_no_index_written(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_pandas_parquet_1_0_rountrip(tempdir):
size = 10000
np.random.seed(0)
@@ -358,6 +374,7 @@ def test_pandas_parquet_1_0_rountrip(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_multiple_path_types(tempdir):
# Test compatibility with PEP 519 path-like objects
path = tempdir / 'zzz.parquet'
@@ -376,6 +393,7 @@ def test_multiple_path_types(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_pandas_column_selection(tempdir):
size = 10000
np.random.seed(0)
@@ -431,6 +449,7 @@ def _test_dataframe(size=10000, seed=0):
return df
+@pytest.mark.pandas
def test_pandas_parquet_native_file_roundtrip(tempdir):
df = _test_dataframe(10000)
arrow_table = pa.Table.from_pandas(df)
@@ -442,6 +461,7 @@ def test_pandas_parquet_native_file_roundtrip(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_parquet_incremental_file_build(tempdir):
df = _test_dataframe(100)
df['unique_id'] = 0
@@ -468,6 +488,7 @@ def test_parquet_incremental_file_build(tempdir):
tm.assert_frame_equal(result.to_pandas(), expected)
+@pytest.mark.pandas
def test_read_pandas_column_subset(tempdir):
df = _test_dataframe(10000)
arrow_table = pa.Table.from_pandas(df)
@@ -479,6 +500,7 @@ def test_read_pandas_column_subset(tempdir):
tm.assert_frame_equal(df[['strings', 'uint8']], df_read)
+@pytest.mark.pandas
def test_pandas_parquet_empty_roundtrip(tempdir):
df = _test_dataframe(0)
arrow_table = pa.Table.from_pandas(df)
@@ -490,6 +512,7 @@ def test_pandas_parquet_empty_roundtrip(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_pandas_parquet_pyfile_roundtrip(tempdir):
filename = tempdir / 'pandas_pyfile_roundtrip.parquet'
size = 5
@@ -513,6 +536,7 @@ def test_pandas_parquet_pyfile_roundtrip(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_pandas_parquet_configuration_options(tempdir):
size = 10000
np.random.seed(0)
@@ -561,6 +585,7 @@ def make_sample_file(table_or_df):
return pq.ParquetFile(buf)
+@pytest.mark.pandas
def test_parquet_metadata_api():
df = alltypes_sample(size=10000)
df = df.reindex(columns=sorted(df.columns))
@@ -639,6 +664,7 @@ def test_parquet_metadata_api():
col_meta.index_page_offset
+@pytest.mark.pandas
@pytest.mark.parametrize(
(
'data',
@@ -706,6 +732,7 @@ def test_parquet_column_statistics_api(data, type, physical_type, min_value,
assert stat.physical_type == physical_type
+@pytest.mark.pandas
def test_compare_schemas():
df = alltypes_sample(size=10000)
@@ -754,6 +781,7 @@ def test_validate_schema_write_table(tempdir):
w.write_table(simple_table)
+@pytest.mark.pandas
def test_column_of_arrays(tempdir):
df, schema = dataframe_with_arrays()
@@ -765,6 +793,7 @@ def test_column_of_arrays(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_coerce_timestamps(tempdir):
from collections import OrderedDict
# ARROW-622
@@ -806,6 +835,7 @@ def test_coerce_timestamps(tempdir):
coerce_timestamps='unknown')
+@pytest.mark.pandas
def test_coerce_timestamps_truncated(tempdir):
"""
ARROW-2555: Test that we can truncate timestamps when coercing if
@@ -835,6 +865,7 @@ def test_coerce_timestamps_truncated(tempdir):
tm.assert_frame_equal(df_expected, df_ms)
+@pytest.mark.pandas
def test_column_of_lists(tempdir):
df, schema = dataframe_with_lists(parquet_compatible=True)
@@ -856,6 +887,7 @@ def test_column_of_lists(tempdir):
tm.assert_frame_equal(df, df_read)
+@pytest.mark.pandas
def test_date_time_types():
t1 = pa.date32()
data1 = np.array([17259, 17260, 17261], dtype='int32')
@@ -943,6 +975,7 @@ def test_date_time_types():
_assert_unsupported(a7)
+@pytest.mark.pandas
def test_list_of_datetime_time_roundtrip():
# ARROW-4135
times = pd.to_datetime(['09:00', '09:30', '10:00', '10:30', '11:00',
@@ -978,6 +1011,7 @@ def test_sanitized_spark_field_names():
assert result.schema[0].name == expected_name
+@pytest.mark.pandas
def test_spark_flavor_preserves_pandas_metadata():
df = _test_dataframe(size=100)
df.index = np.arange(0, 10 * len(df), 10)
@@ -998,6 +1032,7 @@ def test_fixed_size_binary():
_check_roundtrip(table)
+@pytest.mark.pandas
def test_multithreaded_read():
df = alltypes_sample(size=10000)
@@ -1015,6 +1050,7 @@ def test_multithreaded_read():
assert table1.equals(table2)
+@pytest.mark.pandas
def test_min_chunksize():
data = pd.DataFrame([np.arange(4)], columns=['A', 'B', 'C', 'D'])
table = pa.Table.from_pandas(data.reset_index())
@@ -1031,6 +1067,7 @@ def test_min_chunksize():
_write_table(table, buf, chunk_size=0)
+@pytest.mark.pandas
def test_pass_separate_metadata():
# ARROW-471
df = alltypes_sample(size=10000)
@@ -1050,6 +1087,7 @@ def test_pass_separate_metadata():
tm.assert_frame_equal(df, fileh.read().to_pandas())
+@pytest.mark.pandas
def test_read_single_row_group():
# ARROW-471
N, K = 10000, 4
@@ -1072,6 +1110,7 @@ def test_read_single_row_group():
tm.assert_frame_equal(df, result.to_pandas())
+@pytest.mark.pandas
def test_read_single_row_group_with_column_subset():
N, K = 10000, 4
df = alltypes_sample(size=N)
@@ -1096,6 +1135,7 @@ def test_read_single_row_group_with_column_subset():
tm.assert_frame_equal(df[cols], result.to_pandas())
+@pytest.mark.pandas
def test_scan_contents():
N, K = 10000, 4
df = alltypes_sample(size=N)
@@ -1112,6 +1152,7 @@ def test_scan_contents():
assert pf.scan_contents(df.columns[:4]) == 10000
+@pytest.mark.pandas
def test_parquet_piece_read(tempdir):
df = _test_dataframe(1000)
table = pa.Table.from_pandas(df)
@@ -1155,11 +1196,13 @@ def test_partition_set_dictionary_type():
set3.dictionary
+@pytest.mark.pandas
def test_read_partitioned_directory(tempdir):
fs = LocalFileSystem.get_instance()
_partition_test_for_filesystem(fs, tempdir)
+@pytest.mark.pandas
def test_create_parquet_dataset_multi_threaded(tempdir):
fs = LocalFileSystem.get_instance()
base_path = tempdir
@@ -1176,6 +1219,7 @@ def test_create_parquet_dataset_multi_threaded(tempdir):
assert len(partitions.levels) == len(manifest.partitions.levels)
+@pytest.mark.pandas
def test_equivalency(tempdir):
fs = LocalFileSystem.get_instance()
base_path = tempdir
@@ -1251,6 +1295,7 @@ def test_equivalency(tempdir):
pq.ParquetDataset(base_path, filesystem=fs, filters=filters)
+@pytest.mark.pandas
def test_cutoff_exclusive_integer(tempdir):
fs = LocalFileSystem.get_instance()
base_path = tempdir
@@ -1284,6 +1329,7 @@ def test_cutoff_exclusive_integer(tempdir):
assert result_list == [2, 3]
+@pytest.mark.pandas
@pytest.mark.xfail(
raises=TypeError,
reason='Loss of type information in creation of categoricals.'
@@ -1330,6 +1376,7 @@ def test_cutoff_exclusive_datetime(tempdir):
assert result_df['dates'].values == expected
+@pytest.mark.pandas
def test_inclusive_integer(tempdir):
fs = LocalFileSystem.get_instance()
base_path = tempdir
@@ -1356,13 +1403,14 @@ def test_inclusive_integer(tempdir):
)
table = dataset.read()
result_df = (table.to_pandas()
- .sort_values(by='index')
- .reset_index(drop=True))
+ .sort_values(by='index')
+ .reset_index(drop=True))
result_list = [int(x) for x in map(int, result_df['integers'].values)]
assert result_list == [2, 3]
+@pytest.mark.pandas
def test_inclusive_set(tempdir):
fs = LocalFileSystem.get_instance()
base_path = tempdir
@@ -1398,6 +1446,7 @@ def test_inclusive_set(tempdir):
assert False not in result_df['boolean'].values
+@pytest.mark.pandas
def test_invalid_pred_op(tempdir):
fs = LocalFileSystem.get_instance()
base_path = tempdir
@@ -1454,6 +1503,7 @@ def s3_example():
fs.rm(bucket_uri, recursive=True)
+@pytest.mark.pandas
@pytest.mark.s3
def test_read_partitioned_directory_s3fs(s3_example):
from pyarrow.filesystem import S3FSWrapper
@@ -1565,11 +1615,13 @@ def _test_read_common_metadata_files(fs, base_path):
assert dataset2.schema.equals(dataset.schema)
+@pytest.mark.pandas
def test_read_common_metadata_files(tempdir):
fs = LocalFileSystem.get_instance()
_test_read_common_metadata_files(fs, tempdir)
+@pytest.mark.pandas
def test_read_metadata_files(tempdir):
fs = LocalFileSystem.get_instance()
@@ -1598,6 +1650,7 @@ def test_read_metadata_files(tempdir):
assert dataset.schema.equals(metadata_schema)
+@pytest.mark.pandas
def test_read_schema(tempdir):
N = 100
df = pd.DataFrame({
@@ -1630,6 +1683,7 @@ def _filter_partition(df, part_keys):
return df[predicate].drop(to_drop, axis=1)
+@pytest.mark.pandas
def test_read_multiple_files(tempdir):
nfiles = 10
size = 5
@@ -1709,6 +1763,7 @@ def test_read_multiple_files(tempdir):
read_multiple_files(mixed_paths)
+@pytest.mark.pandas
def test_dataset_read_pandas(tempdir):
nfiles = 5
size = 5
@@ -1740,6 +1795,7 @@ def test_dataset_read_pandas(tempdir):
tm.assert_frame_equal(result, expected)
+@pytest.mark.pandas
def test_dataset_no_memory_map(tempdir):
# ARROW-2627: Check that we can use ParquetDataset without memory-mapping
dirpath = tempdir / guid()
@@ -1756,6 +1812,7 @@ def test_dataset_no_memory_map(tempdir):
assert dataset.pieces[0].read().equals(table)
+@pytest.mark.pandas
@pytest.mark.parametrize('preserve_index', [True, False])
def test_dataset_read_pandas_common_metadata(tempdir, preserve_index):
# ARROW-1103
@@ -1811,6 +1868,7 @@ def _make_example_multifile_dataset(base_path, nfiles=10, file_nrows=5):
return paths
+@pytest.mark.pandas
def test_ignore_private_directories(tempdir):
dirpath = tempdir / guid()
dirpath.mkdir()
@@ -1825,6 +1883,7 @@ def test_ignore_private_directories(tempdir):
assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
+@pytest.mark.pandas
def test_ignore_hidden_files(tempdir):
dirpath = tempdir / guid()
dirpath.mkdir()
@@ -1842,6 +1901,7 @@ def test_ignore_hidden_files(tempdir):
assert set(map(str, paths)) == set(x.path for x in dataset.pieces)
+@pytest.mark.pandas
def test_multiindex_duplicate_values(tempdir):
num_rows = 3
numbers = list(range(num_rows))
@@ -1863,6 +1923,7 @@ def test_multiindex_duplicate_values(tempdir):
tm.assert_frame_equal(result_df, df)
+@pytest.mark.pandas
def test_write_error_deletes_incomplete_file(tempdir):
# ARROW-1285
df = pd.DataFrame({'a': list('abc'),
@@ -1984,10 +2045,12 @@ def _test_write_to_dataset_no_partitions(base_path, filesystem=None):
assert output_df.equals(input_df)
+@pytest.mark.pandas
def test_write_to_dataset_with_partitions(tempdir):
_test_write_to_dataset_with_partitions(str(tempdir))
+@pytest.mark.pandas
def test_write_to_dataset_with_partitions_and_schema(tempdir):
schema = pa.schema([pa.field('group1', type=pa.string()),
pa.field('group2', type=pa.string()),
@@ -1997,11 +2060,13 @@ def test_write_to_dataset_with_partitions_and_schema(tempdir):
_test_write_to_dataset_with_partitions(str(tempdir), schema=schema)
+@pytest.mark.pandas
def test_write_to_dataset_with_partitions_and_index_name(tempdir):
_test_write_to_dataset_with_partitions(str(tempdir),
index_name='index_name')
+@pytest.mark.pandas
def test_write_to_dataset_no_partitions(tempdir):
_test_write_to_dataset_no_partitions(str(tempdir))
@@ -2019,6 +2084,7 @@ def test_large_table_int32_overflow():
_write_table(table, f)
+@pytest.mark.pandas
@pytest.mark.large_memory
def test_binary_array_overflow_to_chunked():
# ARROW-3762
@@ -2046,6 +2112,7 @@ def test_binary_array_overflow_to_chunked():
assert tbl.equals(read_tbl)
+@pytest.mark.pandas
def test_index_column_name_duplicate(tempdir):
data = {
'close': {
@@ -2070,6 +2137,7 @@ def test_index_column_name_duplicate(tempdir):
tm.assert_frame_equal(result_df, dfx)
+@pytest.mark.pandas
def test_parquet_nested_convenience(tempdir):
# ARROW-1684
df = pd.DataFrame({
@@ -2089,6 +2157,7 @@ def test_parquet_nested_convenience(tempdir):
tm.assert_frame_equal(read.to_pandas(), df)
+@pytest.mark.pandas
def test_backwards_compatible_index_naming(datadir):
expected_string = b"""\
carat cut color clarity depth table price x y z
@@ -2109,6 +2178,7 @@ carat cut color clarity depth table price x y z
tm.assert_frame_equal(result, expected)
+@pytest.mark.pandas
def test_backwards_compatible_index_multi_level_named(datadir):
expected_string = b"""\
carat cut color clarity depth table price x y z
@@ -2133,6 +2203,7 @@ carat cut color clarity depth table price x y z
tm.assert_frame_equal(result, expected)
+@pytest.mark.pandas
def test_backwards_compatible_index_multi_level_some_named(datadir):
expected_string = b"""\
carat cut color clarity depth table price x y z
@@ -2158,6 +2229,7 @@ carat cut color clarity depth table price x y z
tm.assert_frame_equal(result, expected)
+@pytest.mark.pandas
def test_backwards_compatible_column_metadata_handling(datadir):
expected = pd.DataFrame(
{'a': [1, 2, 3], 'b': [.1, .2, .3],
@@ -2177,6 +2249,7 @@ def test_backwards_compatible_column_metadata_handling(datadir):
tm.assert_frame_equal(result, expected[['a']].reset_index(drop=True))
+@pytest.mark.pandas
def test_decimal_roundtrip(tempdir):
num_values = 10
@@ -2202,6 +2275,7 @@ def test_decimal_roundtrip(tempdir):
tm.assert_frame_equal(result, expected)
+@pytest.mark.pandas
@pytest.mark.xfail(
raises=pa.ArrowException, reason='Parquet does not support negative scale'
)
@@ -2216,6 +2290,7 @@ def test_decimal_roundtrip_negative_scale(tempdir):
tm.assert_frame_equal(result, expected)
+@pytest.mark.pandas
def test_parquet_writer_context_obj(tempdir):
df = _test_dataframe(100)
df['unique_id'] = 0
@@ -2240,6 +2315,7 @@ def test_parquet_writer_context_obj(tempdir):
tm.assert_frame_equal(result.to_pandas(), expected)
+@pytest.mark.pandas
def test_parquet_writer_context_obj_with_exception(tempdir):
df = _test_dataframe(100)
df['unique_id'] = 0
@@ -2271,6 +2347,7 @@ def test_parquet_writer_context_obj_with_exception(tempdir):
tm.assert_frame_equal(result.to_pandas(), expected)
+@pytest.mark.pandas
def test_zlib_compression_bug():
# ARROW-3514: "zlib deflate failed, output buffer too small"
table = pa.Table.from_arrays([pa.array(['abc', 'def'])], ['some_col'])
@@ -2282,6 +2359,7 @@ def test_zlib_compression_bug():
tm.assert_frame_equal(roundtrip.to_pandas(), table.to_pandas())
+@pytest.mark.pandas
def test_merging_parquet_tables_with_different_pandas_metadata(tempdir):
# ARROW-3728: Merging Parquet Files - Pandas Meta in Schema Mismatch
schema = pa.schema([
@@ -2328,6 +2406,7 @@ def test_empty_row_groups(tempdir):
assert reader.read_row_group(i).equals(table)
+@pytest.mark.pandas
def test_parquet_writer_with_caller_provided_filesystem():
out = pa.BufferOutputStream()
diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py
index c8a4d33..14eff02 100644
--- a/python/pyarrow/tests/test_plasma.py
+++ b/python/pyarrow/tests/test_plasma.py
@@ -32,7 +32,6 @@ import time
import numpy as np
import pyarrow as pa
-import pandas as pd
DEFAULT_PLASMA_STORE_MEMORY = 10 ** 8
@@ -408,7 +407,9 @@ class TestPlasmaClient(object):
# Assert that they are equal.
np.testing.assert_equal(data, array)
+ @pytest.mark.pandas
def test_store_pandas_dataframe(self):
+ import pandas as pd
import pyarrow.plasma as plasma
d = {'one': pd.Series([1., 2., 3.], index=['a', 'b', 'c']),
'two': pd.Series([1., 2., 3., 4.], index=['a', 'b', 'c', 'd'])}
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 892d65b..406f4ee 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -19,7 +19,6 @@
import pytest
import numpy as np
-import pandas as pd
from pyarrow.compat import unittest, u, unicode_type
import pyarrow as pa
@@ -159,7 +158,9 @@ class TestScalars(unittest.TestCase):
v = arr[3]
assert len(v) == 0
+ @pytest.mark.pandas
def test_timestamp(self):
+ import pandas as pd
arr = pd.date_range('2000-01-01 12:34:56', periods=10).values
units = ['ns', 'us', 'ms', 's']
@@ -185,7 +186,9 @@ class TestScalars(unittest.TestCase):
assert arrow_arr[0].as_py() == expected
assert arrow_arr[0].value * 1000**i == expected.value
+ @pytest.mark.pandas
def test_dictionary(self):
+ import pandas as pd
colors = ['red', 'green', 'blue']
colors_dict = {'red': 0, 'green': 1, 'blue': 2}
values = pd.Series(colors * 4)
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index 75c7f48..d922443 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -20,8 +20,6 @@ import pickle
import pytest
import numpy as np
-import pandas as pd
-
import pyarrow as pa
@@ -493,18 +491,21 @@ def test_empty_table():
assert table.schema == schema
-@pytest.mark.parametrize('data', [
- list(range(10)),
- pd.Categorical(list(range(10))),
- ['foo', 'bar', None, 'baz', 'qux'],
- np.array([
- '2007-07-13T01:23:34.123456789',
- '2006-01-13T12:34:56.432539784',
- '2010-08-13T05:46:57.437699912'
- ], dtype='datetime64[ns]')
-])
-def test_schema_from_pandas(data):
- df = pd.DataFrame({'a': data})
- schema = pa.Schema.from_pandas(df)
- expected = pa.Table.from_pandas(df).schema
- assert schema == expected
+@pytest.mark.pandas
+def test_schema_from_pandas():
+ import pandas as pd
+ inputs = [
+ list(range(10)),
+ pd.Categorical(list(range(10))),
+ ['foo', 'bar', None, 'baz', 'qux'],
+ np.array([
+ '2007-07-13T01:23:34.123456789',
+ '2006-01-13T12:34:56.432539784',
+ '2010-08-13T05:46:57.437699912'
+ ], dtype='datetime64[ns]')
+ ]
+ for data in inputs:
+ df = pd.DataFrame({'a': data})
+ schema = pa.Schema.from_pandas(df)
+ expected = pa.Table.from_pandas(df).schema
+ assert schema == expected
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 073a7c1..91a87bf 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -19,10 +19,7 @@ from collections import OrderedDict, Iterable
import pickle
import numpy as np
-from pandas.util.testing import assert_frame_equal
-import pandas as pd
import pytest
-
import pyarrow as pa
@@ -176,6 +173,7 @@ def test_chunked_array_pickle(data, typ):
assert result.equals(array)
+@pytest.mark.pandas
def test_chunked_array_to_pandas():
data = [
pa.array([-10, -5, 0, 5, 10])
@@ -269,6 +267,7 @@ def test_column_pickle():
assert result.field == field
+@pytest.mark.pandas
def test_column_to_pandas():
data = [
pa.array([-10, -5, 0, 5, 10])
@@ -459,50 +458,11 @@ def test_recordbatch_slice_getitem():
assert batch.slice(len(batch) - 4, 2).equals(batch[-4:-2])
-def test_recordbatch_from_to_pandas():
- data = pd.DataFrame({
- 'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
- 'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
- 'c3': np.random.randn(5),
- 'c4': ['foo', 'bar', None, 'baz', 'qux'],
- 'c5': [False, True, False, True, False]
- })
-
- batch = pa.RecordBatch.from_pandas(data)
- result = batch.to_pandas()
- assert_frame_equal(data, result)
-
-
-def test_recordbatchlist_to_pandas():
- data1 = pd.DataFrame({
- 'c1': np.array([1, 1, 2], dtype='uint32'),
- 'c2': np.array([1.0, 2.0, 3.0], dtype='float64'),
- 'c3': [True, None, False],
- 'c4': ['foo', 'bar', None]
- })
-
- data2 = pd.DataFrame({
- 'c1': np.array([3, 5], dtype='uint32'),
- 'c2': np.array([4.0, 5.0], dtype='float64'),
- 'c3': [True, True],
- 'c4': ['baz', 'qux']
- })
-
- batch1 = pa.RecordBatch.from_pandas(data1)
- batch2 = pa.RecordBatch.from_pandas(data2)
-
- table = pa.Table.from_batches([batch1, batch2])
- result = table.to_pandas()
- data = pd.concat([data1, data2]).reset_index(drop=True)
- assert_frame_equal(data, result)
-
-
def test_recordbatchlist_schema_equals():
- data1 = pd.DataFrame({'c1': np.array([1], dtype='uint32')})
- data2 = pd.DataFrame({'c1': np.array([4.0, 5.0], dtype='float64')})
-
- batch1 = pa.RecordBatch.from_pandas(data1)
- batch2 = pa.RecordBatch.from_pandas(data2)
+ a1 = np.array([1], dtype='uint32')
+ a2 = np.array([4.0, 5.0], dtype='float64')
+ batch1 = pa.RecordBatch.from_arrays([pa.array(a1)], ['c1'])
+ batch2 = pa.RecordBatch.from_arrays([pa.array(a2)], ['c1'])
with pytest.raises(pa.ArrowInvalid):
pa.Table.from_batches([batch1, batch2])
@@ -537,7 +497,11 @@ def test_table_from_batches_and_schema():
pa.Table.from_batches([incompatible_batch], schema)
+@pytest.mark.pandas
def test_table_to_batches():
+ from pandas.util.testing import assert_frame_equal
+ import pandas as pd
+
df1 = pd.DataFrame({'a': list(range(10))})
df2 = pd.DataFrame({'a': list(range(10, 30))})
@@ -809,7 +773,10 @@ def test_concat_tables():
assert result.equals(expected)
+@pytest.mark.pandas
def test_concat_tables_with_different_schema_metadata():
+ import pandas as pd
+
schema = pa.schema([
pa.field('a', pa.string()),
pa.field('b', pa.string()),
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index db71292..4b1f2f4 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -388,7 +388,8 @@ cdef class TimestampType(DataType):
return _pandas_type_map[_Type_TIMESTAMP]
else:
# Return DatetimeTZ
- return pdcompat.make_datetimetz(self.tz)
+ from pyarrow.pandas_compat import make_datetimetz
+ return make_datetimetz(self.tz)
def __reduce__(self):
return timestamp, (self.unit, self.tz)
@@ -745,7 +746,8 @@ cdef class Schema:
str: string
__index_level_0__: int64
"""
- names, types, metadata = pdcompat.dataframe_to_types(
+ from pyarrow.pandas_compat import dataframe_to_types
+ names, types, metadata = dataframe_to_types(
df,
preserve_index=preserve_index
)
diff --git a/python/requirements-test.txt b/python/requirements-test.txt
index 89af5ec..2f0e9e7 100644
--- a/python/requirements-test.txt
+++ b/python/requirements-test.txt
@@ -1,4 +1,5 @@
pandas
pytest
hypothesis
+pytz
pathlib2; python_version < "3.4"
diff --git a/python/scripts/test_imports.py b/python/scripts/test_imports.py
new file mode 100644
index 0000000..c0beb44
--- /dev/null
+++ b/python/scripts/test_imports.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pyarrow as pa # noqa
+import sys
+
+assert 'pandas' not in sys.modules