You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/10/26 02:05:11 UTC
[arrow] branch master updated: ARROW-1484: [C++/Python] Implement
casts between date, time, timestamp units
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 54d5c81 ARROW-1484: [C++/Python] Implement casts between date, time, timestamp units
54d5c81 is described below
commit 54d5c81af0a9cbc6ea551922c795728cd43bd86c
Author: Wes McKinney <we...@twosigma.com>
AuthorDate: Wed Oct 25 22:05:05 2017 -0400
ARROW-1484: [C++/Python] Implement casts between date, time, timestamp units
Several JIRAs here that made sense to tackle together:
* ARROW-1680
* ARROW-1482
* ARROW-1484
* ARROW-1524
This also fixes bugs relating to ignoring the offset in sliced arrays in some of the cast kernel implementations.
cc @BryanCutler @xhochy @cpcloud
Author: Wes McKinney <we...@twosigma.com>
Closes #1245 from wesm/ARROW-1484 and squashes the following commits:
6f6107fc [Wes McKinney] Test that timestamp truncation raises for safe casts
1d28ce27 [Wes McKinney] Fix typo
a820ed90 [Wes McKinney] Add test case for ARROW-1680
ac9845e3 [Wes McKinney] Implement date32<->date64 casts, unit tests
4911b09f [Wes McKinney] Add time casts
6441a954 [Wes McKinney] Write unit tests for timestamp to timestamp casts
1bde233e [Wes McKinney] Cast table for timestamps
569dfb81 [Wes McKinney] Fix existing cast tests with sliced arrays
884f6d25 [Wes McKinney] Baby steps
a0401363 [Wes McKinney] Placeholders for time32/64/timestamp cast functions
fddf466b [Wes McKinney] Stub out unit test for timestamp unit change
---
cpp/src/arrow/compute/cast.cc | 213 +++++++++++++++++++++++++++++-----
cpp/src/arrow/compute/cast.h | 3 +-
cpp/src/arrow/compute/compute-test.cc | 210 ++++++++++++++++++++++++++++++++-
cpp/src/arrow/test-util.h | 21 ++--
cpp/src/arrow/type.h | 12 +-
python/pyarrow/array.pxi | 4 +-
python/pyarrow/includes/libarrow.pxd | 1 +
python/pyarrow/tests/test_array.py | 26 +++++
8 files changed, 440 insertions(+), 50 deletions(-)
diff --git a/cpp/src/arrow/compute/cast.cc b/cpp/src/arrow/compute/cast.cc
index e8bbfd3..68a2b12 100644
--- a/cpp/src/arrow/compute/cast.cc
+++ b/cpp/src/arrow/compute/cast.cc
@@ -25,6 +25,7 @@
#include <sstream>
#include <string>
#include <type_traits>
+#include <utility>
#include "arrow/array.h"
#include "arrow/buffer.h"
@@ -68,6 +69,24 @@
namespace arrow {
namespace compute {
+template <typename T>
+inline const T* GetValuesAs(const ArrayData& data, int i) {
+ return reinterpret_cast<const T*>(data.buffers[i]->data()) + data.offset;
+}
+
+namespace {
+
+void CopyData(const Array& input, ArrayData* output) {
+ auto in_data = input.data();
+ output->length = in_data->length;
+ output->null_count = input.null_count();
+ output->buffers = in_data->buffers;
+ output->offset = in_data->offset;
+ output->child_data = in_data->child_data;
+}
+
+} // namespace
+
// ----------------------------------------------------------------------
// Zero copy casts
@@ -77,7 +96,9 @@ struct is_zero_copy_cast {
};
template <typename O, typename I>
-struct is_zero_copy_cast<O, I, typename std::enable_if<std::is_same<I, O>::value>::type> {
+struct is_zero_copy_cast<
+ O, I, typename std::enable_if<std::is_same<I, O>::value &&
+ !std::is_base_of<ParametricType, O>::value>::type> {
static constexpr bool value = true;
};
@@ -102,10 +123,7 @@ template <typename O, typename I>
struct CastFunctor<O, I, typename std::enable_if<is_zero_copy_cast<O, I>::value>::type> {
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
ArrayData* output) {
- auto in_data = input.data();
- output->null_count = input.null_count();
- output->buffers = in_data->buffers;
- output->child_data = in_data->child_data;
+ CopyData(input, output);
}
};
@@ -119,6 +137,7 @@ struct CastFunctor<T, NullType, typename std::enable_if<
ArrayData* output) {
// Simply initialize data to 0
auto buf = output->buffers[1];
+ DCHECK_EQ(output->offset, 0);
memset(buf->mutable_data(), 0, buf->size());
}
};
@@ -139,12 +158,16 @@ struct CastFunctor<T, BooleanType,
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
ArrayData* output) {
using c_type = typename T::c_type;
- const uint8_t* data = input.data()->buffers[1]->data();
- auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
constexpr auto kOne = static_cast<c_type>(1);
constexpr auto kZero = static_cast<c_type>(0);
+
+ auto in_data = input.data();
+ internal::BitmapReader bit_reader(in_data->buffers[1]->data(), in_data->offset,
+ in_data->length);
+ auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
for (int64_t i = 0; i < input.length(); ++i) {
- *out++ = BitUtil::GetBit(data, i) ? kOne : kZero;
+ *out++ = bit_reader.IsSet() ? kOne : kZero;
+ bit_reader.Next();
}
}
};
@@ -189,7 +212,9 @@ struct CastFunctor<O, I, typename std::enable_if<std::is_same<BooleanType, O>::v
void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
ArrayData* output) {
using in_type = typename I::c_type;
- auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
+ DCHECK_EQ(output->offset, 0);
+
+ const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
uint8_t* out_data = reinterpret_cast<uint8_t*>(output->buffers[1]->mutable_data());
for (int64_t i = 0; i < input.length(); ++i) {
BitUtil::SetBitTo(out_data, i, (*in_data++) != 0);
@@ -204,12 +229,11 @@ struct CastFunctor<O, I,
ArrayData* output) {
using in_type = typename I::c_type;
using out_type = typename O::c_type;
+ DCHECK_EQ(output->offset, 0);
auto in_offset = input.offset();
- const auto& input_buffers = input.data()->buffers;
-
- auto in_data = reinterpret_cast<const in_type*>(input_buffers[1]->data()) + in_offset;
+ const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
if (!options.allow_int_overflow) {
@@ -217,14 +241,15 @@ struct CastFunctor<O, I,
constexpr in_type kMin = static_cast<in_type>(std::numeric_limits<out_type>::min());
if (input.null_count() > 0) {
- const uint8_t* is_valid = input_buffers[0]->data();
- int64_t is_valid_offset = in_offset;
+ internal::BitmapReader is_valid_reader(input.data()->buffers[0]->data(),
+ in_offset, input.length());
for (int64_t i = 0; i < input.length(); ++i) {
- if (ARROW_PREDICT_FALSE(BitUtil::GetBit(is_valid, is_valid_offset++) &&
+ if (ARROW_PREDICT_FALSE(is_valid_reader.IsSet() &&
(*in_data > kMax || *in_data < kMin))) {
ctx->SetStatus(Status::Invalid("Integer value out of bounds"));
}
*out_data++ = static_cast<out_type>(*in_data++);
+ is_valid_reader.Next();
}
} else {
for (int64_t i = 0; i < input.length(); ++i) {
@@ -251,7 +276,7 @@ struct CastFunctor<O, I,
using in_type = typename I::c_type;
using out_type = typename O::c_type;
- auto in_data = reinterpret_cast<const in_type*>(input.data()->buffers[1]->data());
+ const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
for (int64_t i = 0; i < input.length(); ++i) {
*out_data++ = static_cast<out_type>(*in_data++);
@@ -260,6 +285,125 @@ struct CastFunctor<O, I,
};
// ----------------------------------------------------------------------
+// From one timestamp to another
+
+template <typename in_type, typename out_type>
+inline void ShiftTime(FunctionContext* ctx, const CastOptions& options,
+ const bool is_multiply, const int64_t factor, const Array& input,
+ ArrayData* output) {
+ const in_type* in_data = GetValuesAs<in_type>(*input.data(), 1);
+ auto out_data = reinterpret_cast<out_type*>(output->buffers[1]->mutable_data());
+
+ if (is_multiply) {
+ for (int64_t i = 0; i < input.length(); i++) {
+ out_data[i] = static_cast<out_type>(in_data[i] * factor);
+ }
+ } else {
+ if (options.allow_time_truncate) {
+ for (int64_t i = 0; i < input.length(); i++) {
+ out_data[i] = static_cast<out_type>(in_data[i] / factor);
+ }
+ } else {
+ for (int64_t i = 0; i < input.length(); i++) {
+ out_data[i] = static_cast<out_type>(in_data[i] / factor);
+ if (input.IsValid(i) && (out_data[i] * factor != in_data[i])) {
+ std::stringstream ss;
+ ss << "Casting from " << input.type()->ToString() << " to "
+ << output->type->ToString() << " would lose data: " << in_data[i];
+ ctx->SetStatus(Status::Invalid(ss.str()));
+ break;
+ }
+ }
+ }
+ }
+}
+
+namespace {
+
+// {is_multiply, factor}
+const std::pair<bool, int64_t> kTimeConversionTable[4][4] = {
+ {{true, 1}, {true, 1000}, {true, 1000000}, {true, 1000000000L}}, // SECOND
+ {{false, 1000}, {true, 1}, {true, 1000}, {true, 1000000}}, // MILLI
+ {{false, 1000000}, {false, 1000}, {true, 1}, {true, 1000}}, // MICRO
+ {{false, 1000000000L}, {false, 1000000}, {false, 1000}, {true, 1}}, // NANO
+};
+
+} // namespace
+
+template <>
+struct CastFunctor<TimestampType, TimestampType> {
+ void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+ ArrayData* output) {
+ // If units are the same, zero copy, otherwise convert
+ const auto& in_type = static_cast<const TimestampType&>(*input.type());
+ const auto& out_type = static_cast<const TimestampType&>(*output->type);
+
+ if (in_type.unit() == out_type.unit()) {
+ CopyData(input, output);
+ return;
+ }
+
+ std::pair<bool, int64_t> conversion =
+ kTimeConversionTable[static_cast<int>(in_type.unit())]
+ [static_cast<int>(out_type.unit())];
+
+ ShiftTime<int64_t, int64_t>(ctx, options, conversion.first, conversion.second, input,
+ output);
+ }
+};
+
+// ----------------------------------------------------------------------
+// From one time32 or time64 to another
+
+template <typename O, typename I>
+struct CastFunctor<O, I,
+ typename std::enable_if<std::is_base_of<TimeType, I>::value &&
+ std::is_base_of<TimeType, O>::value>::type> {
+ void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+ ArrayData* output) {
+ using in_t = typename I::c_type;
+ using out_t = typename O::c_type;
+
+ // If units are the same, zero copy, otherwise convert
+ const auto& in_type = static_cast<const I&>(*input.type());
+ const auto& out_type = static_cast<const O&>(*output->type);
+
+ if (in_type.unit() == out_type.unit()) {
+ CopyData(input, output);
+ return;
+ }
+
+ std::pair<bool, int64_t> conversion =
+ kTimeConversionTable[static_cast<int>(in_type.unit())]
+ [static_cast<int>(out_type.unit())];
+
+ ShiftTime<in_t, out_t>(ctx, options, conversion.first, conversion.second, input,
+ output);
+ }
+};
+
+// ----------------------------------------------------------------------
+// Between date32 and date64
+
+constexpr int64_t kMillisecondsInDay = 86400000;
+
+template <>
+struct CastFunctor<Date64Type, Date32Type> {
+ void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+ ArrayData* output) {
+ ShiftTime<int32_t, int64_t>(ctx, options, true, kMillisecondsInDay, input, output);
+ }
+};
+
+template <>
+struct CastFunctor<Date32Type, Date64Type> {
+ void operator()(FunctionContext* ctx, const CastOptions& options, const Array& input,
+ ArrayData* output) {
+ ShiftTime<int64_t, int32_t>(ctx, options, false, kMillisecondsInDay, input, output);
+ }
+};
+
+// ----------------------------------------------------------------------
// Dictionary to other things
template <typename IndexType>
@@ -271,9 +415,8 @@ void UnpackFixedSizeBinaryDictionary(FunctionContext* ctx, const Array& indices,
internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
indices.length());
- const index_c_type* in =
- reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
- indices.offset();
+ const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
+
uint8_t* out = output->buffers[1]->mutable_data();
int32_t byte_width =
static_cast<const FixedSizeBinaryType&>(*output->type).byte_width();
@@ -336,9 +479,7 @@ Status UnpackBinaryDictionary(FunctionContext* ctx, const Array& indices,
internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
indices.length());
- const index_c_type* in =
- reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
- indices.offset();
+ const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
for (int64_t i = 0; i < indices.length(); ++i) {
if (valid_bits_reader.IsSet()) {
int32_t length;
@@ -409,9 +550,7 @@ void UnpackPrimitiveDictionary(const Array& indices, const c_type* dictionary,
internal::BitmapReader valid_bits_reader(indices.null_bitmap_data(), indices.offset(),
indices.length());
- const index_c_type* in =
- reinterpret_cast<const index_c_type*>(indices.data()->buffers[1]->data()) +
- indices.offset();
+ const index_c_type* in = GetValuesAs<index_c_type>(*indices.data(), 1);
for (int64_t i = 0; i < indices.length(); ++i) {
if (valid_bits_reader.IsSet()) {
out[i] = dictionary[in[i]];
@@ -436,9 +575,8 @@ struct CastFunctor<T, DictionaryType,
DCHECK(values_type.Equals(*output->type))
<< "Dictionary type: " << values_type << " target type: " << (*output->type);
- auto dictionary =
- reinterpret_cast<const c_type*>(type.dictionary()->data()->buffers[1]->data()) +
- type.dictionary()->offset();
+ const c_type* dictionary = GetValuesAs<c_type>(*type.dictionary()->data(), 1);
+
auto out = reinterpret_cast<c_type*>(output->buffers[1]->mutable_data());
const Array& indices = *dict_array.indices();
switch (indices.type()->id()) {
@@ -481,6 +619,9 @@ static Status AllocateIfNotPreallocated(FunctionContext* ctx, const Array& input
int64_t bitmap_size = BitUtil::BytesForBits(length);
RETURN_NOT_OK(ctx->Allocate(bitmap_size, &validity_bitmap));
memset(validity_bitmap->mutable_data(), 0, bitmap_size);
+ } else if (input.offset() != 0) {
+ RETURN_NOT_OK(CopyBitmap(ctx->memory_pool(), validity_bitmap->data(), input.offset(),
+ length, &validity_bitmap));
}
if (out->buffers.size() == 2) {
@@ -598,13 +739,21 @@ class CastKernel : public UnaryKernel {
FN(Int64Type, Time64Type); \
FN(Int64Type, Date64Type);
-#define DATE32_CASES(FN, IN_TYPE) FN(Date32Type, Date32Type);
+#define DATE32_CASES(FN, IN_TYPE) \
+ FN(Date32Type, Date32Type); \
+ FN(Date32Type, Date64Type);
-#define DATE64_CASES(FN, IN_TYPE) FN(Date64Type, Date64Type);
+#define DATE64_CASES(FN, IN_TYPE) \
+ FN(Date64Type, Date64Type); \
+ FN(Date64Type, Date32Type);
-#define TIME32_CASES(FN, IN_TYPE) FN(Time32Type, Time32Type);
+#define TIME32_CASES(FN, IN_TYPE) \
+ FN(Time32Type, Time32Type); \
+ FN(Time32Type, Time64Type);
-#define TIME64_CASES(FN, IN_TYPE) FN(Time64Type, Time64Type);
+#define TIME64_CASES(FN, IN_TYPE) \
+ FN(Time64Type, Time32Type); \
+ FN(Time64Type, Time64Type);
#define TIMESTAMP_CASES(FN, IN_TYPE) FN(TimestampType, TimestampType);
diff --git a/cpp/src/arrow/compute/cast.h b/cpp/src/arrow/compute/cast.h
index 7a07512..d7bde20 100644
--- a/cpp/src/arrow/compute/cast.h
+++ b/cpp/src/arrow/compute/cast.h
@@ -34,9 +34,10 @@ class FunctionContext;
class UnaryKernel;
struct CastOptions {
- CastOptions() : allow_int_overflow(false) {}
+ CastOptions() : allow_int_overflow(false), allow_time_truncate(false) {}
bool allow_int_overflow;
+ bool allow_time_truncate;
};
/// \since 0.7.0
diff --git a/cpp/src/arrow/compute/compute-test.cc b/cpp/src/arrow/compute/compute-test.cc
index 8a59517..8a7ef92 100644
--- a/cpp/src/arrow/compute/compute-test.cc
+++ b/cpp/src/arrow/compute/compute-test.cc
@@ -68,7 +68,7 @@ class TestCast : public ComputeFixture, public TestBase {
const std::shared_ptr<DataType>& out_type, const CastOptions& options) {
std::shared_ptr<Array> result;
ASSERT_OK(Cast(&ctx_, input, out_type, options, &result));
- AssertArraysEqual(expected, *result);
+ ASSERT_ARRAYS_EQUAL(expected, *result);
}
template <typename InType, typename I_TYPE>
@@ -105,6 +105,11 @@ class TestCast : public ComputeFixture, public TestBase {
ArrayFromVector<OutType, O_TYPE>(out_type, out_values, &expected);
}
CheckPass(*input, *expected, out_type, options);
+
+ // Check a sliced variant
+ if (input->length() > 1) {
+ CheckPass(*input->Slice(1), *expected->Slice(1), out_type, options);
+ }
}
};
@@ -270,6 +275,205 @@ TEST_F(TestCast, ToIntDowncastUnsafe) {
options);
}
+TEST_F(TestCast, TimestampToTimestamp) {
+ CastOptions options;
+
+ auto CheckTimestampCast = [this](
+ const CastOptions& options, TimeUnit::type from_unit, TimeUnit::type to_unit,
+ const std::vector<int64_t>& from_values, const std::vector<int64_t>& to_values,
+ const std::vector<bool>& is_valid) {
+ CheckCase<TimestampType, int64_t, TimestampType, int64_t>(
+ timestamp(from_unit), from_values, is_valid, timestamp(to_unit), to_values,
+ options);
+ };
+
+ vector<bool> is_valid = {true, false, true, true, true};
+
+ // Multiply promotions
+ vector<int64_t> v1 = {0, 100, 200, 1, 2};
+ vector<int64_t> e1 = {0, 100000, 200000, 1000, 2000};
+ CheckTimestampCast(options, TimeUnit::SECOND, TimeUnit::MILLI, v1, e1, is_valid);
+
+ vector<int64_t> v2 = {0, 100, 200, 1, 2};
+ vector<int64_t> e2 = {0, 100000000L, 200000000L, 1000000, 2000000};
+ CheckTimestampCast(options, TimeUnit::SECOND, TimeUnit::MICRO, v2, e2, is_valid);
+
+ vector<int64_t> v3 = {0, 100, 200, 1, 2};
+ vector<int64_t> e3 = {0, 100000000000L, 200000000000L, 1000000000L, 2000000000L};
+ CheckTimestampCast(options, TimeUnit::SECOND, TimeUnit::NANO, v3, e3, is_valid);
+
+ vector<int64_t> v4 = {0, 100, 200, 1, 2};
+ vector<int64_t> e4 = {0, 100000, 200000, 1000, 2000};
+ CheckTimestampCast(options, TimeUnit::MILLI, TimeUnit::MICRO, v4, e4, is_valid);
+
+ vector<int64_t> v5 = {0, 100, 200, 1, 2};
+ vector<int64_t> e5 = {0, 100000000L, 200000000L, 1000000, 2000000};
+ CheckTimestampCast(options, TimeUnit::MILLI, TimeUnit::NANO, v5, e5, is_valid);
+
+ vector<int64_t> v6 = {0, 100, 200, 1, 2};
+ vector<int64_t> e6 = {0, 100000, 200000, 1000, 2000};
+ CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::NANO, v6, e6, is_valid);
+
+ // Zero copy
+ std::shared_ptr<Array> arr;
+ vector<int64_t> v7 = {0, 70000, 2000, 1000, 0};
+ ArrayFromVector<TimestampType, int64_t>(timestamp(TimeUnit::SECOND), is_valid, v7,
+ &arr);
+ CheckZeroCopy(*arr, timestamp(TimeUnit::SECOND));
+
+ // Divide, truncate
+ vector<int64_t> v8 = {0, 100123, 200456, 1123, 2456};
+ vector<int64_t> e8 = {0, 100, 200, 1, 2};
+
+ options.allow_time_truncate = true;
+ CheckTimestampCast(options, TimeUnit::MILLI, TimeUnit::SECOND, v8, e8, is_valid);
+ CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::MILLI, v8, e8, is_valid);
+ CheckTimestampCast(options, TimeUnit::NANO, TimeUnit::MICRO, v8, e8, is_valid);
+
+ vector<int64_t> v9 = {0, 100123000, 200456000, 1123000, 2456000};
+ vector<int64_t> e9 = {0, 100, 200, 1, 2};
+ CheckTimestampCast(options, TimeUnit::MICRO, TimeUnit::SECOND, v9, e9, is_valid);
+ CheckTimestampCast(options, TimeUnit::NANO, TimeUnit::MILLI, v9, e9, is_valid);
+
+ vector<int64_t> v10 = {0, 100123000000L, 200456000000L, 1123000000L, 2456000000};
+ vector<int64_t> e10 = {0, 100, 200, 1, 2};
+ CheckTimestampCast(options, TimeUnit::NANO, TimeUnit::SECOND, v10, e10, is_valid);
+
+ // Disallow truncate, failures
+ options.allow_time_truncate = false;
+ CheckFails<TimestampType>(timestamp(TimeUnit::MILLI), v8, is_valid,
+ timestamp(TimeUnit::SECOND), options);
+ CheckFails<TimestampType>(timestamp(TimeUnit::MICRO), v8, is_valid,
+ timestamp(TimeUnit::MILLI), options);
+ CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v8, is_valid,
+ timestamp(TimeUnit::MICRO), options);
+ CheckFails<TimestampType>(timestamp(TimeUnit::MICRO), v9, is_valid,
+ timestamp(TimeUnit::SECOND), options);
+ CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v9, is_valid,
+ timestamp(TimeUnit::MILLI), options);
+ CheckFails<TimestampType>(timestamp(TimeUnit::NANO), v10, is_valid,
+ timestamp(TimeUnit::SECOND), options);
+}
+
+TEST_F(TestCast, TimeToTime) {
+ CastOptions options;
+
+ vector<bool> is_valid = {true, false, true, true, true};
+
+ // Multiply promotions
+ vector<int32_t> v1 = {0, 100, 200, 1, 2};
+ vector<int32_t> e1 = {0, 100000, 200000, 1000, 2000};
+ CheckCase<Time32Type, int32_t, Time32Type, int32_t>(
+ time32(TimeUnit::SECOND), v1, is_valid, time32(TimeUnit::MILLI), e1, options);
+
+ vector<int32_t> v2 = {0, 100, 200, 1, 2};
+ vector<int64_t> e2 = {0, 100000000L, 200000000L, 1000000, 2000000};
+ CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+ time32(TimeUnit::SECOND), v2, is_valid, time64(TimeUnit::MICRO), e2, options);
+
+ vector<int32_t> v3 = {0, 100, 200, 1, 2};
+ vector<int64_t> e3 = {0, 100000000000L, 200000000000L, 1000000000L, 2000000000L};
+ CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+ time32(TimeUnit::SECOND), v3, is_valid, time64(TimeUnit::NANO), e3, options);
+
+ vector<int32_t> v4 = {0, 100, 200, 1, 2};
+ vector<int64_t> e4 = {0, 100000, 200000, 1000, 2000};
+ CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+ time32(TimeUnit::MILLI), v4, is_valid, time64(TimeUnit::MICRO), e4, options);
+
+ vector<int32_t> v5 = {0, 100, 200, 1, 2};
+ vector<int64_t> e5 = {0, 100000000L, 200000000L, 1000000, 2000000};
+ CheckCase<Time32Type, int32_t, Time64Type, int64_t>(
+ time32(TimeUnit::MILLI), v5, is_valid, time64(TimeUnit::NANO), e5, options);
+
+ vector<int64_t> v6 = {0, 100, 200, 1, 2};
+ vector<int64_t> e6 = {0, 100000, 200000, 1000, 2000};
+ CheckCase<Time64Type, int64_t, Time64Type, int64_t>(
+ time64(TimeUnit::MICRO), v6, is_valid, time64(TimeUnit::NANO), e6, options);
+
+ // Zero copy
+ std::shared_ptr<Array> arr;
+ vector<int64_t> v7 = {0, 70000, 2000, 1000, 0};
+ ArrayFromVector<Time64Type, int64_t>(time64(TimeUnit::MICRO), is_valid, v7, &arr);
+ CheckZeroCopy(*arr, time64(TimeUnit::MICRO));
+
+ // Divide, truncate
+ vector<int32_t> v8 = {0, 100123, 200456, 1123, 2456};
+ vector<int32_t> e8 = {0, 100, 200, 1, 2};
+
+ options.allow_time_truncate = true;
+ CheckCase<Time32Type, int32_t, Time32Type, int32_t>(
+ time32(TimeUnit::MILLI), v8, is_valid, time32(TimeUnit::SECOND), e8, options);
+ CheckCase<Time64Type, int32_t, Time32Type, int32_t>(
+ time64(TimeUnit::MICRO), v8, is_valid, time32(TimeUnit::MILLI), e8, options);
+ CheckCase<Time64Type, int32_t, Time64Type, int32_t>(
+ time64(TimeUnit::NANO), v8, is_valid, time64(TimeUnit::MICRO), e8, options);
+
+ vector<int64_t> v9 = {0, 100123000, 200456000, 1123000, 2456000};
+ vector<int32_t> e9 = {0, 100, 200, 1, 2};
+ CheckCase<Time64Type, int64_t, Time32Type, int32_t>(
+ time64(TimeUnit::MICRO), v9, is_valid, time32(TimeUnit::SECOND), e9, options);
+ CheckCase<Time64Type, int64_t, Time32Type, int32_t>(
+ time64(TimeUnit::NANO), v9, is_valid, time32(TimeUnit::MILLI), e9, options);
+
+ vector<int64_t> v10 = {0, 100123000000L, 200456000000L, 1123000000L, 2456000000};
+ vector<int32_t> e10 = {0, 100, 200, 1, 2};
+ CheckCase<Time64Type, int64_t, Time32Type, int32_t>(
+ time64(TimeUnit::NANO), v10, is_valid, time32(TimeUnit::SECOND), e10, options);
+
+ // Disallow truncate, failures
+
+ options.allow_time_truncate = false;
+ CheckFails<Time32Type>(time32(TimeUnit::MILLI), v8, is_valid, time32(TimeUnit::SECOND),
+ options);
+ CheckFails<Time64Type>(time64(TimeUnit::MICRO), v8, is_valid, time32(TimeUnit::MILLI),
+ options);
+ CheckFails<Time64Type>(time64(TimeUnit::NANO), v8, is_valid, time64(TimeUnit::MICRO),
+ options);
+ CheckFails<Time64Type>(time64(TimeUnit::MICRO), v9, is_valid, time32(TimeUnit::SECOND),
+ options);
+ CheckFails<Time64Type>(time64(TimeUnit::NANO), v9, is_valid, time32(TimeUnit::MILLI),
+ options);
+ CheckFails<Time64Type>(time64(TimeUnit::NANO), v10, is_valid, time32(TimeUnit::SECOND),
+ options);
+}
+
+TEST_F(TestCast, DateToDate) {
+ CastOptions options;
+
+ vector<bool> is_valid = {true, false, true, true, true};
+
+ constexpr int64_t F = 86400000;
+
+ // Multiply promotion
+ vector<int32_t> v1 = {0, 100, 200, 1, 2};
+ vector<int64_t> e1 = {0, 100 * F, 200 * F, F, 2 * F};
+ CheckCase<Date32Type, int32_t, Date64Type, int64_t>(date32(), v1, is_valid, date64(),
+ e1, options);
+
+ // Zero copy
+ std::shared_ptr<Array> arr;
+ vector<int32_t> v2 = {0, 70000, 2000, 1000, 0};
+ vector<int64_t> v3 = {0, 70000, 2000, 1000, 0};
+ ArrayFromVector<Date32Type, int32_t>(date32(), is_valid, v2, &arr);
+ CheckZeroCopy(*arr, date32());
+
+ ArrayFromVector<Date64Type, int64_t>(date64(), is_valid, v3, &arr);
+ CheckZeroCopy(*arr, date64());
+
+ // Divide, truncate
+ vector<int64_t> v8 = {0, 100 * F + 123, 200 * F + 456, F + 123, 2 * F + 456};
+ vector<int32_t> e8 = {0, 100, 200, 1, 2};
+
+ options.allow_time_truncate = true;
+ CheckCase<Date64Type, int64_t, Date32Type, int32_t>(date64(), v8, is_valid, date32(),
+ e8, options);
+
+ // Disallow truncate, failures
+ options.allow_time_truncate = false;
+ CheckFails<Date64Type>(date64(), v8, is_valid, date32(), options);
+}
+
TEST_F(TestCast, ToDouble) {
CastOptions options;
vector<bool> is_valid = {true, false, true, true, true};
@@ -335,7 +539,7 @@ TEST_F(TestCast, FromNull) {
ASSERT_EQ(length, result->null_count());
// OK to look at bitmaps
- AssertArraysEqual(*result, *result);
+ ASSERT_ARRAYS_EQUAL(*result, *result);
}
TEST_F(TestCast, PreallocatedMemory) {
@@ -373,7 +577,7 @@ TEST_F(TestCast, PreallocatedMemory) {
std::shared_ptr<Array> expected;
ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, e1, &expected);
- AssertArraysEqual(*expected, *result);
+ ASSERT_ARRAYS_EQUAL(*expected, *result);
}
template <typename TestType>
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 83ebdea..044fb94 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -281,15 +281,20 @@ Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>&
return builder->Finish(out);
}
-void AssertArraysEqual(const Array& expected, const Array& actual) {
- if (!actual.Equals(expected)) {
- std::stringstream pp_result;
- std::stringstream pp_expected;
+#define ASSERT_ARRAYS_EQUAL(LEFT, RIGHT) \
+ do { \
+ if (!(LEFT).Equals((RIGHT))) { \
+ std::stringstream pp_result; \
+ std::stringstream pp_expected; \
+ \
+ EXPECT_OK(PrettyPrint(RIGHT, 0, &pp_result)); \
+ EXPECT_OK(PrettyPrint(LEFT, 0, &pp_expected)); \
+ FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str(); \
+ } \
+ } while (false)
- EXPECT_OK(PrettyPrint(actual, 0, &pp_result));
- EXPECT_OK(PrettyPrint(expected, 0, &pp_expected));
- FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str();
- }
+void AssertArraysEqual(const Array& expected, const Array& actual) {
+ ASSERT_ARRAYS_EQUAL(expected, actual);
}
#define ASSERT_BATCHES_EQUAL(LEFT, RIGHT) \
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 2030f37..446f4d3 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -228,7 +228,11 @@ class ARROW_EXPORT FloatingPoint : public Number {
virtual Precision precision() const = 0;
};
-class ARROW_EXPORT NestedType : public DataType {
+/// \class ParametricType
+/// \brief A superclass for types having additional metadata
+class ParametricType {};
+
+class ARROW_EXPORT NestedType : public DataType, public ParametricType {
public:
using DataType::DataType;
};
@@ -444,7 +448,7 @@ class ARROW_EXPORT BinaryType : public DataType, public NoExtraMeta {
};
// BinaryType type is represents lists of 1-byte values.
-class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType {
+class ARROW_EXPORT FixedSizeBinaryType : public FixedWidthType, public ParametricType {
public:
static constexpr Type::type type_id = Type::FIXED_SIZE_BINARY;
@@ -611,7 +615,7 @@ static inline std::ostream& operator<<(std::ostream& os, TimeUnit::type unit) {
return os;
}
-class ARROW_EXPORT TimeType : public FixedWidthType {
+class ARROW_EXPORT TimeType : public FixedWidthType, public ParametricType {
public:
TimeUnit::type unit() const { return unit_; }
@@ -650,7 +654,7 @@ class ARROW_EXPORT Time64Type : public TimeType {
std::string name() const override { return "time64"; }
};
-class ARROW_EXPORT TimestampType : public FixedWidthType {
+class ARROW_EXPORT TimestampType : public FixedWidthType, public ParametricType {
public:
using Unit = TimeUnit;
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index c596d2a..72262f0 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -260,8 +260,8 @@ cdef class Array:
type = _ensure_type(target_type)
- if not safe:
- options.allow_int_overflow = 1
+ options.allow_int_overflow = not safe
+ options.allow_time_truncate = not safe
with nogil:
check_status(Cast(_context(), self.ap[0], type.sp_type,
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 0e5d4a8..809bb96 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -747,6 +747,7 @@ cdef extern from "arrow/compute/api.h" namespace "arrow::compute" nogil:
cdef cppclass CCastOptions" arrow::compute::CastOptions":
c_bool allow_int_overflow
+ c_bool allow_time_truncate
CStatus Cast(CFunctionContext* context, const CArray& array,
const shared_ptr[CDataType]& to_type,
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 418076f..e3a4c97 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -290,6 +290,32 @@ def test_cast_integers_unsafe():
_check_cast_case(case, safe=False)
+def test_cast_timestamp_unit():
+ # ARROW-1680
+ val = datetime.datetime.now()
+ s = pd.Series([val])
+ s_nyc = s.dt.tz_localize('tzlocal()').dt.tz_convert('America/New_York')
+
+ us_with_tz = pa.timestamp('us', tz='America/New_York')
+ arr = pa.Array.from_pandas(s_nyc, type=us_with_tz)
+
+ arr2 = pa.Array.from_pandas(s, type=pa.timestamp('us'))
+
+ assert arr[0].as_py() == s_nyc[0]
+ assert arr2[0].as_py() == s[0]
+
+ # Disallow truncation
+ arr = pa.array([123123], type='int64').cast(pa.timestamp('ms'))
+ expected = pa.array([123], type='int64').cast(pa.timestamp('s'))
+
+ target = pa.timestamp('s')
+ with pytest.raises(ValueError):
+ arr.cast(target)
+
+ result = arr.cast(target, safe=False)
+ assert result.equals(expected)
+
+
def test_cast_signed_to_unsigned():
safe_cases = [
(np.array([0, 1, 2, 3], dtype='i1'), pa.uint8(),
--
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <co...@arrow.apache.org>'].