You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "pitrou (via GitHub)" <gi...@apache.org> on 2023/08/10 14:20:16 UTC
[GitHub] [arrow] pitrou commented on a diff in pull request #36846: GH-36845: [C++][Python] Allow type promotion on `pa.concat_tables`
pitrou commented on code in PR #36846:
URL: https://github.com/apache/arrow/pull/36846#discussion_r1290143821
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
Review Comment:
Looks like this doesn't include all fields? `promote_dictionary_ordered` is missing at least.
##########
cpp/src/arrow/table.h:
##########
@@ -23,6 +23,7 @@
#include <vector>
#include "arrow/chunked_array.h" // IWYU pragma: keep
+#include "arrow/compute/cast.h"
Review Comment:
It would be better to avoid pulling compute headers in this `.h` file.
Perhaps you can forward-declare `CastOptions` and pass it by const-ref:
```c++
namespace compute {
class CastOptions
};
ARROW_EXPORT
Result<std::shared_ptr<Table>> PromoteTableToSchema(
const std::shared_ptr<Table>& table, const std::shared_ptr<Schema>& schema,
MemoryPool* pool = default_memory_pool());
ARROW_EXPORT
Result<std::shared_ptr<Table>> PromoteTableToSchema(
const std::shared_ptr<Table>& table, const std::shared_ptr<Schema>& schema,
const compute::CastOptions&, MemoryPool* pool = default_memory_pool());
```
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
Review Comment:
```suggestion
return Status::TypeError("Could not merge dictionary index types");
```
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
Review Comment:
Same here (`promote_numeric_width`), also, I'm unconvinced by the bit width equivalence between integer and floating.
For example, an int32 cannot always be represented exactly in the 24 bits of a float32 mantissa.
So I would say (assuming `promote_numeric_width` is true)
* int width <= 8 ⇒ promoted float width = max(16, other float width)
* int width <= 16 ⇒ promoted float width = max(32, other float width)
* otherwise ⇒ promoted float width = max(64, other float width)
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
Review Comment:
This could simply call `MaybeMergeNumericTypes`?
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
Review Comment:
Shouldn't this be conditioned on `promote_numeric_width`?
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
Review Comment:
Note the `promote_numeric_width` check is missing here, so better factor it out into `WidenDecimals`?
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ }
+
+ if (options.promote_numeric_width) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ if (is_floating(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ } else if (is_signed_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ } else if (is_unsigned_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = uint64();
+ } else if (max_width >= 32) {
+ promoted_type = uint32();
+ } else if (max_width >= 16) {
+ promoted_type = uint16();
+ } else {
+ promoted_type = uint8();
+ }
+ }
+ }
+
+ return promoted ? promoted_type : nullptr;
+}
+
+// Merge temporal types based on options. Returns nullptr for non-binary types.
+Result<std::shared_ptr<DataType>> MaybeMergeBinaryTypes(
+ std::shared_ptr<DataType>& promoted_type, std::shared_ptr<DataType>& other_type,
+ const Field::MergeOptions& options) {
+ if (options.promote_binary) {
+ if (other_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(promoted_type->id())) {
+ return MakeBinary(*promoted_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ other_type->id() == Type::FIXED_SIZE_BINARY) {
+ return binary();
+ } else if (is_string(promoted_type->id()) && is_binary(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (is_binary(promoted_type->id()) && is_string(other_type->id())) {
+ return MakeBinary(*promoted_type);
+ }
+
+ if ((promoted_type->id() == Type::STRING && other_type->id() == Type::LARGE_STRING) ||
+ (promoted_type->id() == Type::LARGE_STRING && other_type->id() == Type::STRING)) {
+ return large_utf8();
+ } else if ((promoted_type->id() == Type::BINARY &&
+ other_type->id() == Type::LARGE_BINARY) ||
+ (promoted_type->id() == Type::LARGE_BINARY &&
+ other_type->id() == Type::BINARY)) {
+ return large_binary();
+ }
+ }
+
+ return nullptr;
+}
+
+// Merge list types based on options. Returns nullptr for non-list types.
+Result<std::shared_ptr<DataType>> MaybeMergeListTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ bool promoted = false;
+
+ if (promoted_type->id() == Type::FIXED_SIZE_LIST &&
+ other_type->id() == Type::FIXED_SIZE_LIST) {
+ const auto& left = checked_cast<const FixedSizeListType&>(*promoted_type);
+ const auto& right = checked_cast<const FixedSizeListType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto value_field,
+ left.value_field()->MergeWith(
+ *right.value_field()->WithName(left.value_field()->name()), options));
+ if (left.list_size() == right.list_size()) {
+ return fixed_size_list(std::move(value_field), left.list_size());
+ }
+ return Status::TypeError("Cannot merge fixed_size_list of different sizes");
+ } else if (promoted_type->id() == Type::MAP && other_type->id() == Type::MAP) {
+ const auto& left = checked_cast<const MapType&>(*promoted_type);
+ const auto& right = checked_cast<const MapType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto key_field,
+ left.key_field()->MergeWith(
+ *right.key_field()->WithName(left.key_field()->name()), options));
+ ARROW_ASSIGN_OR_RAISE(
+ auto item_field,
+ left.item_field()->MergeWith(
+ *right.item_field()->WithName(left.item_field()->name()), options));
+ return map(std::move(key_field->type()), std::move(item_field),
+ /*keys_sorted=*/left.keys_sorted() && right.keys_sorted());
+ } else if (is_list(promoted_type->id()) && is_list(other_type->id())) {
+ const auto& left = checked_cast<const BaseListType&>(*promoted_type);
+ const auto& right = checked_cast<const BaseListType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto value_field,
+ left.value_field()->MergeWith(
+ *right.value_field()->WithName(left.value_field()->name()), options));
+ if (promoted_type->id() == Type::LARGE_LIST || other_type->id() == Type::LARGE_LIST) {
+ return large_list(std::move(value_field));
+ } else {
+ return list(std::move(value_field));
+ }
+ } else if (promoted_type->id() == Type::STRUCT && other_type->id() == Type::STRUCT) {
+ SchemaBuilder builder(SchemaBuilder::CONFLICT_APPEND, options);
+ // Add the LHS fields. Duplicates will be preserved.
+ RETURN_NOT_OK(builder.AddFields(promoted_type->fields()));
+
+ // Add the RHS fields. Duplicates will be merged, unless the field was
+ // already a duplicate, in which case we error (since we don't know which
+ // field to merge with).
+ builder.SetPolicy(SchemaBuilder::CONFLICT_MERGE);
+ RETURN_NOT_OK(builder.AddFields(other_type->fields()));
+
+ ARROW_ASSIGN_OR_RAISE(auto schema, builder.Finish());
+ return struct_(schema->fields());
+ }
+
+ return promoted ? promoted_type : nullptr;
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ if (promoted_type->Equals(*other_type)) return promoted_type;
+
+ bool promoted = false;
+ if (options.promote_nullability) {
+ if (promoted_type->id() == Type::NA) {
+ return other_type;
+ } else if (other_type->id() == Type::NA) {
+ return promoted_type;
+ }
+ } else if (promoted_type->id() == Type::NA || other_type->id() == Type::NA) {
+ return Status::Invalid("Cannot merge type with null unless promote_nullability=true");
Review Comment:
`TypeError` here as well?
##########
python/pyarrow/table.pxi:
##########
@@ -5066,6 +5073,16 @@ def concat_tables(tables, c_bool promote=False, MemoryPool memory_pool=None):
for table in tables:
c_tables.push_back(table.sp_table)
+ if field_merge_options is not None:
+ if isinstance(field_merge_options, str):
+ if field_merge_options == "permissive":
+ options.field_merge_options = CField.CMergeOptions.Permissive()
+ elif field_merge_options == "default":
+ options.field_merge_options = CField.CMergeOptions.Defaults()
+ else:
+ options.field_merge_options = cython.cast(
+ FieldMergeOptions, field_merge_options).c_options
Review Comment:
You can simply write:
```suggestion
options.field_merge_options = (<FieldMergeOptions> field_merge_options).c_options
```
... and remove the `import cython` at the top.
##########
cpp/src/arrow/type_test.cc:
##########
@@ -892,6 +892,150 @@ class TestUnifySchemas : public TestSchema {
<< lhs_field->ToString() << " vs " << rhs_field->ToString();
}
}
+
+ void CheckUnifyAsymmetric(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_OK_AND_ASSIGN(auto merged, field1->MergeWith(field2, options));
+ AssertFieldEqual(merged, expected);
+ }
+
+ void CheckUnify(const std::shared_ptr<Field>& field1,
+ const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ CheckUnifyAsymmetric(field1, field2, expected, options);
+ CheckUnifyAsymmetric(field2, field1, expected, options);
+ }
+
+ void CheckUnifyFailsInvalid(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_RAISES(Invalid, field1->MergeWith(field2, options));
+ ASSERT_RAISES(Invalid, field2->MergeWith(field1, options));
+ }
Review Comment:
I think it would be nicer to also check a part of the error message. For example (untested, but you get the idea):
```suggestion
void CheckUnifyFailsInvalid(
const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
const Field::MergeOptions& options = Field::MergeOptions::Defaults(),
const std::string& match_message = "") {
ARROW_SCOPED_TRACE("options: ", options);
ARROW_SCOPED_TRACE("field2: ", field2->ToString());
ARROW_SCOPED_TRACE("field1: ", field1->ToString());
EXPECT_RAISES_WITH_MESSAGE_THAT(
Invalid, ::testing::HasSubstr(match_message), field1->MergeWith(field2, options));
EXPECT_RAISES_WITH_MESSAGE_THAT(
Invalid, ::testing::HasSubstr(match_message), field2->MergeWith(field1, options));
}
```
##########
cpp/src/arrow/type.h:
##########
@@ -397,14 +397,63 @@ class ARROW_EXPORT Field : public detail::Fingerprintable,
/// \brief Options that control the behavior of `MergeWith`.
/// Options are to be added to allow type conversions, including integer
/// widening, promotion from integer to float, or conversion to or from boolean.
- struct MergeOptions {
+ struct ARROW_EXPORT MergeOptions : public util::ToStringOstreamable<MergeOptions> {
/// If true, a Field of NullType can be unified with a Field of another type.
/// The unified field will be of the other type and become nullable.
/// Nullability will be promoted to the looser option (nullable if one is not
/// nullable).
bool promote_nullability = true;
+ /// Allow a decimal to be unified with another decimal of the same
+ /// width, adjusting scale and precision as appropriate. May fail
+ /// if the adjustment is not possible.
+ bool promote_decimal = false;
+
+ /// Allow a decimal to be promoted to a float. The float type will
+ /// not itself be promoted (e.g. Decimal128 + Float32 = Float32).
+ bool promote_decimal_to_float = false;
+
+ /// Allow an integer to be promoted to a decimal.
+ ///
+ /// May fail if the decimal has insufficient precision to
+ /// accommodate the integer. (See increase_decimal_precision.)
+ bool promote_integer_to_decimal = false;
+
+ /// Allow an integer of a given bit width to be promoted to a
+ /// float; the result will be a float of an equal or greater bit
+ /// width to both of the inputs.
+ bool promote_integer_to_float = false;
+
+ /// Allow an unsigned integer of a given bit width to be promoted
+ /// to a signed integer of the equal or greater bit width.
+ bool promote_integer_sign = false;
+
+ /// Allow an integer, float, or decimal of a given bit width to be
+ /// promoted to an equivalent type of a greater bit width.
+ bool promote_numeric_width = false;
+
+ /// Allow strings to be promoted to binary types.
+ bool promote_binary = false;
Review Comment:
Call this `promote_string_to_binary` for clarity?
##########
python/pyarrow/types.pxi:
##########
@@ -1969,6 +1969,48 @@ cpdef KeyValueMetadata ensure_metadata(object meta, c_bool allow_none=False):
return KeyValueMetadata(meta)
+cdef class FieldMergeOptions(_Weakrefable):
+ """
+ Options controlling how to merge the types of two fields.
+
+ By default, types must match exactly, except the null type can be
+ merged with any other type.
+
+ """
+
+ cdef:
+ CField.CMergeOptions c_options
+
+ __slots__ = ()
+
+ def __init__(self, *):
+ self.c_options = CField.CMergeOptions.Defaults()
+
+ @staticmethod
+ def permissive():
+ """
+ Allow merging generally compatible types (e.g. float64 and int64).
+ """
+ cdef FieldMergeOptions options = FieldMergeOptions()
+ options.c_options = CField.CMergeOptions.Permissive()
+ return options
+
+ def __repr__(self) -> str:
+ return 'FieldMergeOptions(promote_nullability={},promote_decimal={},promote_decimal_to_float={},promote_integer_to_decimal={},promote_integer_to_float={},promote_integer_sign={},promote_numeric_width={},promote_binary={},promote_temporal_unit={},promote_dictionary={},promote_dictionary_ordered={})'.format(
Review Comment:
Two things:
1) can you please ensure lines are not too long?
2) you can use a f-string here, for example:
```python
return (
f'FieldMergeOptions(promote_nullability={self.promote_nullability}, '
f'promote_decimal={self.promote_decimal}, '
...)
```
##########
python/pyarrow/types.pxi:
##########
@@ -3128,6 +3170,8 @@ def unify_schemas(schemas):
----------
schemas : list of Schema
Schemas to merge into a single one.
+ options : FieldMergeOptions, optional
Review Comment:
Here as well, it would be nice to be able to pass this as a string ("default" or "permissive").
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
Review Comment:
```suggestion
return Status::Invalid("Could not merge dictionary value types");
```
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
Review Comment:
Should we instead return an error if `options.promote_numeric_width` is false?
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ }
+
+ if (options.promote_numeric_width) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ if (is_floating(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ } else if (is_signed_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ } else if (is_unsigned_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = uint64();
+ } else if (max_width >= 32) {
+ promoted_type = uint32();
+ } else if (max_width >= 16) {
+ promoted_type = uint16();
+ } else {
+ promoted_type = uint8();
+ }
+ }
+ }
+
+ return promoted ? promoted_type : nullptr;
+}
+
+// Merge temporal types based on options. Returns nullptr for non-binary types.
+Result<std::shared_ptr<DataType>> MaybeMergeBinaryTypes(
+ std::shared_ptr<DataType>& promoted_type, std::shared_ptr<DataType>& other_type,
+ const Field::MergeOptions& options) {
+ if (options.promote_binary) {
+ if (other_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(promoted_type->id())) {
+ return MakeBinary(*promoted_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ other_type->id() == Type::FIXED_SIZE_BINARY) {
+ return binary();
+ } else if (is_string(promoted_type->id()) && is_binary(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (is_binary(promoted_type->id()) && is_string(other_type->id())) {
+ return MakeBinary(*promoted_type);
+ }
+
+ if ((promoted_type->id() == Type::STRING && other_type->id() == Type::LARGE_STRING) ||
+ (promoted_type->id() == Type::LARGE_STRING && other_type->id() == Type::STRING)) {
+ return large_utf8();
+ } else if ((promoted_type->id() == Type::BINARY &&
+ other_type->id() == Type::LARGE_BINARY) ||
+ (promoted_type->id() == Type::LARGE_BINARY &&
+ other_type->id() == Type::BINARY)) {
+ return large_binary();
+ }
+ }
+
+ return nullptr;
+}
+
+// Merge list types based on options. Returns nullptr for non-list types.
+Result<std::shared_ptr<DataType>> MaybeMergeListTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ bool promoted = false;
+
Review Comment:
I would suggest a `promote_list` option to enable changing the list kind: for example from list to large_list. Doing it by default would be surprising to users IMHO.
The value type would always be promoted recursively, however.
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
Review Comment:
Do we want these checks to run even if `options.promote_temporal_unit` is false? This could allow for nicer error messages.
##########
cpp/src/arrow/type.h:
##########
@@ -397,14 +397,63 @@ class ARROW_EXPORT Field : public detail::Fingerprintable,
/// \brief Options that control the behavior of `MergeWith`.
/// Options are to be added to allow type conversions, including integer
/// widening, promotion from integer to float, or conversion to or from boolean.
- struct MergeOptions {
+ struct ARROW_EXPORT MergeOptions : public util::ToStringOstreamable<MergeOptions> {
/// If true, a Field of NullType can be unified with a Field of another type.
/// The unified field will be of the other type and become nullable.
/// Nullability will be promoted to the looser option (nullable if one is not
/// nullable).
bool promote_nullability = true;
+ /// Allow a decimal to be unified with another decimal of the same
+ /// width, adjusting scale and precision as appropriate. May fail
+ /// if the adjustment is not possible.
+ bool promote_decimal = false;
+
+ /// Allow a decimal to be promoted to a float. The float type will
+ /// not itself be promoted (e.g. Decimal128 + Float32 = Float32).
+ bool promote_decimal_to_float = false;
+
+ /// Allow an integer to be promoted to a decimal.
+ ///
+ /// May fail if the decimal has insufficient precision to
+ /// accommodate the integer. (See increase_decimal_precision.)
+ bool promote_integer_to_decimal = false;
+
+ /// Allow an integer of a given bit width to be promoted to a
+ /// float; the result will be a float of an equal or greater bit
+ /// width to both of the inputs.
+ bool promote_integer_to_float = false;
+
+ /// Allow an unsigned integer of a given bit width to be promoted
+ /// to a signed integer of the equal or greater bit width.
+ bool promote_integer_sign = false;
+
+ /// Allow an integer, float, or decimal of a given bit width to be
+ /// promoted to an equivalent type of a greater bit width.
+ bool promote_numeric_width = false;
+
+ /// Allow strings to be promoted to binary types.
+ bool promote_binary = false;
+
+ /// Second to millisecond, Time32 to Time64, Time32(SECOND) to Time32(MILLI), etc
+ bool promote_temporal_unit = false;
+
+ /// Promote dictionary index types to a common type, and unify the
+ /// value types.
Review Comment:
```suggestion
/// Unify dictionary index types and dictionary value types.
```
##########
cpp/src/arrow/type.h:
##########
@@ -397,14 +397,63 @@ class ARROW_EXPORT Field : public detail::Fingerprintable,
/// \brief Options that control the behavior of `MergeWith`.
/// Options are to be added to allow type conversions, including integer
/// widening, promotion from integer to float, or conversion to or from boolean.
- struct MergeOptions {
+ struct ARROW_EXPORT MergeOptions : public util::ToStringOstreamable<MergeOptions> {
/// If true, a Field of NullType can be unified with a Field of another type.
/// The unified field will be of the other type and become nullable.
/// Nullability will be promoted to the looser option (nullable if one is not
/// nullable).
bool promote_nullability = true;
+ /// Allow a decimal to be unified with another decimal of the same
+ /// width, adjusting scale and precision as appropriate. May fail
+ /// if the adjustment is not possible.
+ bool promote_decimal = false;
+
+ /// Allow a decimal to be promoted to a float. The float type will
+ /// not itself be promoted (e.g. Decimal128 + Float32 = Float32).
+ bool promote_decimal_to_float = false;
+
+ /// Allow an integer to be promoted to a decimal.
+ ///
+ /// May fail if the decimal has insufficient precision to
+ /// accommodate the integer. (See increase_decimal_precision.)
+ bool promote_integer_to_decimal = false;
+
+ /// Allow an integer of a given bit width to be promoted to a
+ /// float; the result will be a float of an equal or greater bit
+ /// width to both of the inputs.
+ bool promote_integer_to_float = false;
+
+ /// Allow an unsigned integer of a given bit width to be promoted
+ /// to a signed integer of the equal or greater bit width.
+ bool promote_integer_sign = false;
+
+ /// Allow an integer, float, or decimal of a given bit width to be
+ /// promoted to an equivalent type of a greater bit width.
+ bool promote_numeric_width = false;
+
+ /// Allow strings to be promoted to binary types.
+ bool promote_binary = false;
+
+ /// Second to millisecond, Time32 to Time64, Time32(SECOND) to Time32(MILLI), etc
+ bool promote_temporal_unit = false;
+
+ /// Promote dictionary index types to a common type, and unify the
+ /// value types.
+ bool promote_dictionary = false;
+
+ /// Allow merging ordered and non-ordered dictionaries, else
+ /// error. The result will be ordered if and only if both inputs
Review Comment:
"else error" is implied, isn't it? No need to spell it out explicitly, IMHO.
##########
python/pyarrow/tests/test_table.py:
##########
@@ -1373,6 +1373,15 @@ def test_concat_tables_with_promotion():
pa.array([None, None, 1.0, 2.0], type=pa.float32()),
], ["int64_field", "float_field"]))
+ t3 = pa.Table.from_arrays(
+ [pa.array([1, 2], type=pa.int32())], ["int64_field"])
+ result = pa.concat_tables(
+ [t1, t3], promote=True,
+ field_merge_options=pa.FieldMergeOptions.permissive())
Review Comment:
Can you add a test for it?
##########
cpp/src/arrow/table.cc:
##########
@@ -502,9 +504,16 @@ Result<std::shared_ptr<Table>> PromoteTableToSchema(const std::shared_ptr<Table>
continue;
}
- return Status::Invalid("Unable to promote field ", field->name(),
- ": incompatible types: ", field->type()->ToString(), " vs ",
- current_field->type()->ToString());
+ if (!compute::CanCast(*current_field->type(), *field->type())) {
+ return Status::Invalid("Unable to promote field ", field->name(),
Review Comment:
Did you forget to change this?
##########
cpp/src/arrow/type.h:
##########
@@ -397,14 +397,63 @@ class ARROW_EXPORT Field : public detail::Fingerprintable,
/// \brief Options that control the behavior of `MergeWith`.
/// Options are to be added to allow type conversions, including integer
/// widening, promotion from integer to float, or conversion to or from boolean.
- struct MergeOptions {
+ struct ARROW_EXPORT MergeOptions : public util::ToStringOstreamable<MergeOptions> {
/// If true, a Field of NullType can be unified with a Field of another type.
/// The unified field will be of the other type and become nullable.
/// Nullability will be promoted to the looser option (nullable if one is not
/// nullable).
bool promote_nullability = true;
+ /// Allow a decimal to be unified with another decimal of the same
+ /// width, adjusting scale and precision as appropriate. May fail
+ /// if the adjustment is not possible.
+ bool promote_decimal = false;
+
+ /// Allow a decimal to be promoted to a float. The float type will
+ /// not itself be promoted (e.g. Decimal128 + Float32 = Float32).
+ bool promote_decimal_to_float = false;
+
+ /// Allow an integer to be promoted to a decimal.
+ ///
+ /// May fail if the decimal has insufficient precision to
+ /// accommodate the integer. (See increase_decimal_precision.)
+ bool promote_integer_to_decimal = false;
+
+ /// Allow an integer of a given bit width to be promoted to a
+ /// float; the result will be a float of an equal or greater bit
+ /// width to both of the inputs.
+ bool promote_integer_to_float = false;
+
+ /// Allow an unsigned integer of a given bit width to be promoted
+ /// to a signed integer of the equal or greater bit width.
+ bool promote_integer_sign = false;
+
+ /// Allow an integer, float, or decimal of a given bit width to be
+ /// promoted to an equivalent type of a greater bit width.
+ bool promote_numeric_width = false;
+
+ /// Allow strings to be promoted to binary types.
+ bool promote_binary = false;
Review Comment:
Actually, this seems to be conditioning a lot of other promotions. So the name is fine but the docstring should be fixed, IMHO.
##########
cpp/src/arrow/table_test.cc:
##########
@@ -520,6 +522,36 @@ TEST_F(ConcatenateTablesWithPromotionTest, Simple) {
AssertTablesEqualUnorderedFields(*expected, *result);
}
+TEST_F(ConcatenateTablesWithPromotionTest, Unify) {
+ auto t1 = TableFromJSON(schema({field("f0", int32())}), {"[[0], [1]]"});
+ auto t2 = TableFromJSON(schema({field("f0", int64())}), {"[[2], [3]]"});
+ auto t3 = TableFromJSON(schema({field("f0", null())}), {"[[null], [null]]"});
+
+ auto expected_int64 =
+ TableFromJSON(schema({field("f0", int64())}), {"[[0], [1], [2], [3]]"});
+ auto expected_null =
+ TableFromJSON(schema({field("f0", int32())}), {"[[0], [1], [null], [null]]"});
+
+ ConcatenateTablesOptions options;
+ EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid,
+ ::testing::HasSubstr("Schema at index 1 was different"),
+ ConcatenateTables({t1, t2}, options));
+ EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid,
+ ::testing::HasSubstr("Schema at index 1 was different"),
+ ConcatenateTables({t1, t3}, options));
+
+ options.unify_schemas = true;
+ EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid,
+ ::testing::HasSubstr("Field f0 has incompatible types"),
+ ConcatenateTables({t1, t2}, options));
+ ASSERT_OK_AND_ASSIGN(auto actual, ConcatenateTables({t1, t3}, options));
+ AssertTablesEqual(*expected_null, *actual, /*same_chunk_layout=*/false);
+
+ options.field_merge_options.promote_numeric_width = true;
+ ASSERT_OK_AND_ASSIGN(actual, ConcatenateTables({t1, t2}, options));
+ AssertTablesEqual(*expected_int64, *actual, /*same_chunk_layout=*/false);
+}
Review Comment:
It looks like you haven't addressed this comment?
##########
cpp/src/arrow/type.cc:
##########
@@ -327,14 +709,27 @@ Result<std::shared_ptr<Field>> Field::MergeWith(const Field& other,
return Copy();
}
- if (options.promote_nullability) {
- if (type()->Equals(other.type())) {
- return Copy()->WithNullable(nullable() || other.nullable());
+ auto maybe_promoted_type = MergeTypes(type_, other.type(), options);
+ if (!maybe_promoted_type.ok()) {
+ return maybe_promoted_type.status().WithMessage(
+ "Unable to merge: Field ", name(),
+ " has incompatible types: ", type()->ToString(), " vs ", other.type()->ToString(),
+ ": ", maybe_promoted_type.status().message());
+ }
+ auto promoted_type = *std::move(maybe_promoted_type);
+ if (promoted_type) {
+ bool nullable = nullable_;
+ if (options.promote_nullability) {
+ nullable = nullable || other.nullable() || type_->id() == Type::NA ||
+ other.type()->id() == Type::NA;
+ } else if (nullable_ != other.nullable()) {
+ return Status::Invalid("Unable to merge: Field ", name(),
+ " has incompatible nullability: ", nullable_, " vs ",
+ other.nullable());
Review Comment:
We should probably change these errors to `TypeError`. It may slightly break user code, but for the better (so that one can distinguish between unsupported type casts and invalid value casts).
##########
cpp/src/arrow/type_test.cc:
##########
@@ -983,6 +1127,242 @@ TEST_F(TestUnifySchemas, MoreSchemas) {
utf8_field->WithNullable(true)}));
}
+TEST_F(TestUnifySchemas, Numeric) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ CheckUnify(uint8(),
+ {int8(), uint16(), int16(), uint32(), int32(), uint64(), int64(), float32(),
+ float64()},
+ options);
+ CheckUnify(int8(), {int16(), int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint16(),
+ {int16(), uint32(), int32(), uint64(), int64(), float32(), float64()},
+ options);
+ CheckUnify(int16(), {int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint32(), {int32(), uint64(), int64(), float32(), float64()}, options);
+ CheckUnify(int32(), {int64(), float32(), float64()}, options);
+ CheckUnify(uint64(), {int64(), float64()}, options);
+ CheckUnify(int64(), {float64()}, options);
+ CheckUnify(float16(), {float32(), float64()}, options);
+ CheckUnify(float32(), {float64()}, options);
+ CheckUnify(uint64(), float32(), float64(), options);
+ CheckUnify(int64(), float32(), float64(), options);
+
+ options.promote_integer_sign = false;
+ CheckUnify(uint8(), {uint16(), uint32(), uint64()}, options);
+ CheckUnify(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFailsInvalid(uint8(), {int8(), int16(), int32(), int64()}, options);
+ CheckUnify(uint16(), {uint32(), uint64()}, options);
+ CheckUnify(int16(), {int32(), int64()}, options);
+ CheckUnifyFailsInvalid(uint16(), {int16(), int32(), int64()}, options);
+ CheckUnify(uint32(), {uint64()}, options);
+ CheckUnify(int32(), {int64()}, options);
+ CheckUnifyFailsInvalid(uint32(), {int32(), int64()}, options);
+ CheckUnifyFailsInvalid(uint64(), {int64()}, options);
+
+ options.promote_integer_sign = true;
+ options.promote_integer_to_float = false;
+ CheckUnifyFailsInvalid(IntTypes(), FloatingPointTypes(), options);
+
+ options.promote_integer_to_float = true;
+ options.promote_numeric_width = false;
+ CheckUnifyFailsInvalid(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFailsInvalid(int16(), {int32(), int64()}, options);
+ CheckUnifyFailsInvalid(int32(), {int64()}, options);
+ CheckUnify(int32(), {float32()}, options);
+ CheckUnify(int64(), {float64()}, options);
+ CheckUnify(int32(), float16(), float32(), options);
+}
+
+TEST_F(TestUnifySchemas, Decimal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_decimal_to_float = true;
+ CheckUnify(decimal128(3, 2), {float32(), float64()}, options);
+ CheckUnify(decimal256(3, 2), {float32(), float64()}, options);
+
+ options.promote_integer_to_decimal = true;
+ CheckUnify(int32(), decimal128(3, 2), decimal128(12, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(10, 0), options);
+
+ options.promote_decimal = true;
+ CheckUnify(decimal128(3, 2), decimal128(5, 2), decimal128(5, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 3), decimal128(5, 3), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 1), decimal128(6, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, -2), decimal128(9, 2), options);
+ CheckUnify(decimal128(3, -2), decimal128(5, -2), decimal128(5, -2), options);
+ CheckUnify(decimal128(38, 10), decimal128(38, 5), decimal256(43, 10), options);
+
+ CheckUnify(decimal256(3, 2), decimal256(5, 2), decimal256(5, 2), options);
+ CheckUnify(decimal256(3, 2), decimal256(5, 3), decimal256(5, 3), options);
+ CheckUnify(decimal256(3, 2), decimal256(5, 1), decimal256(6, 2), options);
+ CheckUnify(decimal256(3, 2), decimal256(5, -2), decimal256(9, 2), options);
+ CheckUnify(decimal256(3, -2), decimal256(5, -2), decimal256(5, -2), options);
+
+ // int32() is essentially decimal128(10, 0)
+ CheckUnify(int32(), decimal128(3, 2), decimal128(12, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(10, 0), options);
+ CheckUnify(int64(), decimal128(38, 37), decimal256(56, 37), options);
+
+ CheckUnifyFailsTypeError(decimal256(1, 0), decimal128(1, 0), options);
+
+ options.promote_numeric_width = true;
+ CheckUnify(decimal128(3, 2), decimal256(5, 2), decimal256(5, 2), options);
+ CheckUnify(int32(), decimal128(38, 37), decimal256(47, 37), options);
+ CheckUnifyFailsInvalid(decimal128(38, 10), decimal256(76, 5), options);
+
+ CheckUnifyFailsInvalid(int64(), decimal256(76, 75), options);
+}
+
+TEST_F(TestUnifySchemas, Temporal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_temporal_unit = true;
+ CheckUnify(date32(), {date64()}, options);
+
+ CheckUnify(time32(TimeUnit::SECOND),
+ {time32(TimeUnit::MILLI), time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time32(TimeUnit::MILLI), {time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time64(TimeUnit::MICRO), {time64(TimeUnit::NANO)}, options);
+
+ CheckUnify(
+ duration(TimeUnit::SECOND),
+ {duration(TimeUnit::MILLI), duration(TimeUnit::MICRO), duration(TimeUnit::NANO)},
+ options);
+ CheckUnify(duration(TimeUnit::MILLI),
+ {duration(TimeUnit::MICRO), duration(TimeUnit::NANO)}, options);
+ CheckUnify(duration(TimeUnit::MICRO), {duration(TimeUnit::NANO)}, options);
+
+ CheckUnify(
+ timestamp(TimeUnit::SECOND),
+ {timestamp(TimeUnit::MILLI), timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)},
+ options);
+ CheckUnify(timestamp(TimeUnit::MILLI),
+ {timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)}, options);
+ CheckUnify(timestamp(TimeUnit::MICRO), {timestamp(TimeUnit::NANO)}, options);
+
+ CheckUnifyFailsTypeError(timestamp(TimeUnit::SECOND),
+ timestamp(TimeUnit::SECOND, "UTC"), options);
+ CheckUnifyFailsTypeError(timestamp(TimeUnit::SECOND, "America/New_York"),
+ timestamp(TimeUnit::SECOND, "UTC"), options);
+
+ options.promote_temporal_unit = false;
+ CheckUnifyFailsInvalid(timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO), options);
Review Comment:
(would expect `TypeError` as well as in all/most instances here)
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
Review Comment:
Same here (`promote_numeric_width`), also, I'm unconvinced by the bit width equivalence between integer and floating.
For example, an int32 cannot always be represented exactly in the 24 bits of a float32 mantissa.
So I would say (assuming `promote_numeric_width` is true)
* int width <= 8 ⇒ promoted float width = max(16, other float width)
* int width <= 16 ⇒ promoted float width = max(32, other float width)
* otherwise ⇒ promoted float width = max(64, other float width)
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ }
+
+ if (options.promote_numeric_width) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ if (is_floating(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ } else if (is_signed_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ } else if (is_unsigned_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = uint64();
+ } else if (max_width >= 32) {
+ promoted_type = uint32();
+ } else if (max_width >= 16) {
+ promoted_type = uint16();
+ } else {
+ promoted_type = uint8();
+ }
+ }
+ }
+
+ return promoted ? promoted_type : nullptr;
+}
+
+// Merge temporal types based on options. Returns nullptr for non-binary types.
+Result<std::shared_ptr<DataType>> MaybeMergeBinaryTypes(
+ std::shared_ptr<DataType>& promoted_type, std::shared_ptr<DataType>& other_type,
+ const Field::MergeOptions& options) {
+ if (options.promote_binary) {
+ if (other_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(promoted_type->id())) {
+ return MakeBinary(*promoted_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ other_type->id() == Type::FIXED_SIZE_BINARY) {
+ return binary();
+ } else if (is_string(promoted_type->id()) && is_binary(other_type->id())) {
+ return MakeBinary(*other_type);
Review Comment:
Hmm... so, if `promoted_type` is `LARGE_STRING` and `other_type` is `BINARY`, they will be unified into `BINARY`? I would expect `LARGE_BINARY`... I think you have to logically handle utf8-ness and offset width separately, if you want to avoid spelling out all combinations explicitly.
##########
cpp/src/arrow/type_test.cc:
##########
@@ -983,6 +1127,242 @@ TEST_F(TestUnifySchemas, MoreSchemas) {
utf8_field->WithNullable(true)}));
}
+TEST_F(TestUnifySchemas, Numeric) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ CheckUnify(uint8(),
+ {int8(), uint16(), int16(), uint32(), int32(), uint64(), int64(), float32(),
+ float64()},
+ options);
+ CheckUnify(int8(), {int16(), int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint16(),
+ {int16(), uint32(), int32(), uint64(), int64(), float32(), float64()},
+ options);
+ CheckUnify(int16(), {int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint32(), {int32(), uint64(), int64(), float32(), float64()}, options);
+ CheckUnify(int32(), {int64(), float32(), float64()}, options);
+ CheckUnify(uint64(), {int64(), float64()}, options);
+ CheckUnify(int64(), {float64()}, options);
+ CheckUnify(float16(), {float32(), float64()}, options);
+ CheckUnify(float32(), {float64()}, options);
+ CheckUnify(uint64(), float32(), float64(), options);
+ CheckUnify(int64(), float32(), float64(), options);
+
+ options.promote_integer_sign = false;
+ CheckUnify(uint8(), {uint16(), uint32(), uint64()}, options);
+ CheckUnify(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFailsInvalid(uint8(), {int8(), int16(), int32(), int64()}, options);
+ CheckUnify(uint16(), {uint32(), uint64()}, options);
+ CheckUnify(int16(), {int32(), int64()}, options);
+ CheckUnifyFailsInvalid(uint16(), {int16(), int32(), int64()}, options);
+ CheckUnify(uint32(), {uint64()}, options);
+ CheckUnify(int32(), {int64()}, options);
+ CheckUnifyFailsInvalid(uint32(), {int32(), int64()}, options);
+ CheckUnifyFailsInvalid(uint64(), {int64()}, options);
+
+ options.promote_integer_sign = true;
+ options.promote_integer_to_float = false;
+ CheckUnifyFailsInvalid(IntTypes(), FloatingPointTypes(), options);
+
+ options.promote_integer_to_float = true;
+ options.promote_numeric_width = false;
+ CheckUnifyFailsInvalid(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFailsInvalid(int16(), {int32(), int64()}, options);
+ CheckUnifyFailsInvalid(int32(), {int64()}, options);
+ CheckUnify(int32(), {float32()}, options);
+ CheckUnify(int64(), {float64()}, options);
+ CheckUnify(int32(), float16(), float32(), options);
+}
+
+TEST_F(TestUnifySchemas, Decimal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_decimal_to_float = true;
+ CheckUnify(decimal128(3, 2), {float32(), float64()}, options);
+ CheckUnify(decimal256(3, 2), {float32(), float64()}, options);
+
+ options.promote_integer_to_decimal = true;
+ CheckUnify(int32(), decimal128(3, 2), decimal128(12, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(10, 0), options);
+
+ options.promote_decimal = true;
+ CheckUnify(decimal128(3, 2), decimal128(5, 2), decimal128(5, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 3), decimal128(5, 3), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 1), decimal128(6, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, -2), decimal128(9, 2), options);
+ CheckUnify(decimal128(3, -2), decimal128(5, -2), decimal128(5, -2), options);
+ CheckUnify(decimal128(38, 10), decimal128(38, 5), decimal256(43, 10), options);
+
+ CheckUnify(decimal256(3, 2), decimal256(5, 2), decimal256(5, 2), options);
+ CheckUnify(decimal256(3, 2), decimal256(5, 3), decimal256(5, 3), options);
+ CheckUnify(decimal256(3, 2), decimal256(5, 1), decimal256(6, 2), options);
+ CheckUnify(decimal256(3, 2), decimal256(5, -2), decimal256(9, 2), options);
+ CheckUnify(decimal256(3, -2), decimal256(5, -2), decimal256(5, -2), options);
+
+ // int32() is essentially decimal128(10, 0)
+ CheckUnify(int32(), decimal128(3, 2), decimal128(12, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(10, 0), options);
+ CheckUnify(int64(), decimal128(38, 37), decimal256(56, 37), options);
+
+ CheckUnifyFailsTypeError(decimal256(1, 0), decimal128(1, 0), options);
+
+ options.promote_numeric_width = true;
+ CheckUnify(decimal128(3, 2), decimal256(5, 2), decimal256(5, 2), options);
+ CheckUnify(int32(), decimal128(38, 37), decimal256(47, 37), options);
+ CheckUnifyFailsInvalid(decimal128(38, 10), decimal256(76, 5), options);
+
+ CheckUnifyFailsInvalid(int64(), decimal256(76, 75), options);
+}
+
+TEST_F(TestUnifySchemas, Temporal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_temporal_unit = true;
+ CheckUnify(date32(), {date64()}, options);
+
+ CheckUnify(time32(TimeUnit::SECOND),
+ {time32(TimeUnit::MILLI), time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time32(TimeUnit::MILLI), {time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time64(TimeUnit::MICRO), {time64(TimeUnit::NANO)}, options);
+
+ CheckUnify(
+ duration(TimeUnit::SECOND),
+ {duration(TimeUnit::MILLI), duration(TimeUnit::MICRO), duration(TimeUnit::NANO)},
+ options);
+ CheckUnify(duration(TimeUnit::MILLI),
+ {duration(TimeUnit::MICRO), duration(TimeUnit::NANO)}, options);
+ CheckUnify(duration(TimeUnit::MICRO), {duration(TimeUnit::NANO)}, options);
+
+ CheckUnify(
+ timestamp(TimeUnit::SECOND),
+ {timestamp(TimeUnit::MILLI), timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)},
+ options);
+ CheckUnify(timestamp(TimeUnit::MILLI),
+ {timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)}, options);
+ CheckUnify(timestamp(TimeUnit::MICRO), {timestamp(TimeUnit::NANO)}, options);
+
+ CheckUnifyFailsTypeError(timestamp(TimeUnit::SECOND),
+ timestamp(TimeUnit::SECOND, "UTC"), options);
+ CheckUnifyFailsTypeError(timestamp(TimeUnit::SECOND, "America/New_York"),
+ timestamp(TimeUnit::SECOND, "UTC"), options);
+
+ options.promote_temporal_unit = false;
+ CheckUnifyFailsInvalid(timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO), options);
+}
+
+TEST_F(TestUnifySchemas, Binary) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_binary = true;
+ CheckUnify(utf8(), {large_utf8(), binary(), large_binary()}, options);
+ CheckUnify(binary(), {large_binary()}, options);
+ CheckUnify(fixed_size_binary(2), {fixed_size_binary(2), binary(), large_binary()},
+ options);
+ CheckUnify(fixed_size_binary(2), fixed_size_binary(4), binary(), options);
+
+ options.promote_binary = false;
+ CheckUnifyFailsInvalid({utf8(), binary()}, {large_utf8(), large_binary()});
+ CheckUnifyFailsInvalid(fixed_size_binary(2), BaseBinaryTypes());
+ CheckUnifyFailsInvalid(utf8(), {binary(), large_binary(), fixed_size_binary(2)});
+}
+
+TEST_F(TestUnifySchemas, List) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ CheckUnifyFailsTypeError(fixed_size_list(int8(), 2), {fixed_size_list(int8(), 3)},
+ options);
+
+ options.promote_binary = true;
Review Comment:
I don't understand what `promote_binary` has to do with this?
##########
cpp/src/arrow/type_test.cc:
##########
@@ -892,6 +892,150 @@ class TestUnifySchemas : public TestSchema {
<< lhs_field->ToString() << " vs " << rhs_field->ToString();
}
}
+
+ void CheckUnifyAsymmetric(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_OK_AND_ASSIGN(auto merged, field1->MergeWith(field2, options));
+ AssertFieldEqual(merged, expected);
+ }
+
+ void CheckUnify(const std::shared_ptr<Field>& field1,
+ const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ CheckUnifyAsymmetric(field1, field2, expected, options);
+ CheckUnifyAsymmetric(field2, field1, expected, options);
+ }
+
+ void CheckUnifyFailsInvalid(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_RAISES(Invalid, field1->MergeWith(field2, options));
+ ASSERT_RAISES(Invalid, field2->MergeWith(field1, options));
+ }
+
+ void CheckUnifyFailsTypeError(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_RAISES(TypeError, field1->MergeWith(field2, options));
+ ASSERT_RAISES(TypeError, field2->MergeWith(field1, options));
+ }
+
+ void CheckUnify(const std::shared_ptr<DataType>& left,
+ const std::shared_ptr<DataType>& right,
+ const std::shared_ptr<DataType>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ auto field1 = field("a", left);
+ auto field2 = field("a", right);
+ CheckUnify(field1, field2, field("a", expected), options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnify(field1, field2, field("a", expected, /*nullable=*/false), options);
+
+ field1 = field("a", left);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnify(field1, field2, field("a", expected, /*nullable=*/true), options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right);
+ CheckUnify(field1, field2, field("a", expected, /*nullable=*/true), options);
+ }
+
+ void CheckUnifyAsymmetric(
+ const std::shared_ptr<DataType>& left, const std::shared_ptr<DataType>& right,
+ const std::shared_ptr<DataType>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ auto field1 = field("a", left);
+ auto field2 = field("a", right);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected), options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected, /*nullable=*/false),
+ options);
+
+ field1 = field("a", left);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected, /*nullable=*/true),
+ options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected, /*nullable=*/true),
+ options);
+ }
+
+ void CheckUnify(const std::shared_ptr<DataType>& from,
+ const std::vector<std::shared_ptr<DataType>>& to,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ for (const auto& ty : to) {
+ CheckUnify(from, ty, ty, options);
+ }
+ }
+
+ void CheckUnifyFailsInvalid(
+ const std::shared_ptr<DataType>& left, const std::shared_ptr<DataType>& right,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ auto field1 = field("a", left);
+ auto field2 = field("a", right);
+ CheckUnifyFailsInvalid(field1, field2, options);
+ }
+
+ void CheckUnifyFailsInvalid(
Review Comment:
Not suee this overload is useful since you can just pass both arguments as vectors?
##########
cpp/src/arrow/type_test.cc:
##########
@@ -892,6 +892,150 @@ class TestUnifySchemas : public TestSchema {
<< lhs_field->ToString() << " vs " << rhs_field->ToString();
}
}
+
+ void CheckUnifyAsymmetric(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_OK_AND_ASSIGN(auto merged, field1->MergeWith(field2, options));
+ AssertFieldEqual(merged, expected);
+ }
+
+ void CheckUnify(const std::shared_ptr<Field>& field1,
+ const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ CheckUnifyAsymmetric(field1, field2, expected, options);
+ CheckUnifyAsymmetric(field2, field1, expected, options);
+ }
+
+ void CheckUnifyFailsInvalid(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_RAISES(Invalid, field1->MergeWith(field2, options));
+ ASSERT_RAISES(Invalid, field2->MergeWith(field1, options));
+ }
+
+ void CheckUnifyFailsTypeError(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_RAISES(TypeError, field1->MergeWith(field2, options));
+ ASSERT_RAISES(TypeError, field2->MergeWith(field1, options));
+ }
+
+ void CheckUnify(const std::shared_ptr<DataType>& left,
+ const std::shared_ptr<DataType>& right,
+ const std::shared_ptr<DataType>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ auto field1 = field("a", left);
+ auto field2 = field("a", right);
+ CheckUnify(field1, field2, field("a", expected), options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnify(field1, field2, field("a", expected, /*nullable=*/false), options);
+
+ field1 = field("a", left);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnify(field1, field2, field("a", expected, /*nullable=*/true), options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right);
+ CheckUnify(field1, field2, field("a", expected, /*nullable=*/true), options);
+ }
+
+ void CheckUnifyAsymmetric(
+ const std::shared_ptr<DataType>& left, const std::shared_ptr<DataType>& right,
+ const std::shared_ptr<DataType>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ auto field1 = field("a", left);
+ auto field2 = field("a", right);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected), options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected, /*nullable=*/false),
+ options);
+
+ field1 = field("a", left);
+ field2 = field("a", right, /*nullable=*/false);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected, /*nullable=*/true),
+ options);
+
+ field1 = field("a", left, /*nullable=*/false);
+ field2 = field("a", right);
+ CheckUnifyAsymmetric(field1, field2, field("a", expected, /*nullable=*/true),
+ options);
+ }
+
+ void CheckUnify(const std::shared_ptr<DataType>& from,
Review Comment:
For clarity, name it `CheckPromoteTo`? Otherwise it's a bit difficult to understand what happens when reading at the call site.
##########
cpp/src/arrow/type_test.cc:
##########
@@ -892,6 +892,150 @@ class TestUnifySchemas : public TestSchema {
<< lhs_field->ToString() << " vs " << rhs_field->ToString();
}
}
+
+ void CheckUnifyAsymmetric(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_OK_AND_ASSIGN(auto merged, field1->MergeWith(field2, options));
+ AssertFieldEqual(merged, expected);
+ }
+
+ void CheckUnify(const std::shared_ptr<Field>& field1,
+ const std::shared_ptr<Field>& field2,
+ const std::shared_ptr<Field>& expected,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ CheckUnifyAsymmetric(field1, field2, expected, options);
+ CheckUnifyAsymmetric(field2, field1, expected, options);
+ }
+
+ void CheckUnifyFailsInvalid(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
+ ARROW_SCOPED_TRACE("options: ", options);
+ ARROW_SCOPED_TRACE("field2: ", field2->ToString());
+ ARROW_SCOPED_TRACE("field1: ", field1->ToString());
+ ASSERT_RAISES(Invalid, field1->MergeWith(field2, options));
+ ASSERT_RAISES(Invalid, field2->MergeWith(field1, options));
+ }
+
+ void CheckUnifyFailsTypeError(
+ const std::shared_ptr<Field>& field1, const std::shared_ptr<Field>& field2,
+ const Field::MergeOptions& options = Field::MergeOptions::Defaults()) {
Review Comment:
Same idea here :-)
##########
cpp/src/arrow/type_test.cc:
##########
@@ -983,6 +1091,242 @@ TEST_F(TestUnifySchemas, MoreSchemas) {
utf8_field->WithNullable(true)}));
}
+TEST_F(TestUnifySchemas, Numeric) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ options.promote_integer_float = true;
+ options.promote_integer_sign = true;
+ CheckUnify(uint8(),
+ {int8(), uint16(), int16(), uint32(), int32(), uint64(), int64(), float32(),
+ float64()},
+ options);
+ CheckUnify(int8(), {int16(), int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint16(),
+ {int16(), uint32(), int32(), uint64(), int64(), float32(), float64()},
+ options);
+ CheckUnify(int16(), {int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint32(), {int32(), uint64(), int64(), float32(), float64()}, options);
+ CheckUnify(int32(), {int64(), float32(), float64()}, options);
+ CheckUnify(uint64(), {int64(), float64()}, options);
+ CheckUnify(int64(), {float64()}, options);
+ CheckUnify(float16(), {float32(), float64()}, options);
+ CheckUnify(float32(), {float64()}, options);
+ CheckUnify(uint64(), float32(), float64(), options);
+ CheckUnify(int64(), float32(), float64(), options);
+
+ options.promote_integer_sign = false;
+ CheckUnify(uint8(), {uint16(), uint32(), uint64()}, options);
+ CheckUnify(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFails(uint8(), {int8(), int16(), int32(), int64()}, options);
+ CheckUnify(uint16(), {uint32(), uint64()}, options);
+ CheckUnify(int16(), {int32(), int64()}, options);
+ CheckUnifyFails(uint16(), {int16(), int32(), int64()}, options);
+ CheckUnify(uint32(), {uint64()}, options);
+ CheckUnify(int32(), {int64()}, options);
+ CheckUnifyFails(uint32(), {int32(), int64()}, options);
+ CheckUnifyFails(uint64(), {int64()}, options);
+
+ options.promote_integer_sign = true;
+ options.promote_integer_float = false;
+ CheckUnifyFails(IntTypes(), FloatingPointTypes(), options);
+
+ options.promote_integer_float = true;
+ options.promote_numeric_width = false;
+ CheckUnifyFails(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFails(int16(), {int32(), int64()}, options);
+ CheckUnifyFails(int32(), {int64()}, options);
+}
+
+TEST_F(TestUnifySchemas, Decimal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_decimal_float = true;
+ CheckUnify(decimal128(3, 2), {float32(), float64()}, options);
+ CheckUnify(decimal256(3, 2), {float32(), float64()}, options);
+
+ options.promote_integer_decimal = true;
+ CheckUnify(int32(), decimal128(3, 2), decimal128(3, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(3, -2), options);
+
+ options.promote_decimal = true;
+ CheckUnify(decimal128(3, 2), decimal128(5, 2), decimal128(5, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 3), decimal128(5, 3), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 1), decimal128(6, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, -2), decimal128(9, 2), options);
+ CheckUnify(decimal128(3, -2), decimal128(5, -2), decimal128(5, -2), options);
+
+ // int32() is essentially decimal128(10, 0)
+ CheckUnify(int32(), decimal128(3, 2), decimal128(12, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(10, 0), options);
+
+ CheckUnifyFails(decimal256(1, 0), decimal128(1, 0), options);
+ CheckUnifyFails(int64(), decimal128(38, 37), options);
+
+ options.promote_numeric_width = true;
+ CheckUnify(decimal128(3, 2), decimal256(5, 2), decimal256(5, 2), options);
+ CheckUnify(int32(), decimal128(38, 37), decimal256(47, 37), options);
+
+ CheckUnifyFails(int64(), decimal256(76, 75), options);
+}
+
+TEST_F(TestUnifySchemas, Temporal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_date = true;
+ CheckUnify(date32(), {date64()}, options);
+
+ options.promote_time = true;
+ CheckUnify(time32(TimeUnit::SECOND),
+ {time32(TimeUnit::MILLI), time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time32(TimeUnit::MILLI), {time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time64(TimeUnit::MICRO), {time64(TimeUnit::NANO)}, options);
+
+ options.promote_duration = true;
+ CheckUnify(
+ duration(TimeUnit::SECOND),
+ {duration(TimeUnit::MILLI), duration(TimeUnit::MICRO), duration(TimeUnit::NANO)},
+ options);
+ CheckUnify(duration(TimeUnit::MILLI),
+ {duration(TimeUnit::MICRO), duration(TimeUnit::NANO)}, options);
+ CheckUnify(duration(TimeUnit::MICRO), {duration(TimeUnit::NANO)}, options);
+
+ options.promote_timestamp = true;
+ CheckUnify(
+ timestamp(TimeUnit::SECOND),
+ {timestamp(TimeUnit::MILLI), timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)},
+ options);
+ CheckUnify(timestamp(TimeUnit::MILLI),
+ {timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)}, options);
+ CheckUnify(timestamp(TimeUnit::MICRO), {timestamp(TimeUnit::NANO)}, options);
+
+ CheckUnifyFails(timestamp(TimeUnit::SECOND), timestamp(TimeUnit::SECOND, "UTC"),
+ options);
+ CheckUnifyFails(timestamp(TimeUnit::SECOND, "America/New_York"),
+ timestamp(TimeUnit::SECOND, "UTC"), options);
+}
+
+TEST_F(TestUnifySchemas, Binary) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_large = true;
+ options.promote_binary = true;
+ CheckUnify(utf8(), {large_utf8(), binary(), large_binary()}, options);
+ CheckUnify(binary(), {large_binary()}, options);
+ CheckUnify(fixed_size_binary(2), {fixed_size_binary(2), binary(), large_binary()},
+ options);
+ CheckUnify(fixed_size_binary(2), fixed_size_binary(4), binary(), options);
+
+ options.promote_large = false;
+ CheckUnifyFails({utf8(), binary()}, {large_utf8(), large_binary()});
+ CheckUnifyFails(fixed_size_binary(2), BaseBinaryTypes());
+
+ options.promote_binary = false;
+ CheckUnifyFails(utf8(), {binary(), large_binary(), fixed_size_binary(2)});
+}
+
+TEST_F(TestUnifySchemas, List) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ CheckUnifyFails(fixed_size_list(int8(), 2),
+ {fixed_size_list(int8(), 3), list(int8()), large_list(int8())},
+ options);
+
+ options.promote_large = true;
+ CheckUnify(list(int8()), {large_list(int8())}, options);
+ CheckUnify(fixed_size_list(int8(), 2), {list(int8()), large_list(int8())}, options);
+
+ options.promote_nested = true;
+ CheckUnify(list(int8()), {list(int16()), list(int32()), list(int64())}, options);
+ CheckUnify(fixed_size_list(int8(), 2),
+ {fixed_size_list(int16(), 2), list(int16()), list(int32()), list(int64())},
+ options);
+
+ auto ty = list(field("foo", int8(), /*nullable=*/false));
+ CheckUnifyAsymmetric(ty, list(int8()), list(field("foo", int8(), /*nullable=*/true)),
+ options);
+ CheckUnifyAsymmetric(ty, list(field("bar", int16(), /*nullable=*/false)),
+ list(field("foo", int16(), /*nullable=*/false)), options);
+}
+
+TEST_F(TestUnifySchemas, Map) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_nested = true;
+ options.promote_numeric_width = true;
+
+ CheckUnify(map(int8(), int32()),
+ {map(int8(), int64()), map(int16(), int32()), map(int64(), int64())},
+ options);
+
+ // Do not test field names, since MapType intentionally ignores them in comparisons
+ // See ARROW-7173, ARROW-14999
+ auto ty = map(field("key", int8(), /*nullable=*/false),
+ field("value", int32(), /*nullable=*/false));
+ CheckUnify(ty, map(int8(), int32()),
+ map(field("key", int8(), /*nullable=*/true),
+ field("value", int32(), /*nullable=*/true)),
+ options);
+ CheckUnify(ty,
+ map(field("key", int16(), /*nullable=*/false),
+ field("value", int64(), /*nullable=*/false)),
+ map(field("key", int16(), /*nullable=*/false),
+ field("value", int64(), /*nullable=*/false)),
+ options);
+}
+
+TEST_F(TestUnifySchemas, Struct) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_nested = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+
+ CheckUnify(struct_({}), struct_({field("a", int8())}), struct_({field("a", int8())}),
+ options);
+
+ CheckUnifyAsymmetric(struct_({field("b", utf8())}), struct_({field("a", int8())}),
+ struct_({field("b", utf8()), field("a", int8())}), options);
+ CheckUnifyAsymmetric(struct_({field("a", int8())}), struct_({field("b", utf8())}),
+ struct_({field("a", int8()), field("b", utf8())}), options);
+
+ CheckUnify(struct_({field("b", utf8())}), struct_({field("b", binary())}),
+ struct_({field("b", binary())}), options);
+
+ CheckUnifyAsymmetric(
+ struct_({field("a", int8()), field("b", utf8()), field("a", int64())}),
+ struct_({field("b", binary())}),
+ struct_({field("a", int8()), field("b", binary()), field("a", int64())}), options);
+
+ ASSERT_RAISES(
+ Invalid,
+ field("foo", struct_({field("a", int8()), field("b", utf8()), field("a", int64())}))
+ ->MergeWith(field("foo", struct_({field("a", int64())})), options));
+}
+
+TEST_F(TestUnifySchemas, Dictionary) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_dictionary = true;
+ options.promote_large = true;
+
+ CheckUnify(dictionary(int8(), utf8()),
+ {
+ dictionary(int64(), utf8()),
+ dictionary(int8(), large_utf8()),
+ },
+ options);
+ CheckUnify(dictionary(int8(), utf8(), /*ordered=*/true),
+ {
+ dictionary(int64(), utf8(), /*ordered=*/true),
+ dictionary(int8(), large_utf8(), /*ordered=*/true),
+ },
+ options);
Review Comment:
Ping on this.
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ }
+
+ if (options.promote_numeric_width) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ if (is_floating(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ } else if (is_signed_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ } else if (is_unsigned_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = uint64();
+ } else if (max_width >= 32) {
+ promoted_type = uint32();
+ } else if (max_width >= 16) {
+ promoted_type = uint16();
+ } else {
+ promoted_type = uint8();
+ }
+ }
+ }
+
+ return promoted ? promoted_type : nullptr;
+}
+
+// Merge temporal types based on options. Returns nullptr for non-binary types.
+Result<std::shared_ptr<DataType>> MaybeMergeBinaryTypes(
+ std::shared_ptr<DataType>& promoted_type, std::shared_ptr<DataType>& other_type,
+ const Field::MergeOptions& options) {
+ if (options.promote_binary) {
+ if (other_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(promoted_type->id())) {
+ return MakeBinary(*promoted_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ other_type->id() == Type::FIXED_SIZE_BINARY) {
+ return binary();
+ } else if (is_string(promoted_type->id()) && is_binary(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (is_binary(promoted_type->id()) && is_string(other_type->id())) {
+ return MakeBinary(*promoted_type);
+ }
+
+ if ((promoted_type->id() == Type::STRING && other_type->id() == Type::LARGE_STRING) ||
+ (promoted_type->id() == Type::LARGE_STRING && other_type->id() == Type::STRING)) {
+ return large_utf8();
+ } else if ((promoted_type->id() == Type::BINARY &&
+ other_type->id() == Type::LARGE_BINARY) ||
+ (promoted_type->id() == Type::LARGE_BINARY &&
+ other_type->id() == Type::BINARY)) {
+ return large_binary();
+ }
+ }
+
+ return nullptr;
+}
+
+// Merge list types based on options. Returns nullptr for non-list types.
+Result<std::shared_ptr<DataType>> MaybeMergeListTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ bool promoted = false;
+
+ if (promoted_type->id() == Type::FIXED_SIZE_LIST &&
+ other_type->id() == Type::FIXED_SIZE_LIST) {
+ const auto& left = checked_cast<const FixedSizeListType&>(*promoted_type);
+ const auto& right = checked_cast<const FixedSizeListType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto value_field,
+ left.value_field()->MergeWith(
+ *right.value_field()->WithName(left.value_field()->name()), options));
+ if (left.list_size() == right.list_size()) {
+ return fixed_size_list(std::move(value_field), left.list_size());
+ }
+ return Status::TypeError("Cannot merge fixed_size_list of different sizes");
+ } else if (promoted_type->id() == Type::MAP && other_type->id() == Type::MAP) {
+ const auto& left = checked_cast<const MapType&>(*promoted_type);
+ const auto& right = checked_cast<const MapType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto key_field,
+ left.key_field()->MergeWith(
+ *right.key_field()->WithName(left.key_field()->name()), options));
+ ARROW_ASSIGN_OR_RAISE(
+ auto item_field,
+ left.item_field()->MergeWith(
+ *right.item_field()->WithName(left.item_field()->name()), options));
+ return map(std::move(key_field->type()), std::move(item_field),
+ /*keys_sorted=*/left.keys_sorted() && right.keys_sorted());
+ } else if (is_list(promoted_type->id()) && is_list(other_type->id())) {
+ const auto& left = checked_cast<const BaseListType&>(*promoted_type);
+ const auto& right = checked_cast<const BaseListType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto value_field,
+ left.value_field()->MergeWith(
+ *right.value_field()->WithName(left.value_field()->name()), options));
+ if (promoted_type->id() == Type::LARGE_LIST || other_type->id() == Type::LARGE_LIST) {
+ return large_list(std::move(value_field));
+ } else {
+ return list(std::move(value_field));
+ }
+ } else if (promoted_type->id() == Type::STRUCT && other_type->id() == Type::STRUCT) {
Review Comment:
I'm surprised that this appears in `MaybeMergeListTypes`. Perhaps a separate helper function is desired for clarity?
##########
cpp/src/arrow/type.cc:
##########
@@ -316,6 +299,405 @@ std::shared_ptr<Field> Field::WithNullable(const bool nullable) const {
return std::make_shared<Field>(name_, type_, nullable, metadata_);
}
+Field::MergeOptions Field::MergeOptions::Permissive() {
+ MergeOptions options = Defaults();
+ options.promote_nullability = true;
+ options.promote_decimal = true;
+ options.promote_decimal_to_float = true;
+ options.promote_integer_to_decimal = true;
+ options.promote_integer_to_float = true;
+ options.promote_integer_sign = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+ options.promote_temporal_unit = true;
+ options.promote_dictionary = true;
+ options.promote_dictionary_ordered = false;
+ return options;
+}
+
+std::string Field::MergeOptions::ToString() const {
+ std::stringstream ss;
+ ss << "MergeOptions{";
+ ss << "promote_nullability=" << (promote_nullability ? "true" : "false");
+ ss << ", promote_numeric_width=" << (promote_numeric_width ? "true" : "false");
+ ss << ", promote_integer_to_float=" << (promote_integer_to_float ? "true" : "false");
+ ss << ", promote_integer_to_decimal="
+ << (promote_integer_to_decimal ? "true" : "false");
+ ss << ", promote_decimal_to_float=" << (promote_decimal_to_float ? "true" : "false");
+ ss << ", promote_temporal_unit=" << (promote_temporal_unit ? "true" : "false");
+ ss << ", promote_dictionary=" << (promote_dictionary ? "true" : "false");
+ ss << ", promote_integer_sign=" << (promote_integer_sign ? "true" : "false");
+ ss << ", promote_binary=" << (promote_binary ? "true" : "false");
+ ss << '}';
+ return ss.str();
+}
+
+namespace {
+// Utilities for Field::MergeWith
+
+std::shared_ptr<DataType> MakeBinary(const DataType& type) {
+ switch (type.id()) {
+ case Type::BINARY:
+ case Type::STRING:
+ return binary();
+ case Type::LARGE_BINARY:
+ case Type::LARGE_STRING:
+ return large_binary();
+ default:
+ Unreachable("Hit an unknown type");
+ }
+ return nullptr;
+}
+
+Result<std::shared_ptr<DataType>> WidenDecimals(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type) {
+ const auto& left = checked_cast<const DecimalType&>(*promoted_type);
+ const auto& right = checked_cast<const DecimalType&>(*other_type);
+ const int32_t max_scale = std::max<int32_t>(left.scale(), right.scale());
+ const int32_t common_precision =
+ std::max<int32_t>(left.precision() + max_scale - left.scale(),
+ right.precision() + max_scale - right.scale());
+ if (left.id() == Type::DECIMAL256 || right.id() == Type::DECIMAL256 ||
+ common_precision > BasicDecimal128::kMaxPrecision) {
+ return DecimalType::Make(Type::DECIMAL256, common_precision, max_scale);
+ }
+ return DecimalType::Make(Type::DECIMAL128, common_precision, max_scale);
+}
+
+Result<std::shared_ptr<DataType>> MergeTypes(std::shared_ptr<DataType> promoted_type,
+ std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options);
+
+// Merge two dictionary types, or else give an error.
+Result<std::shared_ptr<DataType>> MergeDictionaryTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ const auto& left = checked_cast<const DictionaryType&>(*promoted_type);
+ const auto& right = checked_cast<const DictionaryType&>(*other_type);
+ if (!options.promote_dictionary_ordered && left.ordered() != right.ordered()) {
+ return Status::TypeError(
+ "Cannot merge ordered and unordered dictionary unless "
+ "promote_dictionary_ordered=true");
+ }
+ Field::MergeOptions index_options = options;
+ index_options.promote_integer_sign = true;
+ index_options.promote_numeric_width = true;
+ ARROW_ASSIGN_OR_RAISE(auto indices,
+ MergeTypes(left.index_type(), right.index_type(), index_options));
+ ARROW_ASSIGN_OR_RAISE(auto values,
+ MergeTypes(left.value_type(), right.value_type(), options));
+ auto ordered = left.ordered() && right.ordered();
+ if (indices && values) {
+ return dictionary(indices, values, ordered);
+ } else if (values) {
+ return Status::Invalid("Could not merge index types");
+ }
+ return Status::Invalid("Could not merge value types");
+}
+
+// Merge temporal types based on options. Returns nullptr for non-temporal types.
+Result<std::shared_ptr<DataType>> MaybeMergeTemporalTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ if (options.promote_temporal_unit) {
+ if (promoted_type->id() == Type::DATE32 && other_type->id() == Type::DATE64) {
+ return date64();
+ }
+ if (promoted_type->id() == Type::DATE64 && other_type->id() == Type::DATE32) {
+ return date64();
+ }
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::DURATION &&
+ other_type->id() == Type::DURATION) {
+ const auto& left = checked_cast<const DurationType&>(*promoted_type);
+ const auto& right = checked_cast<const DurationType&>(*other_type);
+ return duration(std::max(left.unit(), right.unit()));
+ }
+
+ if (options.promote_temporal_unit && is_time(promoted_type->id()) &&
+ is_time(other_type->id())) {
+ const auto& left = checked_cast<const TimeType&>(*promoted_type);
+ const auto& right = checked_cast<const TimeType&>(*other_type);
+ const auto unit = std::max(left.unit(), right.unit());
+ if (unit == TimeUnit::MICRO || unit == TimeUnit::NANO) {
+ return time64(unit);
+ }
+ return time32(unit);
+ }
+
+ if (options.promote_temporal_unit && promoted_type->id() == Type::TIMESTAMP &&
+ other_type->id() == Type::TIMESTAMP) {
+ const auto& left = checked_cast<const TimestampType&>(*promoted_type);
+ const auto& right = checked_cast<const TimestampType&>(*other_type);
+ if (left.timezone().empty() ^ right.timezone().empty()) {
+ return Status::TypeError(
+ "Cannot merge timestamp with timezone and timestamp without timezone");
+ }
+ if (left.timezone() != right.timezone()) {
+ return Status::TypeError("Cannot merge timestamps with differing timezones");
+ }
+ return timestamp(std::max(left.unit(), right.unit()), left.timezone());
+ }
+
+ return nullptr;
+}
+
+// Merge numeric types based on options. Returns nullptr for non-numeric types.
+Result<std::shared_ptr<DataType>> MaybeMergeNumericTypes(
+ std::shared_ptr<DataType> promoted_type, std::shared_ptr<DataType> other_type,
+ const Field::MergeOptions& options) {
+ bool promoted = false;
+ if (options.promote_decimal_to_float) {
+ if (is_decimal(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted_type = other_type;
+ promoted = true;
+ } else if (is_floating(promoted_type->id()) && is_decimal(other_type->id())) {
+ other_type = promoted_type;
+ promoted = true;
+ }
+ }
+
+ if (options.promote_integer_to_decimal &&
+ ((is_decimal(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_decimal(other_type->id()) && is_integer(promoted_type->id())))) {
+ if (is_integer(promoted_type->id()) && is_decimal(other_type->id())) {
+ // Other type is always the int
+ promoted_type.swap(other_type);
+ }
+ ARROW_ASSIGN_OR_RAISE(const int32_t precision,
+ MaxDecimalDigitsForInteger(other_type->id()));
+ ARROW_ASSIGN_OR_RAISE(const auto promoted_decimal,
+ DecimalType::Make(promoted_type->id(), precision, 0));
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, promoted_decimal));
+ return promoted_type;
+ }
+
+ if (options.promote_decimal && is_decimal(promoted_type->id()) &&
+ is_decimal(other_type->id())) {
+ const DecimalType& left = checked_cast<const DecimalType&>(*promoted_type);
+ const DecimalType& right = checked_cast<const DecimalType&>(*other_type);
+ if (!options.promote_numeric_width && left.bit_width() != right.bit_width()) {
+ return Status::TypeError(
+ "Cannot promote decimal128 to decimal256 without promote_numeric_width=true");
+ }
+ ARROW_ASSIGN_OR_RAISE(promoted_type, WidenDecimals(promoted_type, other_type));
+ return promoted_type;
+ }
+
+ if (options.promote_integer_sign && ((is_unsigned_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) ||
+ (is_signed_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ return promoted_type;
+ }
+
+ if (options.promote_integer_to_float &&
+ ((is_floating(promoted_type->id()) && is_integer(other_type->id())) ||
+ (is_integer(promoted_type->id()) && is_floating(other_type->id())))) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ }
+
+ if (options.promote_numeric_width) {
+ const int max_width =
+ std::max<int>(bit_width(promoted_type->id()), bit_width(other_type->id()));
+ if (is_floating(promoted_type->id()) && is_floating(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = float64();
+ } else if (max_width >= 32) {
+ promoted_type = float32();
+ } else {
+ promoted_type = float16();
+ }
+ } else if (is_signed_integer(promoted_type->id()) &&
+ is_signed_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = int64();
+ } else if (max_width >= 32) {
+ promoted_type = int32();
+ } else if (max_width >= 16) {
+ promoted_type = int16();
+ } else {
+ promoted_type = int8();
+ }
+ } else if (is_unsigned_integer(promoted_type->id()) &&
+ is_unsigned_integer(other_type->id())) {
+ promoted = true;
+ if (max_width >= 64) {
+ promoted_type = uint64();
+ } else if (max_width >= 32) {
+ promoted_type = uint32();
+ } else if (max_width >= 16) {
+ promoted_type = uint16();
+ } else {
+ promoted_type = uint8();
+ }
+ }
+ }
+
+ return promoted ? promoted_type : nullptr;
+}
+
+// Merge temporal types based on options. Returns nullptr for non-binary types.
+Result<std::shared_ptr<DataType>> MaybeMergeBinaryTypes(
+ std::shared_ptr<DataType>& promoted_type, std::shared_ptr<DataType>& other_type,
+ const Field::MergeOptions& options) {
+ if (options.promote_binary) {
+ if (other_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(promoted_type->id())) {
+ return MakeBinary(*promoted_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ is_base_binary_like(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (promoted_type->id() == Type::FIXED_SIZE_BINARY &&
+ other_type->id() == Type::FIXED_SIZE_BINARY) {
+ return binary();
+ } else if (is_string(promoted_type->id()) && is_binary(other_type->id())) {
+ return MakeBinary(*other_type);
+ } else if (is_binary(promoted_type->id()) && is_string(other_type->id())) {
+ return MakeBinary(*promoted_type);
+ }
+
+ if ((promoted_type->id() == Type::STRING && other_type->id() == Type::LARGE_STRING) ||
+ (promoted_type->id() == Type::LARGE_STRING && other_type->id() == Type::STRING)) {
+ return large_utf8();
+ } else if ((promoted_type->id() == Type::BINARY &&
+ other_type->id() == Type::LARGE_BINARY) ||
+ (promoted_type->id() == Type::LARGE_BINARY &&
+ other_type->id() == Type::BINARY)) {
+ return large_binary();
+ }
+ }
+
+ return nullptr;
+}
+
+// Merge list types based on options. Returns nullptr for non-list types.
+Result<std::shared_ptr<DataType>> MaybeMergeListTypes(
+ const std::shared_ptr<DataType>& promoted_type,
+ const std::shared_ptr<DataType>& other_type, const Field::MergeOptions& options) {
+ bool promoted = false;
+
+ if (promoted_type->id() == Type::FIXED_SIZE_LIST &&
+ other_type->id() == Type::FIXED_SIZE_LIST) {
+ const auto& left = checked_cast<const FixedSizeListType&>(*promoted_type);
+ const auto& right = checked_cast<const FixedSizeListType&>(*other_type);
+ ARROW_ASSIGN_OR_RAISE(
+ auto value_field,
+ left.value_field()->MergeWith(
+ *right.value_field()->WithName(left.value_field()->name()), options));
+ if (left.list_size() == right.list_size()) {
+ return fixed_size_list(std::move(value_field), left.list_size());
+ }
+ return Status::TypeError("Cannot merge fixed_size_list of different sizes");
Review Comment:
Why not let them promote to a var-size list?
We promote if merging a fixed-size list and a var-size list, AFAICT.
##########
cpp/src/arrow/type_test.cc:
##########
@@ -983,6 +1091,242 @@ TEST_F(TestUnifySchemas, MoreSchemas) {
utf8_field->WithNullable(true)}));
}
+TEST_F(TestUnifySchemas, Numeric) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ options.promote_integer_float = true;
+ options.promote_integer_sign = true;
+ CheckUnify(uint8(),
+ {int8(), uint16(), int16(), uint32(), int32(), uint64(), int64(), float32(),
+ float64()},
+ options);
+ CheckUnify(int8(), {int16(), int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint16(),
+ {int16(), uint32(), int32(), uint64(), int64(), float32(), float64()},
+ options);
+ CheckUnify(int16(), {int32(), int64(), float32(), float64()}, options);
+ CheckUnify(uint32(), {int32(), uint64(), int64(), float32(), float64()}, options);
+ CheckUnify(int32(), {int64(), float32(), float64()}, options);
+ CheckUnify(uint64(), {int64(), float64()}, options);
+ CheckUnify(int64(), {float64()}, options);
+ CheckUnify(float16(), {float32(), float64()}, options);
+ CheckUnify(float32(), {float64()}, options);
+ CheckUnify(uint64(), float32(), float64(), options);
+ CheckUnify(int64(), float32(), float64(), options);
+
+ options.promote_integer_sign = false;
+ CheckUnify(uint8(), {uint16(), uint32(), uint64()}, options);
+ CheckUnify(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFails(uint8(), {int8(), int16(), int32(), int64()}, options);
+ CheckUnify(uint16(), {uint32(), uint64()}, options);
+ CheckUnify(int16(), {int32(), int64()}, options);
+ CheckUnifyFails(uint16(), {int16(), int32(), int64()}, options);
+ CheckUnify(uint32(), {uint64()}, options);
+ CheckUnify(int32(), {int64()}, options);
+ CheckUnifyFails(uint32(), {int32(), int64()}, options);
+ CheckUnifyFails(uint64(), {int64()}, options);
+
+ options.promote_integer_sign = true;
+ options.promote_integer_float = false;
+ CheckUnifyFails(IntTypes(), FloatingPointTypes(), options);
+
+ options.promote_integer_float = true;
+ options.promote_numeric_width = false;
+ CheckUnifyFails(int8(), {int16(), int32(), int64()}, options);
+ CheckUnifyFails(int16(), {int32(), int64()}, options);
+ CheckUnifyFails(int32(), {int64()}, options);
+}
+
+TEST_F(TestUnifySchemas, Decimal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_decimal_float = true;
+ CheckUnify(decimal128(3, 2), {float32(), float64()}, options);
+ CheckUnify(decimal256(3, 2), {float32(), float64()}, options);
+
+ options.promote_integer_decimal = true;
+ CheckUnify(int32(), decimal128(3, 2), decimal128(3, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(3, -2), options);
+
+ options.promote_decimal = true;
+ CheckUnify(decimal128(3, 2), decimal128(5, 2), decimal128(5, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 3), decimal128(5, 3), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, 1), decimal128(6, 2), options);
+ CheckUnify(decimal128(3, 2), decimal128(5, -2), decimal128(9, 2), options);
+ CheckUnify(decimal128(3, -2), decimal128(5, -2), decimal128(5, -2), options);
+
+ // int32() is essentially decimal128(10, 0)
+ CheckUnify(int32(), decimal128(3, 2), decimal128(12, 2), options);
+ CheckUnify(int32(), decimal128(3, -2), decimal128(10, 0), options);
+
+ CheckUnifyFails(decimal256(1, 0), decimal128(1, 0), options);
+ CheckUnifyFails(int64(), decimal128(38, 37), options);
+
+ options.promote_numeric_width = true;
+ CheckUnify(decimal128(3, 2), decimal256(5, 2), decimal256(5, 2), options);
+ CheckUnify(int32(), decimal128(38, 37), decimal256(47, 37), options);
+
+ CheckUnifyFails(int64(), decimal256(76, 75), options);
+}
+
+TEST_F(TestUnifySchemas, Temporal) {
+ auto options = Field::MergeOptions::Defaults();
+
+ options.promote_date = true;
+ CheckUnify(date32(), {date64()}, options);
+
+ options.promote_time = true;
+ CheckUnify(time32(TimeUnit::SECOND),
+ {time32(TimeUnit::MILLI), time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time32(TimeUnit::MILLI), {time64(TimeUnit::MICRO), time64(TimeUnit::NANO)},
+ options);
+ CheckUnify(time64(TimeUnit::MICRO), {time64(TimeUnit::NANO)}, options);
+
+ options.promote_duration = true;
+ CheckUnify(
+ duration(TimeUnit::SECOND),
+ {duration(TimeUnit::MILLI), duration(TimeUnit::MICRO), duration(TimeUnit::NANO)},
+ options);
+ CheckUnify(duration(TimeUnit::MILLI),
+ {duration(TimeUnit::MICRO), duration(TimeUnit::NANO)}, options);
+ CheckUnify(duration(TimeUnit::MICRO), {duration(TimeUnit::NANO)}, options);
+
+ options.promote_timestamp = true;
+ CheckUnify(
+ timestamp(TimeUnit::SECOND),
+ {timestamp(TimeUnit::MILLI), timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)},
+ options);
+ CheckUnify(timestamp(TimeUnit::MILLI),
+ {timestamp(TimeUnit::MICRO), timestamp(TimeUnit::NANO)}, options);
+ CheckUnify(timestamp(TimeUnit::MICRO), {timestamp(TimeUnit::NANO)}, options);
+
+ CheckUnifyFails(timestamp(TimeUnit::SECOND), timestamp(TimeUnit::SECOND, "UTC"),
+ options);
+ CheckUnifyFails(timestamp(TimeUnit::SECOND, "America/New_York"),
+ timestamp(TimeUnit::SECOND, "UTC"), options);
+}
+
+TEST_F(TestUnifySchemas, Binary) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_large = true;
+ options.promote_binary = true;
+ CheckUnify(utf8(), {large_utf8(), binary(), large_binary()}, options);
+ CheckUnify(binary(), {large_binary()}, options);
+ CheckUnify(fixed_size_binary(2), {fixed_size_binary(2), binary(), large_binary()},
+ options);
+ CheckUnify(fixed_size_binary(2), fixed_size_binary(4), binary(), options);
+
+ options.promote_large = false;
+ CheckUnifyFails({utf8(), binary()}, {large_utf8(), large_binary()});
+ CheckUnifyFails(fixed_size_binary(2), BaseBinaryTypes());
+
+ options.promote_binary = false;
+ CheckUnifyFails(utf8(), {binary(), large_binary(), fixed_size_binary(2)});
+}
+
+TEST_F(TestUnifySchemas, List) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_numeric_width = true;
+ CheckUnifyFails(fixed_size_list(int8(), 2),
+ {fixed_size_list(int8(), 3), list(int8()), large_list(int8())},
+ options);
+
+ options.promote_large = true;
+ CheckUnify(list(int8()), {large_list(int8())}, options);
+ CheckUnify(fixed_size_list(int8(), 2), {list(int8()), large_list(int8())}, options);
+
+ options.promote_nested = true;
+ CheckUnify(list(int8()), {list(int16()), list(int32()), list(int64())}, options);
+ CheckUnify(fixed_size_list(int8(), 2),
+ {fixed_size_list(int16(), 2), list(int16()), list(int32()), list(int64())},
+ options);
+
+ auto ty = list(field("foo", int8(), /*nullable=*/false));
+ CheckUnifyAsymmetric(ty, list(int8()), list(field("foo", int8(), /*nullable=*/true)),
+ options);
+ CheckUnifyAsymmetric(ty, list(field("bar", int16(), /*nullable=*/false)),
+ list(field("foo", int16(), /*nullable=*/false)), options);
+}
+
+TEST_F(TestUnifySchemas, Map) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_nested = true;
+ options.promote_numeric_width = true;
+
+ CheckUnify(map(int8(), int32()),
+ {map(int8(), int64()), map(int16(), int32()), map(int64(), int64())},
+ options);
+
+ // Do not test field names, since MapType intentionally ignores them in comparisons
+ // See ARROW-7173, ARROW-14999
+ auto ty = map(field("key", int8(), /*nullable=*/false),
+ field("value", int32(), /*nullable=*/false));
+ CheckUnify(ty, map(int8(), int32()),
+ map(field("key", int8(), /*nullable=*/true),
+ field("value", int32(), /*nullable=*/true)),
+ options);
+ CheckUnify(ty,
+ map(field("key", int16(), /*nullable=*/false),
+ field("value", int64(), /*nullable=*/false)),
+ map(field("key", int16(), /*nullable=*/false),
+ field("value", int64(), /*nullable=*/false)),
+ options);
+}
+
+TEST_F(TestUnifySchemas, Struct) {
+ auto options = Field::MergeOptions::Defaults();
+ options.promote_nested = true;
+ options.promote_numeric_width = true;
+ options.promote_binary = true;
+
+ CheckUnify(struct_({}), struct_({field("a", int8())}), struct_({field("a", int8())}),
+ options);
+
+ CheckUnifyAsymmetric(struct_({field("b", utf8())}), struct_({field("a", int8())}),
+ struct_({field("b", utf8()), field("a", int8())}), options);
+ CheckUnifyAsymmetric(struct_({field("a", int8())}), struct_({field("b", utf8())}),
+ struct_({field("a", int8()), field("b", utf8())}), options);
+
+ CheckUnify(struct_({field("b", utf8())}), struct_({field("b", binary())}),
+ struct_({field("b", binary())}), options);
+
+ CheckUnifyAsymmetric(
+ struct_({field("a", int8()), field("b", utf8()), field("a", int64())}),
+ struct_({field("b", binary())}),
+ struct_({field("a", int8()), field("b", binary()), field("a", int64())}), options);
+
+ ASSERT_RAISES(
+ Invalid,
+ field("foo", struct_({field("a", int8()), field("b", utf8()), field("a", int64())}))
+ ->MergeWith(field("foo", struct_({field("a", int64())})), options));
Review Comment:
For the record, that PR was merged, so you can use it now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org