You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/10/05 09:48:10 UTC
[arrow] 31/45: ARROW-6494: [C++][Dataset] Implement PartitionSchemes
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit a6787c8bcf56a6a28583fc885db7b26834181f47
Author: Benjamin Kietzman <be...@gmail.com>
AuthorDate: Thu Oct 3 13:42:54 2019 -0400
ARROW-6494: [C++][Dataset] Implement PartitionSchemes
Implements SimplePartitonScheme (ignores paths in favor of a preconfigured partition expression) and HivePartitionScheme (which parses paths containing embedded equality expressions)
Closes #5443 from bkietz/6494-Dataset-Implement-basic-P and squashes the following commits:
e385fdcce <Benjamin Kietzman> fix test class; should probably just use unique_ptr
f3acb2128 <Benjamin Kietzman> add Status return + out arg overload of Parse
6d7bf89d2 <Benjamin Kietzman> address review comments
dd9c2ee10 <Benjamin Kietzman> add cpp/.clangd to gitignore
ca1e50647 <Benjamin Kietzman> restrict MakeScalar to ScalarType::ValueType
a7a2e7d73 <Benjamin Kietzman> even more explicit StringScalar constructor for msvc
9e57edf0c <Benjamin Kietzman> amend cpplint to ignore alt_tokens preceding parens
f85f36823 <Benjamin Kietzman> make StringScalar constructor more explicit for msvc
a0ea783a0 <Benjamin Kietzman> revert composability changes
af984240f <Benjamin Kietzman> rewrite scalar factories to avoid implicit conversion
a499cf8c2 <Benjamin Kietzman> refactor vector and_, or_ to avoid recursion
1ed360818 <Benjamin Kietzman> rename to ConstantPartitionScheme
f2b7c9f23 <Benjamin Kietzman> add non equality conjunction partition examples
86485225d <Benjamin Kietzman> add convenience Parse() overload, more Hive tests
52582ef6b <Benjamin Kietzman> lint fixes
57106721f <Benjamin Kietzman> refactor PartitionScheme to allow chaining
cf1c1b58b <Benjamin Kietzman> extract scalar factories from ScalarExpression
a1a12b617 <Benjamin Kietzman> move optional detection/handling to stl.h
ec65f665c <Benjamin Kietzman> refactor partition tests to share more code
01be61c3e <Benjamin Kietzman> first pass at HivePartitionScheme
570fa2246 <Benjamin Kietzman> ARROW-6494: Implement PartitionSchemes
Authored-by: Benjamin Kietzman <be...@gmail.com>
Signed-off-by: Benjamin Kietzman <be...@gmail.com>
---
.gitignore | 1 +
cpp/build-support/cpplint.py | 2 +
cpp/src/arrow/compare.cc | 3 +-
cpp/src/arrow/dataset/CMakeLists.txt | 3 +
cpp/src/arrow/dataset/file_test.cc | 1 +
cpp/src/arrow/dataset/filter.cc | 131 ++++++++++--------
cpp/src/arrow/dataset/filter.h | 45 +++----
cpp/src/arrow/dataset/filter_test.cc | 10 +-
cpp/src/arrow/dataset/partition.cc | 94 +++++++++++++
cpp/src/arrow/dataset/partition.h | 226 +++++++++++++-------------------
cpp/src/arrow/dataset/partition_test.cc | 226 ++++++++++++++++++++++++++++++++
cpp/src/arrow/result.h | 19 ++-
cpp/src/arrow/result_test.cc | 80 +++++++++--
cpp/src/arrow/scalar.cc | 98 ++++++++++----
cpp/src/arrow/scalar.h | 203 ++++++++++++++++++++++------
cpp/src/arrow/scalar_test.cc | 137 +++++++++++++++----
cpp/src/arrow/stl.h | 48 ++++---
cpp/src/arrow/stl_test.cc | 6 +-
cpp/src/arrow/type_traits.h | 42 +-----
19 files changed, 987 insertions(+), 388 deletions(-)
diff --git a/.gitignore b/.gitignore
index 8f1de31..9df5101 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,6 +45,7 @@ perf.data
perf.data.old
cpp/.idea/
+cpp/.clangd/
cpp/apidoc/xml/
docs/example.gz
docs/example1.dat
diff --git a/cpp/build-support/cpplint.py b/cpp/build-support/cpplint.py
index 62a22d6..81a3b2a 100755
--- a/cpp/build-support/cpplint.py
+++ b/cpp/build-support/cpplint.py
@@ -3103,6 +3103,8 @@ def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error):
not Search(r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) and
not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and
not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and
+ not Search(r'\b(' + '|'.join(_ALT_TOKEN_REPLACEMENT.keys()) + r')\b\s+\(',
+ fncall) and
not Search(r'\bcase\s+\(', fncall)):
# TODO(unknown): Space after an operator function seem to be a common
# error, silence those for now by restricting them to highest verbosity.
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index b751c4e..eae54e7 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -886,8 +886,7 @@ class ScalarEqualsVisitor {
Status Visit(const MapScalar& left) {
const auto& right = checked_cast<const MapScalar&>(right_);
- result_ = internal::SharedPtrEquals(left.keys, right.keys) &&
- internal::SharedPtrEquals(left.items, right.items);
+ result_ = internal::SharedPtrEquals(left.value, right.value);
return Status::OK();
}
diff --git a/cpp/src/arrow/dataset/CMakeLists.txt b/cpp/src/arrow/dataset/CMakeLists.txt
index 442e9e6..0a99846 100644
--- a/cpp/src/arrow/dataset/CMakeLists.txt
+++ b/cpp/src/arrow/dataset/CMakeLists.txt
@@ -28,7 +28,9 @@ set(ARROW_DATASET_SRCS
discovery.cc
file_base.cc
filter.cc
+ partition.cc
scanner.cc)
+
set(ARROW_DATASET_LINK_STATIC arrow_static)
set(ARROW_DATASET_LINK_SHARED arrow_shared)
@@ -99,6 +101,7 @@ if(NOT WIN32)
add_arrow_dataset_test(discovery_test)
add_arrow_dataset_test(file_test)
add_arrow_dataset_test(filter_test)
+ add_arrow_dataset_test(partition_test)
add_arrow_dataset_test(scanner_test)
if(ARROW_PARQUET)
diff --git a/cpp/src/arrow/dataset/file_test.cc b/cpp/src/arrow/dataset/file_test.cc
index be6af4c..806dcca 100644
--- a/cpp/src/arrow/dataset/file_test.cc
+++ b/cpp/src/arrow/dataset/file_test.cc
@@ -24,6 +24,7 @@
#include <gtest/gtest.h>
#include "arrow/dataset/api.h"
+#include "arrow/dataset/partition.h"
#include "arrow/dataset/test_util.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
diff --git a/cpp/src/arrow/dataset/filter.cc b/cpp/src/arrow/dataset/filter.cc
index bcdee28..9c059d1 100644
--- a/cpp/src/arrow/dataset/filter.cc
+++ b/cpp/src/arrow/dataset/filter.cc
@@ -19,9 +19,11 @@
#include <algorithm>
#include <cstring>
+#include <numeric>
#include <string>
#include <unordered_map>
#include <utility>
+#include <vector>
#include "arrow/buffer.h"
#include "arrow/buffer_builder.h"
@@ -45,7 +47,11 @@ Result<Datum> ScalarExpression::Evaluate(compute::FunctionContext* ctx,
return value_;
}
-Datum NullDatum() { return Datum(std::make_shared<NullScalar>()); }
+inline std::shared_ptr<ScalarExpression> NullExpression() {
+ return std::make_shared<ScalarExpression>(std::make_shared<BooleanScalar>());
+}
+
+inline Datum NullDatum() { return Datum(std::make_shared<NullScalar>()); }
Result<Datum> FieldExpression::Evaluate(compute::FunctionContext* ctx,
const RecordBatch& batch) const {
@@ -187,23 +193,6 @@ Result<Datum> ComparisonExpression::Evaluate(compute::FunctionContext* ctx,
return std::move(out);
}
-std::shared_ptr<ScalarExpression> ScalarExpression::Make(std::string value) {
- return std::make_shared<ScalarExpression>(
- std::make_shared<StringScalar>(Buffer::FromString(std::move(value))));
-}
-
-std::shared_ptr<ScalarExpression> ScalarExpression::Make(const char* value) {
- return std::make_shared<ScalarExpression>(
- std::make_shared<StringScalar>(Buffer::Wrap(value, std::strlen(value))));
-}
-
-std::shared_ptr<ScalarExpression> ScalarExpression::MakeNull(
- const std::shared_ptr<DataType>& type) {
- std::shared_ptr<Scalar> null;
- DCHECK_OK(arrow::MakeNullScalar(type, &null));
- return Make(std::move(null));
-}
-
struct Comparison {
enum type {
LESS,
@@ -384,7 +373,7 @@ Result<std::shared_ptr<Expression>> ComparisonExpression::Assume(
if (simplified->IsNull()) {
// some subexpression of given is always null, return null
- return ScalarExpression::MakeNull(boolean());
+ return NullExpression();
}
bool trivial;
@@ -404,11 +393,11 @@ Result<std::shared_ptr<Expression>> ComparisonExpression::Assume(
}
if (simplify_to_always) {
- return ScalarExpression::Make(true);
+ return scalar(true);
}
if (simplify_to_never) {
- return ScalarExpression::Make(false);
+ return scalar(false);
}
return Copy();
@@ -420,7 +409,7 @@ Result<std::shared_ptr<Expression>> ComparisonExpression::Assume(
auto simplified = Copy();
for (const auto& operand : {given_and.left_operand(), given_and.right_operand()}) {
if (simplified->IsNull()) {
- return ScalarExpression::MakeNull(boolean());
+ return NullExpression();
}
if (simplified->IsTrivialCondition()) {
@@ -469,11 +458,11 @@ Result<std::shared_ptr<Expression>> ComparisonExpression::AssumeGivenComparison(
if (cmp == Comparison::NULL_) {
// the RHS of e or given was null
- return ScalarExpression::MakeNull(boolean());
+ return NullExpression();
}
- static auto always = ScalarExpression::Make(true);
- static auto never = ScalarExpression::Make(false);
+ static auto always = scalar(true);
+ static auto never = scalar(false);
using compute::CompareOperator;
@@ -619,7 +608,7 @@ Result<std::shared_ptr<Expression>> AndExpression::Assume(const Expression& give
// if either operand is trivially null then so is this AND
if (left_operand->IsNull() || right_operand->IsNull()) {
- return ScalarExpression::MakeNull(boolean());
+ return NullExpression();
}
bool left_trivial, right_trivial;
@@ -637,7 +626,7 @@ Result<std::shared_ptr<Expression>> AndExpression::Assume(const Expression& give
(right_is_trivial && right_trivial == false)) {
// FIXME(bkietz) if left is false and right is a column conaining nulls, this is an
// error because we should be yielding null there rather than false
- return ScalarExpression::Make(false);
+ return scalar(false);
}
// at least one of the operands is trivially true; return the other operand
@@ -650,7 +639,7 @@ Result<std::shared_ptr<Expression>> OrExpression::Assume(const Expression& given
// if either operand is trivially null then so is this OR
if (left_operand->IsNull() || right_operand->IsNull()) {
- return ScalarExpression::MakeNull(boolean());
+ return NullExpression();
}
bool left_trivial, right_trivial;
@@ -668,7 +657,7 @@ Result<std::shared_ptr<Expression>> OrExpression::Assume(const Expression& given
(right_is_trivial && right_trivial == true)) {
// FIXME(bkietz) if left is true but right is a column conaining nulls, this is an
// error because we should be yielding null there rather than true
- return ScalarExpression::Make(true);
+ return scalar(true);
}
// at least one of the operands is trivially false; return the other operand
@@ -679,12 +668,12 @@ Result<std::shared_ptr<Expression>> NotExpression::Assume(const Expression& give
ARROW_ASSIGN_OR_RAISE(auto operand, operand_->Assume(given));
if (operand->IsNull()) {
- return ScalarExpression::MakeNull(boolean());
+ return NullExpression();
}
bool trivial;
if (operand->IsTrivialCondition(&trivial)) {
- return ScalarExpression::Make(!trivial);
+ return scalar(!trivial);
}
return Copy();
@@ -715,34 +704,46 @@ std::string OperatorName(compute::CompareOperator op) {
return "";
}
+// TODO(bkietz) extract this to Scalar::ToString()
+struct ScalarExpressionToString {
+ Status Visit(const BooleanType&) {
+ return Finish(CastValue<BooleanType>().value ? "true" : "false");
+ }
+
+ template <typename T>
+ enable_if_number<T, Status> Visit(const T&) {
+ return Finish(std::to_string(CastValue<T>().value));
+ }
+
+ Status Visit(const StringType&) {
+ return Finish(CastValue<StringType>().value->ToString());
+ }
+
+ Status Visit(const DataType&) { return Finish("TODO(bkietz)"); }
+
+ Status Finish(std::string repr) {
+ *repr_ = std::move(repr);
+ return Status::OK();
+ }
+
+ template <typename T>
+ const typename TypeTraits<T>::ScalarType& CastValue() {
+ return checked_cast<const typename TypeTraits<T>::ScalarType&>(value_);
+ }
+
+ const Scalar& value_;
+ std::string* repr_;
+};
+
std::string ScalarExpression::ToString() const {
if (!value_->is_valid) {
return "scalar<" + value_->type->ToString() + ", null>()";
}
- std::string value;
- switch (value_->type->id()) {
- case Type::BOOL:
- value = checked_cast<const BooleanScalar&>(*value_).value ? "true" : "false";
- break;
- case Type::INT32:
- value = std::to_string(checked_cast<const Int32Scalar&>(*value_).value);
- break;
- case Type::INT64:
- value = std::to_string(checked_cast<const Int64Scalar&>(*value_).value);
- break;
- case Type::DOUBLE:
- value = std::to_string(checked_cast<const DoubleScalar&>(*value_).value);
- break;
- case Type::STRING:
- value = checked_cast<const StringScalar&>(*value_).value->ToString();
- break;
- default:
- value = "TODO(bkietz)";
- break;
- }
-
- return "scalar<" + value_->type->ToString() + ">(" + value + ")";
+ std::string repr;
+ ScalarExpressionToString impl{*value_, &repr};
+ DCHECK_OK(VisitTypeInline(*value_->type, &impl));
+ return "scalar<" + value_->type->ToString() + ">(" + repr + ")";
}
static std::string EulerNotation(std::string fn, const ExpressionVector& operands) {
@@ -865,11 +866,33 @@ std::shared_ptr<AndExpression> and_(std::shared_ptr<Expression> lhs,
return std::make_shared<AndExpression>(std::move(lhs), std::move(rhs));
}
+std::shared_ptr<Expression> and_(const ExpressionVector& subexpressions) {
+ if (subexpressions.size() == 0) {
+ return scalar(true);
+ }
+ return std::accumulate(
+ subexpressions.begin(), subexpressions.end(), std::shared_ptr<Expression>(),
+ [](std::shared_ptr<Expression> acc, const std::shared_ptr<Expression>& next) {
+ return acc == nullptr ? next : and_(std::move(acc), next);
+ });
+}
+
std::shared_ptr<OrExpression> or_(std::shared_ptr<Expression> lhs,
std::shared_ptr<Expression> rhs) {
return std::make_shared<OrExpression>(std::move(lhs), std::move(rhs));
}
+std::shared_ptr<Expression> or_(const ExpressionVector& subexpressions) {
+ if (subexpressions.size() == 0) {
+ return scalar(false);
+ }
+ return std::accumulate(
+ subexpressions.begin(), subexpressions.end(), std::shared_ptr<Expression>(),
+ [](std::shared_ptr<Expression> acc, const std::shared_ptr<Expression>& next) {
+ return acc == nullptr ? next : or_(std::move(acc), next);
+ });
+}
+
std::shared_ptr<NotExpression> not_(std::shared_ptr<Expression> operand) {
return std::make_shared<NotExpression>(std::move(operand));
}
diff --git a/cpp/src/arrow/dataset/filter.h b/cpp/src/arrow/dataset/filter.h
index 3f122c1..a06b936 100644
--- a/cpp/src/arrow/dataset/filter.h
+++ b/cpp/src/arrow/dataset/filter.h
@@ -20,6 +20,7 @@
#include <memory>
#include <string>
#include <utility>
+#include <vector>
#include "arrow/compute/kernel.h"
#include "arrow/compute/kernels/compare.h"
@@ -295,30 +296,6 @@ class ARROW_DS_EXPORT ScalarExpression final : public Expression {
bool Equals(const Expression& other) const override;
- static std::shared_ptr<ScalarExpression> Make(bool value) {
- return std::make_shared<ScalarExpression>(std::make_shared<BooleanScalar>(value));
- }
-
- template <typename T>
- static typename std::enable_if<std::is_integral<T>::value ||
- std::is_floating_point<T>::value,
- std::shared_ptr<ScalarExpression>>::type
- Make(T value) {
- using ScalarType = typename CTypeTraits<T>::ScalarType;
- return std::make_shared<ScalarExpression>(std::make_shared<ScalarType>(value));
- }
-
- static std::shared_ptr<ScalarExpression> Make(std::string value);
-
- static std::shared_ptr<ScalarExpression> Make(const char* value);
-
- static std::shared_ptr<ScalarExpression> Make(std::shared_ptr<Scalar> value) {
- return std::make_shared<ScalarExpression>(std::move(value));
- }
-
- static std::shared_ptr<ScalarExpression> MakeNull(
- const std::shared_ptr<DataType>& type);
-
Result<std::shared_ptr<DataType>> Validate(const Schema& schema) const override;
Result<compute::Datum> Evaluate(compute::FunctionContext* ctx,
@@ -357,17 +334,30 @@ class ARROW_DS_EXPORT FieldExpression final : public Expression {
ARROW_DS_EXPORT std::shared_ptr<AndExpression> and_(std::shared_ptr<Expression> lhs,
std::shared_ptr<Expression> rhs);
+ARROW_DS_EXPORT std::shared_ptr<Expression> and_(const ExpressionVector& subexpressions);
+
ARROW_DS_EXPORT AndExpression operator&&(const Expression& lhs, const Expression& rhs);
ARROW_DS_EXPORT std::shared_ptr<OrExpression> or_(std::shared_ptr<Expression> lhs,
std::shared_ptr<Expression> rhs);
+ARROW_DS_EXPORT std::shared_ptr<Expression> or_(const ExpressionVector& subexpressions);
+
ARROW_DS_EXPORT OrExpression operator||(const Expression& lhs, const Expression& rhs);
ARROW_DS_EXPORT std::shared_ptr<NotExpression> not_(std::shared_ptr<Expression> operand);
ARROW_DS_EXPORT NotExpression operator!(const Expression& rhs);
+inline std::shared_ptr<ScalarExpression> scalar(std::shared_ptr<Scalar> value) {
+ return std::make_shared<ScalarExpression>(std::move(value));
+}
+
+template <typename T>
+auto scalar(T&& value) -> decltype(scalar(MakeScalar(std::forward<T>(value)))) {
+ return scalar(MakeScalar(std::forward<T>(value)));
+}
+
#define COMPARISON_FACTORY(NAME, FACTORY_NAME, OP) \
inline std::shared_ptr<ComparisonExpression> FACTORY_NAME( \
const std::shared_ptr<FieldExpression>& lhs, \
@@ -379,7 +369,7 @@ ARROW_DS_EXPORT NotExpression operator!(const Expression& rhs);
template <typename T> \
ComparisonExpression operator OP(const FieldExpression& lhs, T&& rhs) { \
return ComparisonExpression(compute::CompareOperator::NAME, lhs.Copy(), \
- ScalarExpression::Make(std::forward<T>(rhs))); \
+ scalar(std::forward<T>(rhs))); \
}
COMPARISON_FACTORY(EQUAL, equal, ==)
COMPARISON_FACTORY(NOT_EQUAL, not_equal, !=)
@@ -389,11 +379,6 @@ COMPARISON_FACTORY(LESS, less, <)
COMPARISON_FACTORY(LESS_EQUAL, less_equal, <=)
#undef COMPARISON_FACTORY
-template <typename T>
-auto scalar(T&& value) -> decltype(ScalarExpression::Make(std::forward<T>(value))) {
- return ScalarExpression::Make(std::forward<T>(value));
-}
-
inline std::shared_ptr<FieldExpression> field_ref(std::string name) {
return std::make_shared<FieldExpression>(std::move(name));
}
diff --git a/cpp/src/arrow/dataset/filter_test.cc b/cpp/src/arrow/dataset/filter_test.cc
index a0783e2..8c92b7b 100644
--- a/cpp/src/arrow/dataset/filter_test.cc
+++ b/cpp/src/arrow/dataset/filter_test.cc
@@ -62,8 +62,8 @@ class ExpressionsTest : public ::testing::Test {
ASSERT_TRUE(expr.right_operand()->Equals(rhs));
}
- std::shared_ptr<ScalarExpression> always = ScalarExpression::Make(true);
- std::shared_ptr<ScalarExpression> never = ScalarExpression::Make(false);
+ std::shared_ptr<ScalarExpression> always = scalar(true);
+ std::shared_ptr<ScalarExpression> never = scalar(false);
};
TEST_F(ExpressionsTest, Equality) {
@@ -106,8 +106,8 @@ TEST_F(ExpressionsTest, SimplificationAgainstCompoundCondition) {
}
TEST_F(ExpressionsTest, SimplificationToNull) {
- auto null = ScalarExpression::MakeNull(boolean());
- auto null32 = ScalarExpression::MakeNull(int32());
+ auto null = scalar(std::shared_ptr<Scalar>(new BooleanScalar));
+ auto null32 = scalar(std::shared_ptr<Scalar>(new Int32Scalar));
AssertSimplifiesTo(*equal(field_ref("b"), null32), "b"_ == 3, *null);
AssertSimplifiesTo(*not_equal(field_ref("b"), null32), "b"_ == 3, *null);
@@ -191,7 +191,7 @@ TEST_F(FilterTest, Trivial) {
{"a": 0, "b": 1.0, "in": 0}
])");
- AssertFilter(*ScalarExpression::MakeNull(boolean()),
+ AssertFilter(*scalar(std::shared_ptr<Scalar>(new BooleanScalar)),
{field("a", int32()), field("b", float64())}, R"([
{"a": 0, "b": -0.1, "in": null},
{"a": 0, "b": 0.3, "in": null},
diff --git a/cpp/src/arrow/dataset/partition.cc b/cpp/src/arrow/dataset/partition.cc
new file mode 100644
index 0000000..e30a017
--- /dev/null
+++ b/cpp/src/arrow/dataset/partition.cc
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/dataset/partition.h"
+
+#include <memory>
+#include <utility>
+
+#include "arrow/dataset/filter.h"
+#include "arrow/dataset/scanner.h"
+#include "arrow/filesystem/path_util.h"
+#include "arrow/scalar.h"
+#include "arrow/util/iterator.h"
+#include "arrow/util/stl.h"
+
+namespace arrow {
+namespace dataset {
+
+Result<std::shared_ptr<Expression>> ConvertPartitionKeys(
+ const std::vector<UnconvertedKey>& keys, const Schema& schema) {
+ ExpressionVector subexpressions;
+
+ for (const auto& key : keys) {
+ auto field = schema.GetFieldByName(key.name);
+ if (field == nullptr) {
+ continue;
+ }
+
+ std::shared_ptr<Scalar> converted;
+ RETURN_NOT_OK(Scalar::Parse(field->type(), key.value, &converted));
+ subexpressions.push_back(equal(field_ref(field->name()), scalar(converted)));
+ }
+
+ return and_(subexpressions);
+}
+
+Result<std::shared_ptr<Expression>> ConstantPartitionScheme::Parse(
+ const std::string& path) const {
+ return expression_;
+}
+
+Result<std::shared_ptr<Expression>> SchemaPartitionScheme::Parse(
+ const std::string& path) const {
+ auto segments = fs::internal::SplitAbstractPath(path);
+ if (static_cast<int>(segments.size()) < schema_->num_fields()) {
+ return Status::Invalid("path had too few segments (", segments.size(), ") to parse ",
+ schema_->num_fields(), " fields");
+ }
+
+ std::vector<UnconvertedKey> keys(schema_->num_fields());
+ for (int i = 0; i < schema_->num_fields(); ++i) {
+ keys[i].name = schema_->field(i)->name();
+ keys[i].value = std::move(segments[i]);
+ }
+
+ return ConvertPartitionKeys(keys, *schema_);
+}
+
+std::vector<UnconvertedKey> HivePartitionScheme::GetUnconvertedKeys(
+ const std::string& path) const {
+ auto segments = fs::internal::SplitAbstractPath(path);
+
+ std::vector<UnconvertedKey> keys;
+ for (const auto& segment : segments) {
+ std::smatch matches;
+ static std::regex hive_style("^([^=]+)=(.*)$");
+ if (std::regex_match(segment, matches, hive_style) && matches.size() == 3) {
+ keys.push_back({matches[1].str(), matches[2].str()});
+ }
+ }
+ return keys;
+}
+
+Result<std::shared_ptr<Expression>> HivePartitionScheme::Parse(
+ const std::string& path) const {
+ return ConvertPartitionKeys(GetUnconvertedKeys(path), *schema_);
+}
+
+} // namespace dataset
+} // namespace arrow
diff --git a/cpp/src/arrow/dataset/partition.h b/cpp/src/arrow/dataset/partition.h
index d6adf15..a7ecf8d 100644
--- a/cpp/src/arrow/dataset/partition.h
+++ b/cpp/src/arrow/dataset/partition.h
@@ -17,12 +17,16 @@
#pragma once
+#include <functional>
#include <memory>
+#include <regex>
#include <string>
+#include <unordered_map>
#include <utility>
#include <vector>
#include "arrow/dataset/dataset.h"
+#include "arrow/dataset/filter.h"
#include "arrow/dataset/type_fwd.h"
#include "arrow/dataset/visibility.h"
@@ -30,65 +34,33 @@ namespace arrow {
namespace dataset {
// ----------------------------------------------------------------------
-// Computing partition values
-
-// TODO(wesm): API for computing partition keys derived from raw
-// values. For example, year(value) or hash_function(value) instead of
-// simply value, so a dataset with a timestamp column might group all
-// data with year 2009 in the same partition
-
-// /// \brief
-// class ScalarTransform {
-// public:
-// virtual Status Transform(const std::shared_ptr<Scalar>& input,
-// std::shared_ptr<Scalar>* output) const = 0;
-// };
-
-// class PartitionField {
-// public:
-
-// private:
-// std::string field_name_;
-// };
-
-// ----------------------------------------------------------------------
-// Partition identifiers
-
-/// \brief A partition level identifier which can be used
-///
-/// TODO(wesm): Is this general enough? What other kinds of partition
-/// keys exist and do we need to support them?
-class PartitionKey {
- public:
- const std::vector<std::string>& fields() const { return fields_; }
- const std::vector<std::shared_ptr<Scalar>>& values() const { return values_; }
+// Partition schemes
- private:
- std::vector<std::string> fields_;
- std::vector<std::shared_ptr<Scalar>> values_;
+struct ARROW_DS_EXPORT UnconvertedKey {
+ std::string name, value;
};
-/// \brief Intermediate data structure for data parsed from a string
-/// partition identifier.
+/// \brief Helper function for the common case of combining partition information
+/// consisting of equality expressions into a single conjunction expression.
+/// Fields referenced in keys but absent from schema will be ignored.
+ARROW_DS_EXPORT
+Result<std::shared_ptr<Expression>> ConvertPartitionKeys(
+ const std::vector<UnconvertedKey>& keys, const Schema& schema);
+
+/// \brief Interface for parsing partition expressions from string partition
+/// identifiers.
///
-/// For example, the identifier "foo=5" might be parsed with a single
-/// "foo" field and the value 5. A more complex identifier might be
-/// written as "foo=5,bar=2", which would yield two fields and two
-/// values.
+/// For example, the identifier "foo=5" might be parsed to an equality expression
+/// between the "foo" field and the value 5.
///
/// Some partition schemes may store the field names in a metadata
/// store instead of in file paths, for example
/// dataset_root/2009/11/... could be used when the partition fields
/// are "year" and "month"
-struct PartitionKeyData {
- std::vector<std::string> fields;
- std::vector<std::shared_ptr<Scalar>> values;
-};
-
-// ----------------------------------------------------------------------
-// Partition schemes
-
-/// \brief
+///
+/// Paths are consumed from left to right. Paths must be relative to
+/// the root of a partition; path prefixes must be removed before passing
+/// the path to a scheme for parsing.
class ARROW_DS_EXPORT PartitionScheme {
public:
virtual ~PartitionScheme() = default;
@@ -96,119 +68,99 @@ class ARROW_DS_EXPORT PartitionScheme {
/// \brief The name identifying the kind of partition scheme
virtual std::string name() const = 0;
- virtual bool PathMatchesScheme(const std::string& path) const = 0;
+ /// \brief Parse a path into a partition expression
+ ///
+ /// \param[in] path the partition identifier to parse
+ /// \return the parsed expression
+ virtual Result<std::shared_ptr<Expression>> Parse(const std::string& path) const = 0;
- virtual Status ParseKey(const std::string& path, PartitionKeyData* out) const = 0;
+ /// \brief Status return + out arg overload
+ Status Parse(const std::string& path, std::shared_ptr<Expression>* out) const {
+ return Parse(path).Value(out);
+ }
};
-/// \brief Multi-level, directory based partitioning scheme
-/// originating from Apache Hive with all data files stored in the
-/// leaf directories. Data is partitioned by static values of a
-/// particular column in the schema. Partition keys are represented in
-/// the form $key=$value in directory names
-class ARROW_DS_EXPORT HivePartitionScheme : public PartitionScheme {
+/// \brief Trivial partition scheme which yields an expression provided on construction.
+class ARROW_DS_EXPORT ConstantPartitionScheme : public PartitionScheme {
public:
- /// \brief Return true if path
- bool PathMatchesScheme(const std::string& path) const override;
-
- virtual Status ParseKey(const std::string& path, PartitionKeyData* out) const = 0;
-};
+ explicit ConstantPartitionScheme(std::shared_ptr<Expression> expr)
+ : expression_(std::move(expr)) {}
-// ----------------------------------------------------------------------
-//
-
-// Partitioned datasets come in different forms. Here is an example of
-// a Hive-style partitioned dataset:
-//
-// dataset_root/
-// key1=$k1_v1/
-// key2=$k2_v1/
-// 0.parquet
-// 1.parquet
-// 2.parquet
-// 3.parquet
-// key2=$k2_v2/
-// 0.parquet
-// 1.parquet
-// key1=$k1_v2/
-// key2=$k2_v1/
-// 0.parquet
-// 1.parquet
-// key2=$k2_v2/
-// 0.parquet
-// 1.parquet
-// 2.parquet
-//
-// In this case, the dataset has 11 fragments (11 files) to be
-// scanned, or potentially more if it is configured to split Parquet
-// files at the row group level
-
-class ARROW_DS_EXPORT Partition : public DataSource {
- public:
- std::string type() const override;
+ std::string name() const override { return "constant_partition_scheme"; }
- /// \brief The key for this partition source, may be nullptr,
- /// e.g. for the top-level partitioned source container
- virtual const PartitionKey* key() const = 0;
+ Result<std::shared_ptr<Expression>> Parse(const std::string& path) const override;
- virtual DataFragmentIterator GetFragments(const Selector& selector) = 0;
+ private:
+ std::shared_ptr<Expression> expression_;
};
-/// \brief Simple implementation of Partition, which consists of a
-/// partition identifier, subpartitions, and some data fragments
-class ARROW_DS_EXPORT SimplePartition : public Partition {
+/// \brief SchemaPartitionScheme parses one segment of a path for each field in its
+/// schema. All fields are required, so paths passed to SchemaPartitionScheme::Parse
+/// must contain segments for each field.
+///
+/// For example given schema<year:int16, month:int8> the path "/2009/11" would be
+/// parsed to ("year"_ == 2009 and "month"_ == 11)
+class ARROW_DS_EXPORT SchemaPartitionScheme : public PartitionScheme {
public:
- SimplePartition(std::unique_ptr<PartitionKey> partition_key,
- DataFragmentVector&& data_fragments, PartitionVector&& subpartitions,
- std::shared_ptr<ScanOptions> scan_options = NULLPTR)
- : key_(std::move(partition_key)),
- data_fragments_(std::move(data_fragments)),
- subpartitions_(std::move(subpartitions)),
- scan_options_(scan_options) {}
+ explicit SchemaPartitionScheme(std::shared_ptr<Schema> schema)
+ : schema_(std::move(schema)) {}
- const PartitionKey* key() const override { return key_.get(); }
+ std::string name() const override { return "schema_partition_scheme"; }
- int num_subpartitions() const { return static_cast<int>(subpartitions_.size()); }
+ Result<std::shared_ptr<Expression>> Parse(const std::string& path) const override;
- int num_data_fragments() const { return static_cast<int>(data_fragments__.size()); }
+ const std::shared_ptr<Schema>& schema() { return schema_; }
- const PartitionVector& subpartitions() const { return subpartitions_; }
- const DataFragmentVector& data_fragments() const { return data_fragments_; }
+ protected:
+ std::shared_ptr<Schema> schema_;
+};
- DataFragmentIterator GetFragments(const FilterVector& filters) override;
+/// \brief Multi-level, directory based partitioning scheme
+/// originating from Apache Hive with all data files stored in the
+/// leaf directories. Data is partitioned by static values of a
+/// particular column in the schema. Partition keys are represented in
+/// the form $key=$value in directory names.
+/// Field order is ignored, as are missing or unrecognized field names.
+///
+/// For example given schema<year:int16, month:int8, day:int8> the path
+/// "/day=321/ignored=3.4/year=2009" parses to ("year"_ == 2009 and "day"_ == 321)
+class ARROW_DS_EXPORT HivePartitionScheme : public PartitionScheme {
+ public:
+ explicit HivePartitionScheme(std::shared_ptr<Schema> schema)
+ : schema_(std::move(schema)) {}
- private:
- std::unique_ptr<PartitionKey> key_;
+ std::string name() const override { return "hive_partition_scheme"; }
- /// \brief Data fragments belonging to this partition level. In some
- /// partition schemes such as Hive-style, this member is
- /// mutually-exclusive with subpartitions, where data fragments
- /// occur only in the partition leaves
- std::vector<std::shared_ptr<DataFragment>> data_fragments_;
+ Result<std::shared_ptr<Expression>> Parse(const std::string& path) const override;
- /// \brief Child partitions of this partition
- std::vector<std::shared_ptr<Partition>> subpartitions_;
+ std::vector<UnconvertedKey> GetUnconvertedKeys(const std::string& path) const;
- /// \brief Default scan options to use for data fragments
- std::shared_ptr<ScanOptions> scan_options_;
+ const std::shared_ptr<Schema>& schema() { return schema_; }
+
+ protected:
+ std::shared_ptr<Schema> schema_;
};
-/// \brief A PartitionSource that returns fragments as the result of input iterators
-class ARROW_DS_EXPORT LazyPartition : public Partition {
+/// \brief Implementation provided by lambda or other callable
+class ARROW_DS_EXPORT FunctionPartitionScheme : public PartitionScheme {
public:
- const PartitionKey* key() const override;
-
- DataFragmentIterator GetFragments(const& DataSelector selector) override;
+ explicit FunctionPartitionScheme(
+ std::function<Result<std::shared_ptr<Expression>>(const std::string&)> impl,
+ std::string name = "function_partition_scheme")
+ : impl_(std::move(impl)), name_(std::move(name)) {}
- // TODO(wesm): Iterate over subpartitions
+ std::string name() const override { return name_; }
- protected:
- PartitionIterator partition_iter_;
+ Result<std::shared_ptr<Expression>> Parse(const std::string& path) const override {
+ return impl_(path);
+ }
- // By default, once this source is consumed using GetFragments, it
- // cannot be consumed again. By setting this to true, we cache
- bool cache_manifest_ = false;
+ private:
+ std::function<Result<std::shared_ptr<Expression>>(const std::string&)> impl_;
+ std::string name_;
};
+// TODO(bkietz) use RE2 and named groups to provide RegexpPartitionScheme
+
} // namespace dataset
} // namespace arrow
diff --git a/cpp/src/arrow/dataset/partition_test.cc b/cpp/src/arrow/dataset/partition_test.cc
new file mode 100644
index 0000000..41d90d6
--- /dev/null
+++ b/cpp/src/arrow/dataset/partition_test.cc
@@ -0,0 +1,226 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "arrow/dataset/api.h"
+#include "arrow/dataset/partition.h"
+#include "arrow/dataset/test_util.h"
+#include "arrow/filesystem/localfs.h"
+#include "arrow/filesystem/path_util.h"
+#include "arrow/status.h"
+#include "arrow/testing/gtest_util.h"
+#include "arrow/util/io_util.h"
+
+namespace arrow {
+namespace dataset {
+
+class TestPartitionScheme : public ::testing::Test {
+ public:
+ void AssertParseError(const std::string& path) {
+ ASSERT_RAISES(Invalid, scheme_->Parse(path).status());
+ }
+
+ void AssertParse(const std::string& path, std::shared_ptr<Expression> expected) {
+ for (std::string suffix : {"", "/dat.parquet"}) {
+ ASSERT_OK_AND_ASSIGN(auto parsed, scheme_->Parse(path + suffix));
+
+ ASSERT_NE(parsed, nullptr);
+ ASSERT_TRUE(parsed->Equals(*expected)) << parsed->ToString() << "\n"
+ << expected->ToString();
+ }
+ }
+
+ void AssertParse(const std::string& path, const Expression& expected) {
+ AssertParse(path, expected.Copy());
+ }
+
+ protected:
+ std::shared_ptr<PartitionScheme> scheme_;
+};
+
+TEST_F(TestPartitionScheme, Simple) {
+ auto expr = equal(field_ref("alpha"), scalar<int16_t>(3));
+ scheme_ = std::make_shared<ConstantPartitionScheme>(expr);
+ AssertParse("/hello/world", expr);
+}
+
+TEST_F(TestPartitionScheme, Schema) {
+ scheme_ = std::make_shared<SchemaPartitionScheme>(
+ schema({field("alpha", int32()), field("beta", utf8())}));
+
+ AssertParse("/0/hello", "alpha"_ == int32_t(0) and "beta"_ == "hello");
+ AssertParseError("/world/0"); // reversed order
+ AssertParseError("/3"); // valid alpha, but missing beta
+ AssertParseError("/0.0/foo"); // invalid alpha
+ AssertParseError("/3.25"); // invalid alpha with missing beta
+ AssertParseError(""); // no segments to parse
+
+ // gotcha someday:
+ AssertParse("/0/dat.parquet", "alpha"_ == int32_t(0) and "beta"_ == "dat.parquet");
+
+ AssertParse("/0/foo/ignored=2341", "alpha"_ == int32_t(0) and "beta"_ == "foo");
+}
+
+TEST_F(TestPartitionScheme, Hive) {
+ scheme_ = std::make_shared<HivePartitionScheme>(
+ schema({field("alpha", int32()), field("beta", float32())}));
+
+ AssertParse("/alpha=0/beta=3.25", "alpha"_ == int32_t(0) and "beta"_ == 3.25f);
+ AssertParse("/beta=3.25/alpha=0", "beta"_ == 3.25f and "alpha"_ == int32_t(0));
+ AssertParse("/alpha=0", "alpha"_ == int32_t(0));
+ AssertParse("/beta=3.25", "beta"_ == 3.25f);
+ AssertParse("", scalar(true));
+
+ AssertParse("/alpha=0/unexpected/beta=3.25",
+ "alpha"_ == int32_t(0) and "beta"_ == 3.25f);
+
+ AssertParse("/alpha=0/beta=3.25/ignored=2341",
+ "alpha"_ == int32_t(0) and "beta"_ == 3.25f);
+
+ AssertParse("/ignored=2341", scalar(true));
+
+ AssertParseError("/alpha=0.0/beta=3.25"); // conversion of "0.0" to int32 fails
+}
+
+template <typename T>
+void PopFront(size_t n, std::vector<T>* v) {
+ std::move(v->begin() + n, v->end(), v->begin());
+ v->resize(v->size() - n);
+}
+
+TEST_F(TestPartitionScheme, EtlThenHive) {
+ SchemaPartitionScheme etl_scheme(schema({field("year", int16()), field("month", int8()),
+ field("day", int8()), field("hour", int8())}));
+ HivePartitionScheme alphabeta_scheme(
+ schema({field("alpha", int32()), field("beta", float32())}));
+
+ scheme_ = std::make_shared<FunctionPartitionScheme>(
+ [&](const std::string& path) -> Result<std::shared_ptr<Expression>> {
+ ARROW_ASSIGN_OR_RAISE(auto etl_expr, etl_scheme.Parse(path));
+
+ auto segments = fs::internal::SplitAbstractPath(path);
+ PopFront(etl_scheme.schema()->num_fields(), &segments);
+ ARROW_ASSIGN_OR_RAISE(
+ auto alphabeta_expr,
+ alphabeta_scheme.Parse(fs::internal::JoinAbstractPath(segments)));
+
+ return and_(std::move(etl_expr), std::move(alphabeta_expr));
+ });
+
+ AssertParse("/1999/12/31/00/alpha=0/beta=3.25",
+ "year"_ == int16_t(1999) and "month"_ == int8_t(12) and
+ "day"_ == int8_t(31) and "hour"_ == int8_t(0) and
+ ("alpha"_ == int32_t(0) and "beta"_ == 3.25f));
+
+ AssertParseError("/20X6/03/21/05/alpha=0/beta=3.25");
+}
+
+TEST_F(TestPartitionScheme, Set) {
+ // An adhoc partition scheme which parses segments like "/x in [1 4 5]"
+ // into ("x"_ == 1 or "x"_ == 4 or "x"_ == 5)
+ scheme_ = std::make_shared<FunctionPartitionScheme>(
+ [](const std::string& path) -> Result<std::shared_ptr<Expression>> {
+ std::smatch matches;
+ auto segment = std::move(fs::internal::SplitAbstractPath(path)[0]);
+ static std::regex re("^x in \\[(.*)\\]$");
+ if (!std::regex_match(segment, matches, re) || matches.size() != 2) {
+ return Status::Invalid("regex failed to parse");
+ }
+
+ ExpressionVector subexpressions;
+ std::string element;
+ std::istringstream elements(matches[1]);
+ while (elements >> element) {
+ std::shared_ptr<Scalar> s;
+ RETURN_NOT_OK(Scalar::Parse(int32(), element, &s));
+ subexpressions.push_back(equal(field_ref("x"), scalar(s)));
+ }
+
+ return or_(std::move(subexpressions));
+ });
+
+ AssertParse("/x in [1]", "x"_ == 1);
+ AssertParse("/x in [1 4 5]", "x"_ == 1 or "x"_ == 4 or "x"_ == 5);
+ AssertParse("/x in []", scalar(false));
+}
+
+// An adhoc partition scheme which parses segments like "/x=[-3.25, 0.0)"
+// into ("x"_ >= -3.25 and "x" < 0.0)
+class RangePartitionScheme : public HivePartitionScheme {
+ public:
+ using HivePartitionScheme::HivePartitionScheme;
+
+ std::string name() const override { return "range_partition_scheme"; }
+
+ Result<std::shared_ptr<Expression>> Parse(const std::string& path) const override {
+ ExpressionVector ranges;
+ for (auto key : GetUnconvertedKeys(path)) {
+ std::smatch matches;
+ RETURN_NOT_OK(DoRegex(key.value, &matches));
+
+ auto& min_cmp = matches[1] == "[" ? greater_equal : greater;
+ std::string min_repr = matches[2];
+ std::string max_repr = matches[3];
+ auto& max_cmp = matches[4] == "]" ? less_equal : less;
+
+ const auto& type = schema_->GetFieldByName(key.name)->type();
+ std::shared_ptr<Scalar> min, max;
+ RETURN_NOT_OK(Scalar::Parse(type, min_repr, &min));
+ RETURN_NOT_OK(Scalar::Parse(type, max_repr, &max));
+
+ ranges.push_back(and_(min_cmp(field_ref(key.name), scalar(min)),
+ max_cmp(field_ref(key.name), scalar(max))));
+ }
+ return and_(ranges);
+ }
+
+ static Status DoRegex(const std::string& segment, std::smatch* matches) {
+ static std::regex re(
+ "^"
+ "([\\[\\(])" // open bracket or paren
+ "([^ ]+)" // representation of range minimum
+ " "
+ "([^ ]+)" // representation of range maximum
+ "([\\]\\)])" // close bracket or paren
+ "$");
+
+ if (!std::regex_match(segment, *matches, re) || matches->size() != 5) {
+ return Status::Invalid("regex failed to parse");
+ }
+
+ return Status::OK();
+ }
+};
+
+TEST_F(TestPartitionScheme, Range) {
+ scheme_ = std::make_shared<RangePartitionScheme>(
+ schema({field("x", float64()), field("y", float64()), field("z", float64())}));
+
+ AssertParse("/x=[-1.5 0.0)/y=[0.0 1.5)/z=(1.5 3.0]",
+ ("x"_ >= -1.5 and "x"_ < 0.0) and ("y"_ >= 0.0 and "y"_ < 1.5) and
+ ("z"_ > 1.5 and "z"_ <= 3.0));
+}
+
+} // namespace dataset
+} // namespace arrow
diff --git a/cpp/src/arrow/result.h b/cpp/src/arrow/result.h
index 0bbb045..f71d51e 100644
--- a/cpp/src/arrow/result.h
+++ b/cpp/src/arrow/result.h
@@ -170,9 +170,9 @@ class Result {
/// `T` must be implicitly constructible from `const U &`.
///
/// \param other The value to copy from.
- template <typename U,
- typename E = typename std::enable_if<std::is_constructible<T, U>::value &&
- std::is_convertible<U, T>::value>::type>
+ template <typename U, typename E = typename std::enable_if<
+ std::is_constructible<T, const U&>::value &&
+ std::is_convertible<U, T>::value>::type>
Result(const Result<U>& other) : variant_("unitialized") {
AssignVariant(other.variant_);
}
@@ -191,7 +191,7 @@ class Result {
///
/// \param other The Result object to move from and set to a non-OK status.
template <typename U,
- typename E = typename std::enable_if<std::is_constructible<T, U>::value &&
+ typename E = typename std::enable_if<std::is_constructible<T, U&&>::value &&
std::is_convertible<U, T>::value>::type>
Result(Result<U>&& other) : variant_("unitialized") {
AssignVariant(std::move(other.variant_));
@@ -284,6 +284,17 @@ class Result {
}
T operator*() && { return ValueOrDie(); }
+ /// Helper method for using Results in Status returning out arg functions
+ template <typename U, typename E = typename std::enable_if<
+ std::is_constructible<U, T>::value>::type>
+ Status Value(U* out) && {
+ if (!ok()) {
+ return status();
+ }
+ *out = U(std::move(arrow::util::get<T>(variant_)));
+ return Status::OK();
+ }
+
private:
// Assignment is disabled by default so we need to destruct/reconstruct
// the value.
diff --git a/cpp/src/arrow/result_test.cc b/cpp/src/arrow/result_test.cc
index 92a2cde..caecb11 100644
--- a/cpp/src/arrow/result_test.cc
+++ b/cpp/src/arrow/result_test.cc
@@ -67,17 +67,31 @@ struct ImplicitlyCopyConvertible {
struct MoveOnlyDataType {
explicit MoveOnlyDataType(int x) : data(new int(x)) {}
- MoveOnlyDataType(MoveOnlyDataType&& other) : data(other.data) { other.data = nullptr; }
-
MoveOnlyDataType(const MoveOnlyDataType& other) = delete;
MoveOnlyDataType& operator=(const MoveOnlyDataType& other) = delete;
- ~MoveOnlyDataType() {
- delete data;
- data = nullptr;
+ MoveOnlyDataType(MoveOnlyDataType&& other) { MoveFrom(other); }
+ MoveOnlyDataType& operator=(MoveOnlyDataType&& other) {
+ MoveFrom(other);
+ return *this;
+ }
+
+ ~MoveOnlyDataType() { Destroy(); }
+
+ void Destroy() {
+ if (data != nullptr) {
+ delete data;
+ data = nullptr;
+ }
}
- int* data;
+ void MoveFrom(MoveOnlyDataType& other) {
+ Destroy();
+ data = other.data;
+ other.data = nullptr;
+ }
+
+ int* data = nullptr;
};
struct ImplicitlyMoveConvertible {
@@ -397,8 +411,8 @@ TYPED_TEST(ResultTest, MoveAssignmentSelfOkStatus) {
// Verify that a Result object can be constructed from a move-only type.
TEST(ResultTest, InitializationMoveOnlyType) {
- std::string* str = new std::string(kStringElement);
- std::unique_ptr<std::string> value(str);
+ std::unique_ptr<std::string> value(new std::string(kStringElement));
+ auto str = value.get();
Result<std::unique_ptr<std::string>> result(std::move(value));
ASSERT_TRUE(result.ok());
@@ -407,8 +421,8 @@ TEST(ResultTest, InitializationMoveOnlyType) {
// Verify that a Result object can be move-constructed from a move-only type.
TEST(ResultTest, MoveConstructorMoveOnlyType) {
- std::string* str = new std::string(kStringElement);
- std::unique_ptr<std::string> value(str);
+ std::unique_ptr<std::string> value(new std::string(kStringElement));
+ auto str = value.get();
Result<std::unique_ptr<std::string>> result1(std::move(value));
Result<std::unique_ptr<std::string>> result2(std::move(result1));
@@ -424,8 +438,8 @@ TEST(ResultTest, MoveConstructorMoveOnlyType) {
// Verify that a Result object can be move-assigned to from a Result object
// containing a move-only type.
TEST(ResultTest, MoveAssignmentMoveOnlyType) {
- std::string* str = new std::string(kStringElement);
- std::unique_ptr<std::string> value(str);
+ std::unique_ptr<std::string> value(new std::string(kStringElement));
+ auto str = value.get();
Result<std::unique_ptr<std::string>> result1(std::move(value));
Result<std::unique_ptr<std::string>> result2(Status(kErrorCode, kErrorMessage));
@@ -443,8 +457,8 @@ TEST(ResultTest, MoveAssignmentMoveOnlyType) {
// Verify that a value can be moved out of a Result object via ValueOrDie().
TEST(ResultTest, ValueOrDieMovedValue) {
- std::string* str = new std::string(kStringElement);
- std::unique_ptr<std::string> value(str);
+ std::unique_ptr<std::string> value(new std::string(kStringElement));
+ auto str = value.get();
Result<std::unique_ptr<std::string>> result(std::move(value));
std::unique_ptr<std::string> moved_value = std::move(result).ValueOrDie();
@@ -475,6 +489,44 @@ TEST(ResultTest, TemplateValueMoveConstruction) {
EXPECT_EQ(*result.ValueOrDie().move_only.data, kIntElement);
}
+// Verify that a Result<T> can be unpacked to T
+TEST(ResultTest, StatusReturnAdapterCopyValue) {
+ Result<CopyOnlyDataType> result(CopyOnlyDataType{kIntElement});
+ CopyOnlyDataType copy_only{0};
+
+ EXPECT_TRUE(std::move(result).Value(©_only).ok());
+ EXPECT_EQ(copy_only.data, kIntElement);
+}
+
+// Verify that a Result<T> can be unpacked to some U, where U is
+// a type which has a constructor taking a const T &.
+TEST(ResultTest, StatusReturnAdapterCopyAndConvertValue) {
+ Result<CopyOnlyDataType> result(CopyOnlyDataType{kIntElement});
+ ImplicitlyCopyConvertible implicitly_convertible(CopyOnlyDataType{0});
+
+ EXPECT_TRUE(std::move(result).Value(&implicitly_convertible).ok());
+ EXPECT_EQ(implicitly_convertible.copy_only.data, kIntElement);
+}
+
+// Verify that a Result<T> can be unpacked to T
+TEST(ResultTest, StatusReturnAdapterMoveValue) {
+ Result<MoveOnlyDataType> result(MoveOnlyDataType{kIntElement});
+ MoveOnlyDataType move_only{0};
+
+ EXPECT_TRUE(std::move(result).Value(&move_only).ok());
+ EXPECT_EQ(*move_only.data, kIntElement);
+}
+
+// Verify that a Result<T> can be unpacked to some U, where U is
+// a type which has a constructor taking a T &&.
+TEST(ResultTest, StatusReturnAdapterMoveAndConvertValue) {
+ Result<MoveOnlyDataType> result(MoveOnlyDataType{kIntElement});
+ ImplicitlyMoveConvertible implicitly_convertible(MoveOnlyDataType{0});
+
+ EXPECT_TRUE(std::move(result).Value(&implicitly_convertible).ok());
+ EXPECT_EQ(*implicitly_convertible.move_only.data, kIntElement);
+}
+
// Verify that a Result<U> is assignable to a Result<T>, where T
// is a type which has an implicit constructor taking a const U &.
TEST(ResultTest, TemplateCopyAssign) {
diff --git a/cpp/src/arrow/scalar.cc b/cpp/src/arrow/scalar.cc
index 1001990..909d50c 100644
--- a/cpp/src/arrow/scalar.cc
+++ b/cpp/src/arrow/scalar.cc
@@ -18,6 +18,8 @@
#include "arrow/scalar.h"
#include <memory>
+#include <string>
+#include <utility>
#include "arrow/array.h"
#include "arrow/buffer.h"
@@ -26,6 +28,7 @@
#include "arrow/util/checked_cast.h"
#include "arrow/util/decimal.h"
#include "arrow/util/logging.h"
+#include "arrow/util/parsing.h"
#include "arrow/visitor_inline.h"
namespace arrow {
@@ -58,6 +61,9 @@ DurationScalar::DurationScalar(int64_t value, const std::shared_ptr<DataType>& t
DCHECK_EQ(Type::DURATION, type->id());
}
+MonthIntervalScalar::MonthIntervalScalar(int32_t value, bool is_valid)
+ : internal::PrimitiveScalar{month_interval(), is_valid}, value(value) {}
+
MonthIntervalScalar::MonthIntervalScalar(int32_t value,
const std::shared_ptr<DataType>& type,
bool is_valid)
@@ -68,6 +74,10 @@ MonthIntervalScalar::MonthIntervalScalar(int32_t value,
}
DayTimeIntervalScalar::DayTimeIntervalScalar(DayTimeIntervalType::DayMilliseconds value,
+ bool is_valid)
+ : internal::PrimitiveScalar{day_time_interval(), is_valid}, value(value) {}
+
+DayTimeIntervalScalar::DayTimeIntervalScalar(DayTimeIntervalType::DayMilliseconds value,
const std::shared_ptr<DataType>& type,
bool is_valid)
: internal::PrimitiveScalar{type, is_valid}, value(value) {
@@ -76,6 +86,9 @@ DayTimeIntervalScalar::DayTimeIntervalScalar(DayTimeIntervalType::DayMillisecond
checked_cast<IntervalType*>(type.get())->interval_type());
}
+StringScalar::StringScalar(std::string s)
+ : StringScalar(Buffer::FromString(std::move(s)), true) {}
+
FixedSizeBinaryScalar::FixedSizeBinaryScalar(const std::shared_ptr<Buffer>& value,
const std::shared_ptr<DataType>& type,
bool is_valid)
@@ -95,36 +108,23 @@ BaseListScalar::BaseListScalar(const std::shared_ptr<Array>& value,
BaseListScalar::BaseListScalar(const std::shared_ptr<Array>& value, bool is_valid)
: BaseListScalar(value, value->type(), is_valid) {}
-MapScalar::MapScalar(const std::shared_ptr<Array>& keys,
- const std::shared_ptr<Array>& items,
- const std::shared_ptr<DataType>& type, bool is_valid)
- : Scalar{type, is_valid}, keys(keys), items(items) {}
-
-MapScalar::MapScalar(const std::shared_ptr<Array>& keys,
- const std::shared_ptr<Array>& values, bool is_valid)
- : MapScalar(keys, values, map(keys->type(), values->type()), is_valid) {}
-
FixedSizeListScalar::FixedSizeListScalar(const std::shared_ptr<Array>& value,
const std::shared_ptr<DataType>& type,
bool is_valid)
- : Scalar{type, is_valid}, value(value) {
+ : BaseListScalar(value, type, is_valid) {
ARROW_CHECK_EQ(value->length(),
checked_cast<const FixedSizeListType*>(type.get())->list_size());
}
-FixedSizeListScalar::FixedSizeListScalar(const std::shared_ptr<Array>& value,
- bool is_valid)
- : FixedSizeListScalar(value, value->type(), is_valid) {}
-
+// TODO(bkietz) This doesn't need a factory. Just rewrite all scalars to be generically
+// constructible (is_simple_scalar should apply to all scalars)
struct MakeNullImpl {
- template <typename T>
- using ScalarType = typename TypeTraits<T>::ScalarType;
-
- template <typename T>
- typename std::enable_if<std::is_default_constructible<ScalarType<T>>::value,
- Status>::type
- Visit(const T&) {
- *out_ = std::make_shared<ScalarType<T>>();
+ template <typename T, typename ScalarType = typename TypeTraits<T>::ScalarType,
+ typename ValueType = typename ScalarType::ValueType,
+ typename Enable = typename std::enable_if<
+ internal::is_simple_scalar<ScalarType>::value>::type>
+ Status Visit(const T&) {
+ *out_ = std::make_shared<ScalarType>(ValueType(), type_, false);
return Status::OK();
}
@@ -132,7 +132,7 @@ struct MakeNullImpl {
return Status::NotImplemented("construcing null scalars of type ", t);
}
- std::shared_ptr<DataType> type_;
+ const std::shared_ptr<DataType>& type_;
std::shared_ptr<Scalar>* out_;
};
@@ -142,4 +142,56 @@ Status MakeNullScalar(const std::shared_ptr<DataType>& type,
return VisitTypeInline(*type, &impl);
}
+struct ScalarParseImpl {
+ template <typename T, typename Converter = internal::StringConverter<T>,
+ typename Value = typename Converter::value_type>
+ Status Visit(const T& t) {
+ Value value;
+ if (!Converter{type_}(s_.data(), s_.size(), &value)) {
+ return Status::Invalid("error parsing '", s_, "' as scalar of type ", t);
+ }
+ return Finish(std::move(value));
+ }
+
+ Status Visit(const BinaryType&) { return FinishWithBuffer(); }
+
+ Status Visit(const LargeBinaryType&) { return FinishWithBuffer(); }
+
+ Status Visit(const FixedSizeBinaryType& t) { return FinishWithBuffer(); }
+
+ Status Visit(const DataType& t) {
+ return Status::NotImplemented("parsing scalars of type ", t);
+ }
+
+ template <typename Arg>
+ Status Finish(Arg&& arg) {
+ return MakeScalar(type_, std::forward<Arg>(arg), out_);
+ }
+
+ Status FinishWithBuffer() { return Finish(Buffer::FromString(s_.to_string())); }
+
+ ScalarParseImpl(const std::shared_ptr<DataType>& type, util::string_view s,
+ std::shared_ptr<Scalar>* out)
+ : type_(type), s_(s), out_(out) {}
+
+ const std::shared_ptr<DataType>& type_;
+ util::string_view s_;
+ std::shared_ptr<Scalar>* out_;
+};
+
+Status Scalar::Parse(const std::shared_ptr<DataType>& type, util::string_view s,
+ std::shared_ptr<Scalar>* out) {
+ ScalarParseImpl impl = {type, s, out};
+ return VisitTypeInline(*type, &impl);
+}
+
+namespace internal {
+Status CheckBufferLength(const FixedSizeBinaryType* t, const std::shared_ptr<Buffer>* b) {
+ return t->byte_width() == (*b)->size()
+ ? Status::OK()
+ : Status::Invalid("buffer length ", (*b)->size(), " is not compatible with ",
+ *t);
+}
+} // namespace internal
+
} // namespace arrow
diff --git a/cpp/src/arrow/scalar.h b/cpp/src/arrow/scalar.h
index c407ebc..5ddb815 100644
--- a/cpp/src/arrow/scalar.h
+++ b/cpp/src/arrow/scalar.h
@@ -24,12 +24,16 @@
#pragma once
#include <memory>
+#include <string>
+#include <utility>
#include <vector>
#include "arrow/type.h"
#include "arrow/type_fwd.h"
#include "arrow/type_traits.h"
#include "arrow/util/decimal.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/string_view.h"
#include "arrow/util/visibility.h"
namespace arrow {
@@ -53,6 +57,9 @@ struct ARROW_EXPORT Scalar {
return false;
}
+ static Status Parse(const std::shared_ptr<DataType>& type, util::string_view s,
+ std::shared_ptr<Scalar>* out);
+
protected:
Scalar(const std::shared_ptr<DataType>& type, bool is_valid)
: type(type), is_valid(is_valid) {}
@@ -66,6 +73,22 @@ struct ARROW_EXPORT NullScalar : public Scalar {
namespace internal {
+template <typename T, typename Enable = void>
+struct is_simple_scalar : std::false_type {};
+
+template <typename T>
+struct is_simple_scalar<
+ T,
+ typename std::enable_if<
+ // scalar has a single extra data member named "value" with type "ValueType"
+ std::is_same<decltype(std::declval<T>().value), typename T::ValueType>::value &&
+ // scalar is constructible from (value, type, is_valid)
+ std::is_constructible<T, typename T::ValueType, std::shared_ptr<DataType>,
+ bool>::value>::type> : std::true_type {};
+
+template <typename T, typename R = void>
+using enable_if_simple_scalar = std::enable_if<is_simple_scalar<T>::value, R>;
+
struct ARROW_EXPORT PrimitiveScalar : public Scalar {
using Scalar::Scalar;
};
@@ -73,29 +96,44 @@ struct ARROW_EXPORT PrimitiveScalar : public Scalar {
} // namespace internal
struct ARROW_EXPORT BooleanScalar : public internal::PrimitiveScalar {
+ using ValueType = bool;
+
bool value;
+
explicit BooleanScalar(bool value, bool is_valid = true)
: internal::PrimitiveScalar{boolean(), is_valid}, value(value) {}
+
BooleanScalar() : BooleanScalar(false, false) {}
+
+ BooleanScalar(bool value, const std::shared_ptr<DataType>& type, bool is_valid = true)
+ : BooleanScalar(value, is_valid) {
+ ARROW_CHECK_EQ(type->id(), Type::BOOL);
+ }
};
-template <typename Type>
+template <typename T>
struct NumericScalar : public internal::PrimitiveScalar {
- using T = typename Type::c_type;
- T value;
+ using ValueType = typename T::c_type;
- explicit NumericScalar(T value, bool is_valid = true)
- : NumericScalar(value, TypeTraits<Type>::type_singleton(), is_valid) {}
+ ValueType value;
+
+ explicit NumericScalar(ValueType value, bool is_valid = true)
+ : internal::PrimitiveScalar(TypeTraits<T>::type_singleton(), is_valid),
+ value(value) {}
NumericScalar() : NumericScalar(0, false) {}
- protected:
- explicit NumericScalar(T value, const std::shared_ptr<DataType>& type, bool is_valid)
- : internal::PrimitiveScalar{type, is_valid}, value(value) {}
+ NumericScalar(ValueType value, const std::shared_ptr<DataType>& type,
+ bool is_valid = true)
+ : NumericScalar(value, is_valid) {
+ ARROW_CHECK_EQ(type->id(), T::type_id);
+ }
};
-template <typename Type>
+template <typename T>
struct BaseBinaryScalar : public Scalar {
+ using ValueType = std::shared_ptr<Buffer>;
+
std::shared_ptr<Buffer> value;
protected:
@@ -105,35 +143,47 @@ struct BaseBinaryScalar : public Scalar {
};
struct ARROW_EXPORT BinaryScalar : public BaseBinaryScalar<BinaryType> {
+ BinaryScalar(const std::shared_ptr<Buffer>& value,
+ const std::shared_ptr<DataType>& type, bool is_valid = true)
+ : BaseBinaryScalar(value, type, is_valid) {}
+
explicit BinaryScalar(const std::shared_ptr<Buffer>& value, bool is_valid = true)
- : BaseBinaryScalar(value, binary(), is_valid) {}
+ : BinaryScalar(value, binary(), is_valid) {}
BinaryScalar() : BinaryScalar(NULLPTR, false) {}
-
- protected:
- using BaseBinaryScalar::BaseBinaryScalar;
};
struct ARROW_EXPORT StringScalar : public BinaryScalar {
+ StringScalar(const std::shared_ptr<Buffer>& value,
+ const std::shared_ptr<DataType>& type, bool is_valid = true)
+ : BinaryScalar(value, type, is_valid) {}
+
explicit StringScalar(const std::shared_ptr<Buffer>& value, bool is_valid = true)
- : BinaryScalar(value, utf8(), is_valid) {}
+ : StringScalar(value, utf8(), is_valid) {}
+
+ explicit StringScalar(std::string s);
StringScalar() : StringScalar(NULLPTR, false) {}
};
struct ARROW_EXPORT LargeBinaryScalar : public BaseBinaryScalar<LargeBinaryType> {
+ LargeBinaryScalar(const std::shared_ptr<Buffer>& value,
+ const std::shared_ptr<DataType>& type, bool is_valid = true)
+ : BaseBinaryScalar(value, type, is_valid) {}
+
explicit LargeBinaryScalar(const std::shared_ptr<Buffer>& value, bool is_valid = true)
- : BaseBinaryScalar(value, large_binary(), is_valid) {}
+ : LargeBinaryScalar(value, large_binary(), is_valid) {}
LargeBinaryScalar() : LargeBinaryScalar(NULLPTR, false) {}
-
- protected:
- using BaseBinaryScalar::BaseBinaryScalar;
};
struct ARROW_EXPORT LargeStringScalar : public LargeBinaryScalar {
+ LargeStringScalar(const std::shared_ptr<Buffer>& value,
+ const std::shared_ptr<DataType>& type, bool is_valid = true)
+ : LargeBinaryScalar(value, type, is_valid) {}
+
explicit LargeStringScalar(const std::shared_ptr<Buffer>& value, bool is_valid = true)
- : LargeBinaryScalar(value, large_utf8(), is_valid) {}
+ : LargeStringScalar(value, large_utf8(), is_valid) {}
LargeStringScalar() : LargeStringScalar(NULLPTR, false) {}
};
@@ -155,59 +205,86 @@ class ARROW_EXPORT Date64Scalar : public NumericScalar<Date64Type> {
class ARROW_EXPORT Time32Scalar : public internal::PrimitiveScalar {
public:
- int32_t value;
+ using ValueType = int32_t;
+
Time32Scalar(int32_t value, const std::shared_ptr<DataType>& type,
bool is_valid = true);
+
+ int32_t value;
};
class ARROW_EXPORT Time64Scalar : public internal::PrimitiveScalar {
public:
- int64_t value;
+ using ValueType = int64_t;
+
Time64Scalar(int64_t value, const std::shared_ptr<DataType>& type,
bool is_valid = true);
+
+ int64_t value;
};
class ARROW_EXPORT TimestampScalar : public internal::PrimitiveScalar {
public:
- int64_t value;
+ using ValueType = int64_t;
+
TimestampScalar(int64_t value, const std::shared_ptr<DataType>& type,
bool is_valid = true);
+
+ int64_t value;
};
class ARROW_EXPORT DurationScalar : public internal::PrimitiveScalar {
public:
- int64_t value;
+ using ValueType = int64_t;
+
DurationScalar(int64_t value, const std::shared_ptr<DataType>& type,
bool is_valid = true);
+
+ int64_t value;
};
class ARROW_EXPORT MonthIntervalScalar : public internal::PrimitiveScalar {
public:
- int32_t value;
+ using ValueType = int32_t;
+
+ explicit MonthIntervalScalar(int32_t value, bool is_valid = true);
MonthIntervalScalar(int32_t value, const std::shared_ptr<DataType>& type,
bool is_valid = true);
+
+ int32_t value;
};
class ARROW_EXPORT DayTimeIntervalScalar : public internal::PrimitiveScalar {
public:
- DayTimeIntervalType::DayMilliseconds value;
+ using ValueType = DayTimeIntervalType::DayMilliseconds;
+
+ explicit DayTimeIntervalScalar(DayTimeIntervalType::DayMilliseconds value,
+ bool is_valid = true);
+
DayTimeIntervalScalar(DayTimeIntervalType::DayMilliseconds value,
const std::shared_ptr<DataType>& type, bool is_valid = true);
+
+ DayTimeIntervalType::DayMilliseconds value;
};
struct ARROW_EXPORT Decimal128Scalar : public Scalar {
- Decimal128 value;
+ using ValueType = Decimal128;
+
Decimal128Scalar(const Decimal128& value, const std::shared_ptr<DataType>& type,
bool is_valid = true);
+
+ Decimal128 value;
};
struct ARROW_EXPORT BaseListScalar : public Scalar {
- std::shared_ptr<Array> value;
+ using ValueType = std::shared_ptr<Array>;
BaseListScalar(const std::shared_ptr<Array>& value,
const std::shared_ptr<DataType>& type, bool is_valid = true);
BaseListScalar(const std::shared_ptr<Array>& value, bool is_valid);
+
+ std::shared_ptr<Array> value;
};
struct ARROW_EXPORT ListScalar : public BaseListScalar {
@@ -218,27 +295,19 @@ struct ARROW_EXPORT LargeListScalar : public BaseListScalar {
using BaseListScalar::BaseListScalar;
};
-struct ARROW_EXPORT MapScalar : public Scalar {
- std::shared_ptr<Array> keys;
- std::shared_ptr<Array> items;
-
- MapScalar(const std::shared_ptr<Array>& keys, const std::shared_ptr<Array>& values,
- const std::shared_ptr<DataType>& type, bool is_valid = true);
-
- MapScalar(const std::shared_ptr<Array>& keys, const std::shared_ptr<Array>& values,
- bool is_valid = true);
+struct ARROW_EXPORT MapScalar : public BaseListScalar {
+ using BaseListScalar::BaseListScalar;
};
-struct ARROW_EXPORT FixedSizeListScalar : public Scalar {
- std::shared_ptr<Array> value;
-
+struct ARROW_EXPORT FixedSizeListScalar : public BaseListScalar {
FixedSizeListScalar(const std::shared_ptr<Array>& value,
const std::shared_ptr<DataType>& type, bool is_valid = true);
- explicit FixedSizeListScalar(const std::shared_ptr<Array>& value, bool is_valid = true);
+ using BaseListScalar::BaseListScalar;
};
struct ARROW_EXPORT StructScalar : public Scalar {
+ using ValueType = std::vector<std::shared_ptr<Scalar>>;
std::vector<std::shared_ptr<Scalar>> value;
};
@@ -253,4 +322,58 @@ ARROW_EXPORT
Status MakeNullScalar(const std::shared_ptr<DataType>& type,
std::shared_ptr<Scalar>* null);
+namespace internal {
+
+inline Status CheckBufferLength(...) { return Status::OK(); }
+
+ARROW_EXPORT Status CheckBufferLength(const FixedSizeBinaryType* t,
+ const std::shared_ptr<Buffer>* b);
+
+}; // namespace internal
+
+template <typename ValueRef>
+struct MakeScalarImpl {
+ template <
+ typename T, typename ScalarType = typename TypeTraits<T>::ScalarType,
+ typename ValueType = typename ScalarType::ValueType,
+ typename Enable = typename std::enable_if<
+ internal::is_simple_scalar<ScalarType>::value &&
+ std::is_same<ValueType, typename std::decay<ValueRef>::type>::value>::type>
+ Status Visit(const T& t) {
+ ARROW_RETURN_NOT_OK(internal::CheckBufferLength(&t, &value_));
+ *out_ = std::make_shared<ScalarType>(ValueType(static_cast<ValueRef>(value_)), type_,
+ true);
+ return Status::OK();
+ }
+
+ Status Visit(const DataType& t) {
+ return Status::NotImplemented("constructing scalars of type ", t, " from ", value_);
+ }
+
+ const std::shared_ptr<DataType>& type_;
+ ValueRef value_;
+ std::shared_ptr<Scalar>* out_;
+};
+
+template <typename Value>
+Status MakeScalar(const std::shared_ptr<DataType>& type, Value&& value,
+ std::shared_ptr<Scalar>* out) {
+ MakeScalarImpl<Value&&> impl = {type, std::forward<Value>(value), out};
+ return VisitTypeInline(*type, &impl);
+}
+
+/// \brief type inferring scalar factory
+template <typename Value, typename Traits = CTypeTraits<typename std::decay<Value>::type>,
+ typename ScalarType = typename Traits::ScalarType,
+ typename Enable = decltype(ScalarType(std::declval<Value>(),
+ Traits::type_singleton(), true))>
+std::shared_ptr<Scalar> MakeScalar(Value value) {
+ return std::make_shared<ScalarType>(std::move(value), Traits::type_singleton(), true);
+}
+
+template <size_t N>
+std::shared_ptr<Scalar> MakeScalar(const char (&value)[N]) {
+ return std::make_shared<StringScalar>(value);
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/scalar_test.cc b/cpp/src/arrow/scalar_test.cc
index 6bd2055..baf3aad 100644
--- a/cpp/src/arrow/scalar_test.cc
+++ b/cpp/src/arrow/scalar_test.cc
@@ -15,23 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
#include <memory>
#include <string>
#include <utility>
@@ -87,6 +70,21 @@ TYPED_TEST(TestNumericScalar, Basics) {
ASSERT_FALSE(stack_val.is_valid);
}
+TYPED_TEST(TestNumericScalar, MakeScalar) {
+ using T = typename TypeParam::c_type;
+ using ScalarType = typename TypeTraits<TypeParam>::ScalarType;
+
+ std::shared_ptr<Scalar> three = MakeScalar(static_cast<T>(3));
+ ASSERT_TRUE(ScalarType(3).Equals(three));
+
+ ASSERT_OK(
+ MakeScalar(TypeTraits<TypeParam>::type_singleton(), static_cast<T>(3), &three));
+ ASSERT_TRUE(ScalarType(3).Equals(three));
+
+ ASSERT_OK(Scalar::Parse(TypeTraits<TypeParam>::type_singleton(), "3", &three));
+ ASSERT_TRUE(ScalarType(3).Equals(three));
+}
+
TEST(TestBinaryScalar, Basics) {
std::string data = "test data";
auto buf = std::make_shared<Buffer>(data);
@@ -121,6 +119,17 @@ TEST(TestBinaryScalar, Basics) {
ASSERT_FALSE(null_value2.is_valid);
}
+TEST(TestStringScalar, MakeScalar) {
+ auto three = MakeScalar("three");
+ ASSERT_TRUE(StringScalar("three").Equals(three));
+
+ ASSERT_OK(MakeScalar(utf8(), Buffer::FromString("three"), &three));
+ ASSERT_TRUE(StringScalar("three").Equals(three));
+
+ ASSERT_OK(Scalar::Parse(utf8(), "three", &three));
+ ASSERT_TRUE(StringScalar("three").Equals(three));
+}
+
TEST(TestFixedSizeBinaryScalar, Basics) {
std::string data = "test data";
auto buf = std::make_shared<Buffer>(data);
@@ -133,6 +142,23 @@ TEST(TestFixedSizeBinaryScalar, Basics) {
ASSERT_TRUE(value.type->Equals(*ex_type));
}
+TEST(TestFixedSizeBinaryScalar, MakeScalar) {
+ std::string data = "test data";
+ auto buf = std::make_shared<Buffer>(data);
+ auto type = fixed_size_binary(9);
+
+ std::shared_ptr<Scalar> s;
+ ASSERT_OK(MakeScalar(type, buf, &s));
+ ASSERT_TRUE(FixedSizeBinaryScalar(buf, type).Equals(s));
+
+ ASSERT_OK(Scalar::Parse(type, util::string_view(data), &s));
+ ASSERT_TRUE(FixedSizeBinaryScalar(buf, type).Equals(s));
+
+ // wrong length:
+ ASSERT_RAISES(Invalid, MakeScalar(type, Buffer::FromString(data.substr(3)), &s));
+ ASSERT_RAISES(Invalid, Scalar::Parse(type, util::string_view(data).substr(3), &s));
+}
+
TEST(TestDateScalars, Basics) {
int32_t i32_val = 1;
Date32Scalar date32_val(i32_val);
@@ -151,6 +177,17 @@ TEST(TestDateScalars, Basics) {
ASSERT_FALSE(date64_null.is_valid);
}
+TEST(TestDateScalars, MakeScalar) {
+ std::shared_ptr<Scalar> s;
+ ASSERT_OK(MakeScalar(date32(), int32_t(1), &s));
+ ASSERT_TRUE(Date32Scalar(1).Equals(s));
+
+ ASSERT_OK(MakeScalar(date64(), int64_t(1), &s));
+ ASSERT_TRUE(Date64Scalar(1).Equals(s));
+
+ ASSERT_RAISES(NotImplemented, Scalar::Parse(date64(), "", &s));
+}
+
TEST(TestTimeScalars, Basics) {
auto type1 = time32(TimeUnit::MILLI);
auto type2 = time32(TimeUnit::SECOND);
@@ -176,6 +213,29 @@ TEST(TestTimeScalars, Basics) {
ASSERT_TRUE(time64_null.type->Equals(*type4));
}
+TEST(TestTimeScalars, MakeScalar) {
+ auto type1 = time32(TimeUnit::MILLI);
+ auto type2 = time32(TimeUnit::SECOND);
+ auto type3 = time64(TimeUnit::MICRO);
+ auto type4 = time64(TimeUnit::NANO);
+
+ std::shared_ptr<Scalar> s;
+
+ ASSERT_OK(MakeScalar(type1, int32_t(1), &s));
+ ASSERT_TRUE(Time32Scalar(1, type1).Equals(s));
+
+ ASSERT_OK(MakeScalar(type2, int32_t(1), &s));
+ ASSERT_TRUE(Time32Scalar(1, type2).Equals(s));
+
+ ASSERT_OK(MakeScalar(type3, int64_t(1), &s));
+ ASSERT_TRUE(Time64Scalar(1, type3).Equals(s));
+
+ ASSERT_OK(MakeScalar(type4, int64_t(1), &s));
+ ASSERT_TRUE(Time64Scalar(1, type4).Equals(s));
+
+ ASSERT_RAISES(NotImplemented, Scalar::Parse(type4, "", &s));
+}
+
TEST(TestTimestampScalars, Basics) {
auto type1 = timestamp(TimeUnit::MILLI);
auto type2 = timestamp(TimeUnit::SECOND);
@@ -199,6 +259,37 @@ TEST(TestTimestampScalars, Basics) {
ASSERT_FALSE(ts_val2.Equals(ts_null));
}
+TEST(TestTimestampScalars, MakeScalar) {
+ auto type1 = timestamp(TimeUnit::MILLI);
+ auto type2 = timestamp(TimeUnit::SECOND);
+ auto type3 = timestamp(TimeUnit::MICRO);
+ auto type4 = timestamp(TimeUnit::NANO);
+
+ std::shared_ptr<Scalar> s;
+
+ util::string_view epoch_plus_1s = "1970-01-01 00:00:01";
+
+ ASSERT_OK(MakeScalar(type1, int64_t(1), &s));
+ ASSERT_TRUE(TimestampScalar(1, type1).Equals(s));
+ ASSERT_OK(Scalar::Parse(type1, epoch_plus_1s, &s));
+ ASSERT_TRUE(TimestampScalar(1000, type1).Equals(s));
+
+ ASSERT_OK(MakeScalar(type2, int64_t(1), &s));
+ ASSERT_TRUE(TimestampScalar(1, type2).Equals(s));
+ ASSERT_OK(Scalar::Parse(type2, epoch_plus_1s, &s));
+ ASSERT_TRUE(TimestampScalar(1, type2).Equals(s));
+
+ ASSERT_OK(MakeScalar(type3, int64_t(1), &s));
+ ASSERT_TRUE(TimestampScalar(1, type3).Equals(s));
+ ASSERT_OK(Scalar::Parse(type3, epoch_plus_1s, &s));
+ ASSERT_TRUE(TimestampScalar(1000 * 1000, type3).Equals(s));
+
+ ASSERT_OK(MakeScalar(type4, int64_t(1), &s));
+ ASSERT_TRUE(TimestampScalar(1, type4).Equals(s));
+ ASSERT_OK(Scalar::Parse(type4, epoch_plus_1s, &s));
+ ASSERT_TRUE(TimestampScalar(1000 * 1000 * 1000, type4).Equals(s));
+}
+
TEST(TestDurationScalars, Basics) {
auto type1 = duration(TimeUnit::MILLI);
auto type2 = duration(TimeUnit::SECOND);
@@ -227,9 +318,9 @@ TEST(TestMonthIntervalScalars, Basics) {
int32_t val1 = 1;
int32_t val2 = 2;
- MonthIntervalScalar ts_val1(val1, type);
- MonthIntervalScalar ts_val2(val2, type);
- MonthIntervalScalar ts_null(val2, type, false);
+ MonthIntervalScalar ts_val1(val1);
+ MonthIntervalScalar ts_val2(val2);
+ MonthIntervalScalar ts_null(val2, false);
ASSERT_EQ(val1, ts_val1.value);
ASSERT_EQ(val2, ts_null.value);
@@ -249,9 +340,9 @@ TEST(TestDayTimeIntervalScalars, Basics) {
DayTimeIntervalType::DayMilliseconds val1 = {1, 1};
DayTimeIntervalType::DayMilliseconds val2 = {2, 2};
- DayTimeIntervalScalar ts_val1(val1, type);
- DayTimeIntervalScalar ts_val2(val2, type);
- DayTimeIntervalScalar ts_null(val2, type, false);
+ DayTimeIntervalScalar ts_val1(val1);
+ DayTimeIntervalScalar ts_val2(val2);
+ DayTimeIntervalScalar ts_null(val2, false);
ASSERT_EQ(val1, ts_val1.value);
ASSERT_EQ(val2, ts_null.value);
diff --git a/cpp/src/arrow/stl.h b/cpp/src/arrow/stl.h
index 3238cf2..f063e91 100644
--- a/cpp/src/arrow/stl.h
+++ b/cpp/src/arrow/stl.h
@@ -42,12 +42,33 @@ namespace stl {
namespace internal {
+template <typename T, typename = void>
+struct is_optional_like : public std::false_type {};
+
+template <typename T, typename = void>
+struct is_dereferencable : public std::false_type {};
+
+template <typename T>
+struct is_dereferencable<T, arrow::internal::void_t<decltype(*std::declval<T>())>>
+ : public std::true_type {};
+
+template <typename T>
+struct is_optional_like<
+ T, typename std::enable_if<
+ std::is_constructible<bool, T>::value && is_dereferencable<T>::value &&
+ !std::is_array<typename std::remove_reference<T>::type>::value>::type>
+ : public std::true_type {};
+
template <size_t N, typename Tuple>
-using BareTupleElement = typename std::remove_const<typename std::remove_reference<
- typename std::tuple_element<N, Tuple>::type>::type>::type;
+using BareTupleElement =
+ typename std::decay<typename std::tuple_element<N, Tuple>::type>::type;
} // namespace internal
+template <typename T, typename R = void>
+using enable_if_optional_like =
+ typename std::enable_if<internal::is_optional_like<T>::value, R>::type;
+
/// Traits meta class to map standard C/C++ types to equivalent Arrow types.
template <typename T, typename Enable = void>
struct ConversionTraits {};
@@ -81,7 +102,6 @@ Status AppendListValues(CBuilderType<ValueCType>& value_builder, Range&& cell_ra
size_t j) { \
return array.Value(j); \
} \
- constexpr static bool nullable = false; \
}; \
\
template <> \
@@ -111,7 +131,6 @@ struct ConversionTraits<std::string> : public CTypeTraits<std::string> {
static std::string GetEntry(const StringArray& array, size_t j) {
return array.GetString(j);
}
- constexpr static bool nullable = false;
};
/// Append cell range elements as a single value to the list builder.
@@ -159,19 +178,15 @@ struct ConversionTraits<std::vector<ValueCType>>
}
return vec;
}
-
- constexpr static bool nullable = false;
};
template <typename Optional>
struct ConversionTraits<Optional, enable_if_optional_like<Optional>>
- : public CTypeTraits<Optional> {
- // Dependent names from base template class needs to be brought into scope.
- using typename CTypeTraits<Optional>::OptionalInnerType;
- using typename CTypeTraits<Optional>::ArrowType;
- using CTypeTraits<Optional>::type_singleton;
-
- constexpr static bool nullable = true;
+ : public CTypeTraits<typename std::decay<decltype(*std::declval<Optional>())>::type> {
+ using OptionalInnerType =
+ typename std::decay<decltype(*std::declval<Optional>())>::type;
+ using typename CTypeTraits<OptionalInnerType>::ArrowType;
+ using CTypeTraits<OptionalInnerType>::type_singleton;
static Status AppendRow(typename TypeTraits<ArrowType>::BuilderType& builder,
const Optional& cell) {
@@ -200,8 +215,8 @@ struct SchemaFromTuple {
const std::vector<std::string>& names) {
std::vector<std::shared_ptr<Field>> ret =
SchemaFromTuple<Tuple, N - 1>::MakeSchemaRecursion(names);
- std::shared_ptr<DataType> type = CTypeTraits<Element>::type_singleton();
- ret.push_back(field(names[N - 1], type, ConversionTraits<Element>::nullable));
+ auto type = ConversionTraits<Element>::type_singleton();
+ ret.push_back(field(names[N - 1], type, internal::is_optional_like<Element>::value));
return ret;
}
@@ -232,7 +247,8 @@ struct SchemaFromTuple {
std::vector<std::shared_ptr<Field>> ret =
SchemaFromTuple<Tuple, N - 1>::MakeSchemaRecursionT(names);
std::shared_ptr<DataType> type = ConversionTraits<Element>::type_singleton();
- ret.push_back(field(get<N - 1>(names), type, ConversionTraits<Element>::nullable));
+ ret.push_back(
+ field(get<N - 1>(names), type, internal::is_optional_like<Element>::value));
return ret;
}
diff --git a/cpp/src/arrow/stl_test.cc b/cpp/src/arrow/stl_test.cc
index 1e93c7e..f259662 100644
--- a/cpp/src/arrow/stl_test.cc
+++ b/cpp/src/arrow/stl_test.cc
@@ -123,8 +123,6 @@ namespace stl {
template <>
struct ConversionTraits<CustomOptionalTypeMock>
: public CTypeTraits<CustomOptionalTypeMock> {
- constexpr static bool nullable = true;
-
static Status AppendRow(typename TypeTraits<ArrowType>::BuilderType& builder,
const CustomOptionalTypeMock& cell) {
if (cell) {
@@ -137,9 +135,7 @@ struct ConversionTraits<CustomOptionalTypeMock>
template <>
struct ConversionTraits<TestInt32Type> : public CTypeTraits<TestInt32Type> {
- constexpr static bool nullable = false;
-
- // AppendRow is not needed, since it shouldn't be called.
+ // AppendRow is not needed, explicitly elide an implementation
};
template <>
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index c2deddc..5278127 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -34,7 +34,7 @@ namespace arrow {
template <typename T>
struct TypeTraits {};
-template <typename T, typename Enable = void>
+template <typename T>
struct CTypeTraits {};
template <>
@@ -171,6 +171,7 @@ struct TypeTraits<DayTimeIntervalType> {
return elements * static_cast<int64_t>(sizeof(DayTimeIntervalType::DayMilliseconds));
}
constexpr static bool is_parameter_free = true;
+ static std::shared_ptr<DataType> type_singleton() { return day_time_interval(); }
};
template <>
@@ -183,6 +184,7 @@ struct TypeTraits<MonthIntervalType> {
return elements * static_cast<int64_t>(sizeof(int32_t));
}
constexpr static bool is_parameter_free = true;
+ static std::shared_ptr<DataType> type_singleton() { return month_interval(); }
};
template <>
@@ -283,9 +285,10 @@ struct CTypeTraits<std::string> : public TypeTraits<StringType> {
};
template <>
-struct CTypeTraits<char*> : public TypeTraits<StringType> {
- using ArrowType = StringType;
-};
+struct CTypeTraits<const char*> : public CTypeTraits<std::string> {};
+
+template <size_t N>
+struct CTypeTraits<const char (&)[N]> : public CTypeTraits<std::string> {};
template <>
struct CTypeTraits<DayTimeIntervalType::DayMilliseconds>
@@ -377,39 +380,8 @@ struct make_void {
template <typename... Ts>
using void_t = typename make_void<Ts...>::type;
-template <typename T, typename = void>
-struct is_dereferencable : public std::false_type {};
-
-template <typename T>
-struct is_dereferencable<T, void_t<decltype(*std::declval<T>())>>
- : public std::true_type {};
-
-template <typename T, typename = void>
-struct is_optional_like : public std::false_type {};
-
-template <typename T>
-struct is_optional_like<T,
- typename std::enable_if<std::is_constructible<bool, T>::value &&
- is_dereferencable<T>::value>::type>
- : public std::true_type {};
-
} // namespace internal
-template <typename T, typename R = void>
-using enable_if_optional_like =
- typename std::enable_if<internal::is_optional_like<T>::value, R>::type;
-
-template <typename Optional>
-struct CTypeTraits<Optional, enable_if_optional_like<Optional>> {
- using OptionalInnerType =
- typename std::decay<decltype(*std::declval<Optional>())>::type;
- using ArrowType = typename CTypeTraits<OptionalInnerType>::ArrowType;
-
- static std::shared_ptr<::arrow::DataType> type_singleton() {
- return CTypeTraits<OptionalInnerType>::type_singleton();
- }
-};
-
//
// Useful type predicates
//