You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2021/12/03 19:25:53 UTC
[GitHub] [arrow] AlvinJ15 opened a new pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
AlvinJ15 opened a new pull request #11853:
URL: https://github.com/apache/arrow/pull/11853
ARROW-1699: [C++] forward, backward fill kernel functions
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] ursabot edited a comment on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
ursabot edited a comment on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005743192
Benchmark runs are scheduled for baseline = 67a29fdffec3c2646b29aa07b49729305aac0d38 and contender = ec38aebb36e99e54e69089cbc6a623a616575dde. ec38aebb36e99e54e69089cbc6a623a616575dde is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Finished :arrow_down:0.0% :arrow_up:0.0%] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/a9958e4a8da04ccf834dc20ef43bbb5e...1b5e3a4ab1db415ca90ec502fccedb4f/)
[Failed] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/9f3bf8138e5148209c3760429894d52b...ffe8032605144c5388649c3edf75a52a/)
[Scheduled] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/85986fd142c94225a708903d62926e83...924ad696374942ce813258bde7c2334e/)
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python. Runs only benchmarks with cloud = True
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r778107623
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
Or actually, frankly: just add those cases to ConstantArrayGenerator::Numeric, e.g.
```cpp
case Date32Type:
return Int32(size, ...)->View(date32());
```
https://github.com/apache/arrow/blob/91e3ac53e2e21736ce6291d73fc37da6fa21259d/cpp/src/arrow/testing/generator.h#L129-L157
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-989538380
@lidavidm all comments were solved
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777819999
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
same as the next comment, no support for rand.Numeric( ) for DateType, even if I use rand.ArrayOf no support for ConstantArrayGenerator::Numeric(DateTypes).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] ursabot edited a comment on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
ursabot edited a comment on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005743192
Benchmark runs are scheduled for baseline = 67a29fdffec3c2646b29aa07b49729305aac0d38 and contender = ec38aebb36e99e54e69089cbc6a623a616575dde. ec38aebb36e99e54e69089cbc6a623a616575dde is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Finished :arrow_down:0.0% :arrow_up:0.0%] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/a9958e4a8da04ccf834dc20ef43bbb5e...1b5e3a4ab1db415ca90ec502fccedb4f/)
[Failed] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/9f3bf8138e5148209c3760429894d52b...ffe8032605144c5388649c3edf75a52a/)
[Scheduled] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/85986fd142c94225a708903d62926e83...924ad696374942ce813258bde7c2334e/)
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python. Runs only benchmarks with cloud = True
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm closed pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm closed pull request #11853:
URL: https://github.com/apache/arrow/pull/11853
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005735261
Ah shoot, I forgot to ask you to update the docs. I'll file a minor PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777509436
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -489,17 +872,57 @@ void RegisterVectorReplace(FunctionRegistry* registry) {
}
add_primitive_kernel(null());
add_primitive_kernel(boolean());
- add_kernel(Type::FIXED_SIZE_BINARY, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
- add_kernel(Type::DECIMAL128, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
- add_kernel(Type::DECIMAL256, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
+ add_kernel(Type::FIXED_SIZE_BINARY, Functor<FixedSizeBinaryType>::Exec);
+ add_kernel(Type::DECIMAL128, Functor<FixedSizeBinaryType>::Exec);
+ add_kernel(Type::DECIMAL256, Functor<FixedSizeBinaryType>::Exec);
for (const auto& ty : BaseBinaryTypes()) {
- add_kernel(ty->id(), GenerateTypeAgnosticVarBinaryBase<ReplaceWithMaskFunctor>(*ty));
+ add_kernel(ty->id(), GenerateTypeAgnosticVarBinaryBase<Functor>(*ty));
}
// TODO: list types
DCHECK_OK(registry->AddFunction(std::move(func)));
// TODO(ARROW-9431): "replace_with_indices"
}
+
+const FunctionDoc replace_with_mask_doc(
+ "Replace items selected with a mask",
+ ("Given an array and a boolean mask (either scalar or of equal length),\n"
+ "along with replacement values (either scalar or array),\n"
+ "each element of the array for which the corresponding mask element is\n"
+ "true will be replaced by the next value from the replacements,\n"
+ "or with null if the mask is null.\n"
+ "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
+ {"values", "mask", "replacements"});
+
+const FunctionDoc fill_null_forward_doc(
+ "Carry non-null values forward to fill null slots",
+ ("Given an array, propagate last valid observation forward to next valid\n"
+ "or nothing if all previous values are null. "),
Review comment:
nit: remove the trailing space (here and below).
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,406 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_forward: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData* array_with_current = &*values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, *array_with_current,
Review comment:
DCHECK_OK will crash the program if an error occurs, instead of returning it to the caller.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,406 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_forward: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData* array_with_current = &*values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, *array_with_current,
Review comment:
We use it below for kernel registration because we expect that to never fail, but we do expect kernels themselves to fail normally.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
Review comment:
Ping here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,406 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_forward: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData* array_with_current = &*values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, *array_with_current,
Review comment:
Do not use DCHECK_OK for this. Use RETURN_NOT_OK.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,406 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_forward: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData* array_with_current = &*values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, *array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = &*chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_backward operation: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = 0;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData* array_with_current =
+ &(*values->chunk(/*first_chunk=*/chunks_length - 1)->data());
+ int64_t last_valid_value_offset = -1;
+ auto chunks = values->chunks();
+ for (int i = chunks_length - 1; i >= 0; --i) {
+ const auto& chunk = chunks[i];
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullBackwardArray(ctx, *chunk->data(), out, *array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = &*chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ reverse(new_chunks.begin(), new_chunks.end());
Review comment:
nit: explicitly use `std::reverse`?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
@AlvinJ15: what is the reason for this check? If there is one, please let me know - I'm just wondering why it's here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,406 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_forward: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData* array_with_current = &*values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, *array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = &*chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented("Unsupported type for fill_null_backward operation: ",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, const ArrayData& array,
+ Datum* out, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = 0;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData* array_with_current =
+ &(*values->chunk(/*first_chunk=*/chunks_length - 1)->data());
+ int64_t last_valid_value_offset = -1;
+ auto chunks = values->chunks();
+ for (int i = chunks_length - 1; i >= 0; --i) {
+ const auto& chunk = chunks[i];
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullBackwardArray(ctx, *chunk->data(), out, *array_with_current,
Review comment:
Ditto here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r778106647
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
Then please document why the check is needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] ursabot edited a comment on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
ursabot edited a comment on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005743192
Benchmark runs are scheduled for baseline = 67a29fdffec3c2646b29aa07b49729305aac0d38 and contender = ec38aebb36e99e54e69089cbc6a623a616575dde. ec38aebb36e99e54e69089cbc6a623a616575dde is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Finished :arrow_down:0.0% :arrow_up:0.0%] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/a9958e4a8da04ccf834dc20ef43bbb5e...1b5e3a4ab1db415ca90ec502fccedb4f/)
[Scheduled] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/9f3bf8138e5148209c3760429894d52b...ffe8032605144c5388649c3edf75a52a/)
[Scheduled] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/85986fd142c94225a708903d62926e83...924ad696374942ce813258bde7c2334e/)
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python. Runs only benchmarks with cloud = True
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-992498224
For the failures with chunked arrays, the kernel needs these options set:
```diff
diff --git a/cpp/src/arrow/compute/kernels/vector_replace.cc b/cpp/src/arrow/compute/kernels/vector_replace.cc
index d85965cfd..8ee4bbfe1 100644
--- a/cpp/src/arrow/compute/kernels/vector_replace.cc
+++ b/cpp/src/arrow/compute/kernels/vector_replace.cc
@@ -799,6 +799,8 @@ void RegisterVectorFunction(FunctionRegistry* registry,
kernel.mem_allocation = MemAllocation::type::PREALLOCATE;
kernel.signature = Functor<FixedType>::GetSignature(get_id.id);
kernel.exec = std::move(exec);
+ kernel.can_execute_chunkwise = false;
+ kernel.output_chunked = false;
DCHECK_OK(func->AddKernel(std::move(kernel)));
};
auto add_primitive_kernel = [&](detail::GetTypeId get_id) {
diff --git a/cpp/src/arrow/compute/kernels/vector_replace_test.cc b/cpp/src/arrow/compute/kernels/vector_replace_test.cc
index 742facf19..48a0b40f5 100644
--- a/cpp/src/arrow/compute/kernels/vector_replace_test.cc
+++ b/cpp/src/arrow/compute/kernels/vector_replace_test.cc
@@ -110,7 +110,7 @@ class TestReplaceKernel : public ::testing::Test {
const std::shared_ptr<ChunkedArray> array,
const std::shared_ptr<ChunkedArray>& expected) {
ASSERT_OK_AND_ASSIGN(auto actual, func(Datum(*array), nullptr));
- AssertChunkedEqual(*expected, *actual.chunked_array());
+ AssertChunkedEquivalent(*expected, *actual.chunked_array());
}
```
The test still fails, but I'll leave that for further debugging.
Basically, with a chunked array input, by default, the compute infrastructure will split up the inputs and feed the kernel one chunk at a time, instead of the entire chunked array. In that case, it expects the kernel to output Array, not ChunkedArray, as it will assemble the final ChunkedArray itself. Also, in that case, it expects the kernel to keep track of its own state (in KernelState in KernelContext). Setting these options tells the compute infrastructure not to do this splitting and not to expect an Array output.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r770531035
##########
File path: cpp/src/arrow/compute/api_vector.cc
##########
@@ -220,6 +220,14 @@ Result<Datum> ReplaceWithMask(const Datum& values, const Datum& mask,
return CallFunction("replace_with_mask", {values, mask, replacements}, ctx);
}
+Result<Datum> FillForwardNull(const Datum& values, ExecContext* ctx) {
Review comment:
We should also change the wrapper names if we change the function name.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -489,17 +926,57 @@ void RegisterVectorReplace(FunctionRegistry* registry) {
}
add_primitive_kernel(null());
add_primitive_kernel(boolean());
- add_kernel(Type::FIXED_SIZE_BINARY, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
- add_kernel(Type::DECIMAL128, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
- add_kernel(Type::DECIMAL256, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
+ add_kernel(Type::FIXED_SIZE_BINARY, Functor<FixedType>::Exec);
+ add_kernel(Type::DECIMAL128, Functor<FixedType>::Exec);
+ add_kernel(Type::DECIMAL256, Functor<FixedType>::Exec);
for (const auto& ty : BaseBinaryTypes()) {
- add_kernel(ty->id(), GenerateTypeAgnosticVarBinaryBase<ReplaceWithMaskFunctor>(*ty));
+ add_kernel(ty->id(), GenerateTypeAgnosticVarBinaryBase<Functor>(*ty));
}
// TODO: list types
DCHECK_OK(registry->AddFunction(std::move(func)));
// TODO(ARROW-9431): "replace_with_indices"
}
+
+const FunctionDoc replace_with_mask_doc(
+ "Replace items selected with a mask",
+ ("Given an array and a boolean mask (either scalar or of equal length),\n"
+ "along with replacement values (either scalar or array),\n"
+ "each element of the array for which the corresponding mask element is\n"
+ "true will be replaced by the next value from the replacements,\n"
+ "or with null if the mask is null.\n"
+ "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
+ {"values", "mask", "replacements"});
+
+const FunctionDoc fill_null_forward_doc(
+ "Replace null values using forward mode",
+ ("Given an array, propagate last valid observation forward to next valid\n"
+ "or nothing if all previous values are null. "),
+ {"values"});
+
+const FunctionDoc fill_null_backward_doc(
+ "Replace null values using forward mode",
Review comment:
Backward mode?
Also perhaps something like "Carry non-null values backward to fill null slots" might be clearer.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
Review comment:
I still don't think it's worth allocating a scalar over just calling `CopyData` in a loop. As long as it gets inlined it will be fine.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
Review comment:
I don't see why this is templated, it should be independent of the type.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
Review comment:
Ah, we would have to specialize for forward/backward fill to do this.
BTW @AlvinJ15 if I get something wrong or you have an explanation for why something is the way it is, please let me know!
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * direction;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reversed.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto reversed_bitmap = arrow::internal::ReverseBitmap(
+ ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap.ValueOrDie()->data(), output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+} // namespace
+
+template <template <class> class Functor,
+class FixedType = FixedSizeBinaryType>
Review comment:
Ping here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
Review comment:
Instead of updating `has_current_value` on every iteration, why not update it at the end of the function?
Why are we copying `has_current_value` to a local variable but not `current_value_offset`? I would either copy both or copy neither and expect the compiler to optimize both.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
Review comment:
Can we also add a test for NullType?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
Review comment:
Why not just set the kernel to preallocate the bitmap below? (You would have to have separate registration functions for these kernels instead of combining it with the replace_with_mask registration.)
```
kernel.null_handling = NullHandling::type::COMPUTED_PREALLOCATE;
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
Review comment:
1) Pass this by const reference and in the function, keep a `ArrayData* source_array` that can be updated
2) array_scalars isn't being properly updated below, though again I think we should just not use array_scalars
3) This would be clearer if it was named `previous_chunk` and `current_chunk` or something
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
Review comment:
Is this true? What if the chunk is all null?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
Review comment:
Why are we calling `ToResult`?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size / 8));
+
+ array_output->data()->length = values->length();
+ auto array_out = Datum(*array_output);
+ DCHECK_OK(FillForwardArray(ctx, *chunk->data(), &array_out, array_with_current,
+ &has_current_value, &has_current_offset));
+ if (has_current_value) {
+ array_with_current = *chunk->data();
+ }
+ if (array_out.length() > 0) {
+ new_chunks.push_back(array_out.make_array());
+ }
+ }
+ }
+
+ *out = Datum(*ToResult(values));
Review comment:
Why are we assigning to `out` twice here?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
Review comment:
This seems unused.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size / 8));
Review comment:
Also for future reference, there should be `ctx->AllocateBitmap`.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size / 8));
+
+ array_output->data()->length = values->length();
+ auto array_out = Datum(*array_output);
Review comment:
Instead of this, we can adjust `FillNullInDirectionImpl` to do the allocation (since it knows exactly how big the buffers need to be) and then we can specify that these kernels do not preallocate any data. (Again, we will have to separate the registration for these kernels from the registration for ReplaceWithMask.)
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
Why the check here?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillForwardNull, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[0]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillBackwardNull, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullForwardSliced) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/666);
+ int64_t len_random = 64;
+ int64_t len_null = 64;
+ int64_t slices = 15;
+ std::vector<std::shared_ptr<Array>> vector_values;
+ std::vector<std::shared_ptr<Array>> vector_filled;
+ for (int i = 0; i < slices; i++) {
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/5, /*max=*/200, /*null_count=*/0);
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ vector_values.push_back(array_random);
+ vector_values.push_back(array_null);
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ vector_filled.push_back(array_random);
+ vector_filled.push_back(array_null_filled);
+ }
+ }
+
+ if (vector_values.size() && vector_filled.size()) {
+ ASSERT_OK_AND_ASSIGN(auto value_array, Concatenate(vector_values));
+ ASSERT_OK_AND_ASSIGN(auto result_array, Concatenate(vector_filled));
+
+ this->AssertFillNullArray(FillForwardNull, value_array, result_array);
+ uint8_t slice_length = len_null + len_random;
+ for (int64_t slice = 0; slice < slices; slice++) {
+ auto sliced_array = value_array->Slice(slice * slice_length, slice_length);
+ auto sliced_array_expected =
+ result_array->Slice(slice * slice_length, slice_length);
+ this->AssertFillNullArray(FillForwardNull, sliced_array, sliced_array_expected);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardSliced) {
Review comment:
For testing slicing, I think it's OK to hardcode some cases.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
Ditto below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r765437081
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
Review comment:
This is because for the backward pass, I need to append to the builder but in reverse order, so first I add all offsets and append to the builder using a reverse iterator. And I use VisitNullBitmapInline instead of VisitArrayValuesInline because for backward I need to save values in another vector and iterate in reverse, then I think is best just save the offset as integers instead the entire values before append to the builder.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
Review comment:
The null_bitmap == nullptr is been evaluated when call this function using the array.MayHaveNulls()
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
Review comment:
I created a flag for use the current_chunk or the last_valid_value_chunk, instead of reassign the pointers.
`use_current_chunk ? current_chunk : last_valid_value_chunk,`
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
For Date Types which are part of the `NumericBasedTypes`, there's not exist a Random implementation, and it return an nullptr array_random
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
Review comment:
I just allocate the offsets not the values, because there is no method for VisitArrayValuesInline/VisitNullBitmapInline in reverse order
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
Review comment:
```
ReplaceWithMask<Type>::CopyData(*array.type, out_values, block_start_offset, *in,
current_value_offset, block.length);
bit_util::SetBitsTo(out_bitmap, block_start_offset, block.length, true);
```
```
static void CopyData(const DataType&, uint8_t* out, const int64_t out_offset,
const Scalar& in, const int64_t in_offset, const int64_t length) {
T* begin = reinterpret_cast<T*>(out + (out_offset * sizeof(T)));
T* end = begin + length;
std::fill(begin, end, UnboxScalar<Type>::Unbox(in));
}
```
This is for fill the entire block with the same value, otherwise I need to copy with a for loop using the offsets to the entire block.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * direction;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reversed.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto reversed_bitmap = arrow::internal::ReverseBitmap(
+ ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap.ValueOrDie()->data(), output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+} // namespace
+
+template <template <class> class Functor,
+class FixedType = FixedSizeBinaryType>
Review comment:
deleted.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
Review comment:
`has_current_value` is a flag for tell if the current chunk has a nonNull last value, and the offset of this value is stored in `current_value_offset` for be used with the current chunk for future chunks fills.
Removed the copy on local variables and just assign new values.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
Review comment:
@lidavidm For the backward, due the null_bitmap is in reversed order, when I append to the builder, all values are append in reverse order, so I need to reverse all values. So instead of add the values in a vector and reverse it, I'll think is best to add just the offsets values in a vector then reverse(vector of integers instead vector data), and then use the builder for append the values using the offsets.
I don't have any solution for specialize this method, because there not exist a preppend-like method for builder.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
Review comment:
In the normal case when the null_bitmap block has 0 and 1, it use the CopyData with buffers, but in this case the entire block is full 0 so I use the second CopyData which need a Scalar for fill the entire block with the same value.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * direction;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reversed.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto reversed_bitmap = arrow::internal::ReverseBitmap(
+ ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap.ValueOrDie()->data(), output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+} // namespace
+
+template <template <class> class Functor,
+class FixedType = FixedSizeBinaryType>
Review comment:
Following the ReplaceWithMask implementation
```
add_primitive_kernel(null());
add_primitive_kernel(boolean());
add_kernel(Type::FIXED_SIZE_BINARY, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
add_kernel(Type::DECIMAL128, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
add_kernel(Type::DECIMAL256, ReplaceWithMaskFunctor<FixedSizeBinaryType>::Exec);
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
Review comment:
added at bottom of `vector_replace_test.cc`
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
Review comment:
Added a for loop, instead copyData with scalar and fill
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
Review comment:
condition fixed.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
Review comment:
Code removed.
The allocation was for the ChunkedArrays, because the `Datum out` was created for every chunk.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size / 8));
Review comment:
Is the same Allocate(x/8) with AllocateBitmap(x)?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillForwardNull, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[0]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillBackwardNull, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullForwardSliced) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/666);
+ int64_t len_random = 64;
+ int64_t len_null = 64;
+ int64_t slices = 15;
+ std::vector<std::shared_ptr<Array>> vector_values;
+ std::vector<std::shared_ptr<Array>> vector_filled;
+ for (int i = 0; i < slices; i++) {
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/5, /*max=*/200, /*null_count=*/0);
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ vector_values.push_back(array_random);
+ vector_values.push_back(array_null);
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ vector_filled.push_back(array_random);
+ vector_filled.push_back(array_null_filled);
+ }
+ }
+
+ if (vector_values.size() && vector_filled.size()) {
+ ASSERT_OK_AND_ASSIGN(auto value_array, Concatenate(vector_values));
+ ASSERT_OK_AND_ASSIGN(auto result_array, Concatenate(vector_filled));
+
+ this->AssertFillNullArray(FillForwardNull, value_array, result_array);
+ uint8_t slice_length = len_null + len_random;
+ for (int64_t slice = 0; slice < slices; slice++) {
+ auto sliced_array = value_array->Slice(slice * slice_length, slice_length);
+ auto sliced_array_expected =
+ result_array->Slice(slice * slice_length, slice_length);
+ this->AssertFillNullArray(FillForwardNull, sliced_array, sliced_array_expected);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardSliced) {
Review comment:
@lidavidm I added some test cases, could you check it again? I assumed every slice is a new Array and apply the forward/backward fill
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size / 8));
+
+ array_output->data()->length = values->length();
+ auto array_out = Datum(*array_output);
Review comment:
removed this part.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
For Date Types, there's not exist a Random implementation, and it return an `nullptr` array_random
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
Review comment:
with an if-else if-else, the if and else block they would not repeat the same code?. I put this because if there exist a non-null value for fill(from the current or the previous chunk) it will fill the value or not, and both conditions are independents.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[0],
+ ctx->AllocateBitmap(chunk->length()));
Review comment:
@lidavidm I refactored this, using the code above and Copy for the result of every chunked array, there was no need to edit kernels different than replace_with_mask, and all tests passed, Can you check it again?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
Review comment:
I decided to remove the function `LastElementOffset(array, direction);`, because it was only used two times.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
+ *values->chunk(/*first_chunk=*/chunks_length - 1)->data();
+ int64_t last_valid_value_offset = -1;
+ auto chunks = values->chunks();
+ for (int i = chunks_length - 1; i >= 0; --i) {
+ const auto& chunk = chunks[i];
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
Review comment:
Fixed. I forgot to use the same implementation as `Forward`. Now both have the same implementation.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
> Just a reminder here.
For Date Types which are part of the NumericBasedTypes, there's not exist a Random implementation, and it return an nullptr array_random
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
Review comment:
Could you check it again and see if I did in the correct way
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r778387239
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -792,5 +850,763 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+class TestFillNullType : public TestReplaceKernel<NullType> {
+ std::shared_ptr<DataType> type() override { return default_type_instance<NullType>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.ArrayOf(this->type(), len_random, /*nulls=*/0);
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null, MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.ArrayOf(this->type(), len_random, /*nulls=*/0);
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null, MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled = ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[0]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullBackward, value_array, result_array);
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullForwardSliced) {
+ auto first_input_array =
+ this->array("[1, 4, null, null, 7, null, null, 8, 9, 5, null]");
+ auto expected_slice_length_2 =
+ this->array("[1, 4, null, null, 7, 7, null, 8, 9, 5, null]");
+ auto expected_slice_length_3 = this->array("[1, 4, 4, null, 7, 7, null, 8, 9, 5, 5]");
+ auto expected_slice_length_4 = this->array("[1, 4, 4, 4, 7, 7, 7, 8, 9, 5, 5]");
+ this->AssertFillNullArraySlices(FillNullForward, first_input_array,
+ expected_slice_length_2, expected_slice_length_3,
+ expected_slice_length_4);
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullForwardSliced) {
+ auto first_input_array = this->array(
+ R"(["avb", "iyy", null, null, "mnh", null, null, "ttr", "gfd", "lkj", null])");
+ auto expected_slice_length_2 = this->array(
+ R"(["avb", "iyy", null, null, "mnh", "mnh", null, "ttr", "gfd", "lkj", null])");
+ auto expected_slice_length_3 = this->array(
+ R"(["avb", "iyy", "iyy", null, "mnh", "mnh", null, "ttr", "gfd", "lkj", "lkj"])");
+ auto expected_slice_length_4 = this->array(
+ R"(["avb", "iyy", "iyy", "iyy", "mnh", "mnh", "mnh", "ttr", "gfd", "lkj", "lkj"])");
+
+ this->AssertFillNullArraySlices(FillNullForward, first_input_array,
+ expected_slice_length_2, expected_slice_length_3,
+ expected_slice_length_4);
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullForwardRandomSliced) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/666);
+ int64_t len_random = 64;
+ int64_t len_null = 64;
+ int64_t slices = 15;
+ std::vector<std::shared_ptr<Array>> vector_values;
+ std::vector<std::shared_ptr<Array>> vector_filled;
+ for (int i = 0; i < slices; i++) {
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/5, /*max=*/200, /*null_count=*/0);
+ if (array_random) {
Review comment:
Can these checks be eliminated?
That said, I'm not sure we need random tests for slicing. I think the hardcoded ones should be sufficient.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r765779187
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
Review comment:
Please add a comment noting that this is for fixed-size types only.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * direction;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reversed.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto reversed_bitmap = arrow::internal::ReverseBitmap(
+ ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap.ValueOrDie()->data(), output, direction);
Review comment:
Use ARROW_ASSIGN_OR_RAISE, not ValueOrDie.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
Review comment:
nit: this can just be one if-else if-else instead of a nested if-else
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
Review comment:
We can just use OptionalBitBlockCounter.
##########
File path: cpp/src/arrow/util/bitmap_ops.cc
##########
@@ -177,6 +240,21 @@ Result<std::shared_ptr<Buffer>> InvertBitmap(MemoryPool* pool, const uint8_t* da
return TransferBitmap<TransferMode::Invert>(pool, data, offset, length);
}
+Result<std::shared_ptr<Buffer>> ReverseBitmap(MemoryPool* pool, const uint8_t* data,
+ int64_t offset, int64_t length) {
+ ARROW_ASSIGN_OR_RAISE(auto buffer, AllocateEmptyBitmap(length, pool));
+ uint8_t* dest = buffer->mutable_data();
+
+ ReverseBlockOffsets(data, offset, length, /*start_offset=*/0, dest);
+
+ int64_t num_bytes = bit_util::BytesForBits(length);
+ int64_t bits_to_zero = num_bytes * 8 - length;
+ for (int64_t i = length; i < length + bits_to_zero; ++i) {
+ bit_util::ClearBit(dest, i);
+ }
Review comment:
AllocateEmptyBitmap already zero-initializes the buffer.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * direction;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reversed.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto reversed_bitmap = arrow::internal::ReverseBitmap(
+ ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap.ValueOrDie()->data(), output, direction);
Review comment:
```suggestion
ARROW_ASSIGN_OR_RAISE(auto reversed_bitmap, arrow::internal::ReverseBitmap(
ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length));
return FillNullExecutor<Type>::ExecFillNull(
ctx, array, reversed_bitmap->data(), output, direction);
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
Review comment:
has_fill_value = true?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +812,284 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullForwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullForwardNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullForwardDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullForwardBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST_SUITE(TestFillNullBackwardNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullBackwardDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBackwardBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullForwardNumeric, FillNullValuesForward) {
Review comment:
Also, we should test ChunkedArray inputs.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +812,284 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullForwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullForwardNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullForwardDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullForwardBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST_SUITE(TestFillNullBackwardNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullBackwardDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBackwardBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullForwardNumeric, FillNullValuesForward) {
Review comment:
Can we also add tests where the input array is sliced?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
Review comment:
Ah wait, we can't use VisitArrayValuesInline. But I don't think we need to pass through the array twice or allocate an intermediate here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +812,284 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullForwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardNumeric : public TestReplaceKernel<T> {
Review comment:
Reminder here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +812,284 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullForwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullForwardNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullForwardDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullForwardBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST_SUITE(TestFillNullBackwardNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullBackwardDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBackwardBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullForwardNumeric, FillNullValuesForward) {
Review comment:
Can we add tests with all-valid values?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
Review comment:
Reminder here. I don't think we need to allocate an extra scalar.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * direction;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reversed.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reversed.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto reversed_bitmap = arrow::internal::ReverseBitmap(
+ ctx->memory_pool(), array.buffers[0]->data(), array.offset, array.length);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap.ValueOrDie()->data(), output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+} // namespace
+
+template <template <class> class Functor,
+class FixedType = FixedSizeBinaryType>
Review comment:
Why is FixedType templated?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
Review comment:
And this should update bitmap_offset. (Or really, it should be updated at the bottom along with write_offset.)
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
Review comment:
Or really, just BitBlockCounter, we should never get here if null_bitmap == nullptr.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
Review comment:
GetParameters -> GetSignature?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
Review comment:
out_offset should be output->offset and in_offset should be array.offset.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
Review comment:
This loop condition could be simplified if we tracked bitmap_offset instead, no?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + direction * bitmap_offset;
Review comment:
Shouldn't this be `write_offset + direction * i`?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,231 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetParameters(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/0, array, /*in_offset=*/0, array.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ auto array_scalars = arrow::MakeArray(array.Copy());
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + direction * (block.length - 1);
Review comment:
We should also have a unit test that covers this case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-990036247
Small naming suggestion: maybe we could use "fill_null_forward" instead of "fill_forward_null" to align it more with the existing "fill_null"
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r764616914
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
Review comment:
This is for boolean arrays:
```
template <typename Type>
struct FillNullExecutor<Type, enable_if_boolean<Type>> {
static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
const uint8_t* reversed_bitmap, ArrayData* output,
uint8_t direction) {
FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction);
return Status::OK();
}
};
```
For null arrays the function does nothing and return the original array.
```
template <typename Type>
struct FillNullExecutor<Type, enable_if_null<Type>> {
static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
const uint8_t* reversed_bitmap, ArrayData* output,
uint8_t direction) {
*output = array;
return Status::OK();
}
};
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1002311169
It's not related to this PR. We can ignore it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r764620164
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -500,6 +731,104 @@ void RegisterVectorReplace(FunctionRegistry* registry) {
// TODO(ARROW-9431): "replace_with_indices"
}
+
+const FunctionDoc fill_forward_null_doc(
+ "Replace null values using forward mode",
+ ("Given an array, propagate last valid observation forward to next valid\n"
+ "or nothing if all previous values are null. "),
+ {"values"});
+
+void RegisterVectorFillForwardNullValues(FunctionRegistry* registry) {
Review comment:
Now I removed both `RegisterVectorFill*NullValues` and put them into ReplaceVector.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r764620164
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -500,6 +731,104 @@ void RegisterVectorReplace(FunctionRegistry* registry) {
// TODO(ARROW-9431): "replace_with_indices"
}
+
+const FunctionDoc fill_forward_null_doc(
+ "Replace null values using forward mode",
+ ("Given an array, propagate last valid observation forward to next valid\n"
+ "or nothing if all previous values are null. "),
+ {"values"});
+
+void RegisterVectorFillForwardNullValues(FunctionRegistry* registry) {
Review comment:
Now I remove both `RegisterVectorFill*NullValues` and put them into ReplaceVector.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] ursabot edited a comment on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
ursabot edited a comment on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005743192
Benchmark runs are scheduled for baseline = 67a29fdffec3c2646b29aa07b49729305aac0d38 and contender = ec38aebb36e99e54e69089cbc6a623a616575dde. ec38aebb36e99e54e69089cbc6a623a616575dde is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Finished :arrow_down:0.0% :arrow_up:0.0%] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/a9958e4a8da04ccf834dc20ef43bbb5e...1b5e3a4ab1db415ca90ec502fccedb4f/)
[Scheduled] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/9f3bf8138e5148209c3760429894d52b...ffe8032605144c5388649c3edf75a52a/)
[Scheduled] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/85986fd142c94225a708903d62926e83...924ad696374942ce813258bde7c2334e/)
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python. Runs only benchmarks with cloud = True
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777547587
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
Review comment:
You can use `.data().get()` instead of `&(*....data())` but it's not a big deal either way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r764851344
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
Review comment:
Ah whoops, right. Sorry for the confusion.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r776355791
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
Review comment:
Why the ternary? The direction is always -1, right?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
Review comment:
Note: you can just declare `class TestFillNullType : public TestReplaceKernel<NullType>` and then use `TEST_F(TestFillNullType, ...)` - it'll save you a bit of mucking about.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
Review comment:
```suggestion
return Status::NotImplemented(
"Unsupported type for fill_null_forward: ",
batch[0].ToString());
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
Just a reminder here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
Review comment:
Can we use `ArrayData*` to avoid a copy?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
Review comment:
```suggestion
return Status::NotImplemented(
"Unsupported type for fill_null_backward operation: ",
batch[0].ToString());
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
Review comment:
nit: why not make this just one if-else if-else?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
Review comment:
The direction is always 1.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
Review comment:
Use `const ArrayData&` or `ArrayData*`.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
+ *values->chunk(/*first_chunk=*/chunks_length - 1)->data();
+ int64_t last_valid_value_offset = -1;
+ auto chunks = values->chunks();
+ for (int i = chunks_length - 1; i >= 0; --i) {
+ const auto& chunk = chunks[i];
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
Review comment:
Also, why are we doing this for all types when we know binary-like types will allocate their own buffer?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
Why the check here?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
+ *values->chunk(/*first_chunk=*/chunks_length - 1)->data();
+ int64_t last_valid_value_offset = -1;
+ auto chunks = values->chunks();
+ for (int i = chunks_length - 1; i >= 0; --i) {
+ const auto& chunk = chunks[i];
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
Review comment:
Why use a builder when we already calculated a buffer size?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/false, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1, -1));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1 && std::get<2>(*it) == -1) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = current_chunk.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto null_bitmap,
+ arrow::internal::CopyBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, null_bitmap->data(), output,
+ direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+
+ ArrayVector new_chunks;
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+
+ for (const auto& chunk : values->chunks()) {
+ if (is_fixed_width(out->type()->id())) {
+ auto* output = out->mutable_array();
+ auto bit_width = checked_cast<const FixedWidthType&>(*output->type).bit_width();
+ auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length());
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
+ }
+ DCHECK_OK(FillNullForwardArray(ctx, *chunk->data(), out, array_with_current,
+ &last_valid_value_offset));
+ if (chunk->null_count() != chunk->length()) {
+ array_with_current = *chunk->data();
+ }
+ new_chunks.push_back(MakeArray(out->make_array()->data()->Copy()));
+ }
+ }
+
+ auto output = std::make_shared<ChunkedArray>(std::move(new_chunks), values->type());
+ *out = Datum(output);
+ return Status::OK();
+ }
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make({InputType::Array(get_id.id)}, OutputType(FirstType));
+ }
+};
+
+template <typename Type>
+struct FillNullBackwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullBackwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullBackwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullBackwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+ int8_t direction = -1;
+
+ if (array.MayHaveNulls()) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto reversed_bitmap,
+ arrow::internal::ReverseBitmap(ctx->memory_pool(), array.buffers[0]->data(),
+ array.offset, array.length));
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reversed_bitmap->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = direction != 1 ? 0 : array.length - 1;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullBackwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ auto chunks_length = values->chunks().size();
+ ArrayData array_with_current =
+ *values->chunk(/*first_chunk=*/chunks_length - 1)->data();
+ int64_t last_valid_value_offset = -1;
+ auto chunks = values->chunks();
+ for (int i = chunks_length - 1; i >= 0; --i) {
+ const auto& chunk = chunks[i];
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
Review comment:
Please note, you can directly create an ArrayData. You don't have to use a builder.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] ursabot edited a comment on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
ursabot edited a comment on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005743192
Benchmark runs are scheduled for baseline = 67a29fdffec3c2646b29aa07b49729305aac0d38 and contender = ec38aebb36e99e54e69089cbc6a623a616575dde. ec38aebb36e99e54e69089cbc6a623a616575dde is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Finished :arrow_down:0.0% :arrow_up:0.0%] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/a9958e4a8da04ccf834dc20ef43bbb5e...1b5e3a4ab1db415ca90ec502fccedb4f/)
[Failed] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/9f3bf8138e5148209c3760429894d52b...ffe8032605144c5388649c3edf75a52a/)
[Finished :arrow_down:0.26% :arrow_up:0.09%] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/85986fd142c94225a708903d62926e83...924ad696374942ce813258bde7c2334e/)
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python. Runs only benchmarks with cloud = True
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777545990
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
Do you mind using `ArrayOf` instead then? It should handle all types: https://github.com/apache/arrow/blob/c6143a2396058dcc31506050238dc0f932aae9ba/cpp/src/arrow/testing/random.h#L380
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-992499323
Also note that I would expect this:
```diff
@@ -1255,7 +1255,7 @@ TYPED_TEST(TestFillNullNumeric, FillForwardChunkedArray) {
this->AssertFillNullChunkedArray(
FillForwardNull,
this->chunked_array({"[1,2,3,null,null,4]", "[5, null, null]", "[null, 7, null]"}),
- this->chunked_array({"[1,2,3,3,3,4]", "[5, 5, 5]", "[null, 7, 7]"}));
+ this->chunked_array({"[1,2,3,3,3,4]", "[5, 5, 5]", "[5, 7, 7]"}));
}
} // namespace compute
} // namespace arrow
```
i.e. the ChunkedArray should behave as one large array, not an array of independent arrays.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777552367
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,460 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values,
+ /*out_offset=*/output->offset, array, /*in_offset=*/0,
+ array.length);
+
+ *has_current_value = false;
+ auto array_scalars = arrow::MakeArray(current_value_array.Copy());
+ bool has_fill_value = *current_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ array.length);
+
+ while (bitmap_offset < array.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *current_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ current_value_array = array;
+ *has_current_value = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ if (block.popcount) {
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ current_value_array, *current_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *current_value_offset = write_value_offset;
+ current_value_array = array;
+ *has_current_value = true;
+ }
+ }
+ } else if (has_fill_value) {
+ uint64_t write_value_offset =
+ block_start_offset + (direction == 1 ? 0 : (-block.length + 1));
+ auto in = *(array_scalars->GetScalar(*current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset, *in,
+ *current_value_offset, block.length);
+ bit_util::SetBitsTo(out_bitmap, write_value_offset, block.length, true);
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ current_value_array, has_current_value,
+ current_value_offset);
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction == 1 ? 0 : array.length - 1;
+ auto current_value_offset = write_offset + direction * (array.length - 1);
+ return current_value_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = current_value_array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = current_value_array.GetValues<offset_type>(1);
+
+ *has_current_value = false;
+ bool has_fill_value = *current_value_offset != -1;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *current_value_offset = array_value_index;
+ *has_current_value = true;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ if (*has_current_value) {
+ const offset_type offset0 = offsets[*current_value_offset];
+ const offset_type offset1 = offsets[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*current_value_offset];
+ const offset_type offset1 = offsets_prev[*current_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData current_value_array,
+ bool* has_current_value, int64_t* current_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+
+ static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ return -1;
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ return FillForwardArray(ctx, array_input, out, array_input, &has_current_value,
+ &has_current_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData current_value_array, bool* has_current_value,
+ int64_t* current_value_offset) {
+ ArrayData* output = out->array().get();
+ if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, current_value_array,
+ has_current_value, current_value_offset);
+ } else {
+ *current_value_offset = FillNullExecutor<Type>::LastElementOffset(array, direction);
+ if (array.length > 0) {
+ *has_current_value = true;
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(*ToResult(values));
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ bool has_current_value = false;
+ int64_t has_current_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, AllocateBuffer(buffer_size, ctx->memory_pool()));
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size / 8));
Review comment:
Yes, it calls `BytesForBits` for you and also zero-initializes the buffer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777548551
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,414 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id) {
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
+
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
+
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+ bool use_current_chunk = false;
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ use_current_chunk = true;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ if (block.popcount) {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ use_current_chunk = true;
+ *last_valid_value_offset = write_value_offset;
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(
+ *current_chunk.type, out_values, write_value_offset,
+ use_current_chunk ? current_chunk : last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& current_chunk,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, const ArrayData& last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(current_chunk.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(current_chunk.length));
+ RETURN_NOT_OK(builder.ReserveData(current_chunk.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : current_chunk.length - 1;
+ const uint8_t* data = current_chunk.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = current_chunk.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ /*tuple for store: <use current_chunk(true) or last_valid_chunk(false),
+ * start offset of the current value, end offset for the current value>*/
+ std::vector<std::tuple<bool, int64_t, int64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, output->offset, current_chunk.length,
+ current_chunk.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
Review comment:
Ah sorry, I see the comments now.
For here: instead of
```
if (A || B) {
if (!B) // A
else // B
} else {
// C
}
```
why not
```
if (A) // A
else if (B) // B
else // C
```
?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r778511482
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -792,5 +850,763 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+class TestFillNullType : public TestReplaceKernel<NullType> {
+ std::shared_ptr<DataType> type() override { return default_type_instance<NullType>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.ArrayOf(this->type(), len_random, /*nulls=*/0);
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null, MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.ArrayOf(this->type(), len_random, /*nulls=*/0);
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null, MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled = ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[0]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullBackward, value_array, result_array);
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullForwardSliced) {
+ auto first_input_array =
+ this->array("[1, 4, null, null, 7, null, null, 8, 9, 5, null]");
+ auto expected_slice_length_2 =
+ this->array("[1, 4, null, null, 7, 7, null, 8, 9, 5, null]");
+ auto expected_slice_length_3 = this->array("[1, 4, 4, null, 7, 7, null, 8, 9, 5, 5]");
+ auto expected_slice_length_4 = this->array("[1, 4, 4, 4, 7, 7, 7, 8, 9, 5, 5]");
+ this->AssertFillNullArraySlices(FillNullForward, first_input_array,
+ expected_slice_length_2, expected_slice_length_3,
+ expected_slice_length_4);
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullForwardSliced) {
+ auto first_input_array = this->array(
+ R"(["avb", "iyy", null, null, "mnh", null, null, "ttr", "gfd", "lkj", null])");
+ auto expected_slice_length_2 = this->array(
+ R"(["avb", "iyy", null, null, "mnh", "mnh", null, "ttr", "gfd", "lkj", null])");
+ auto expected_slice_length_3 = this->array(
+ R"(["avb", "iyy", "iyy", null, "mnh", "mnh", null, "ttr", "gfd", "lkj", "lkj"])");
+ auto expected_slice_length_4 = this->array(
+ R"(["avb", "iyy", "iyy", "iyy", "mnh", "mnh", "mnh", "ttr", "gfd", "lkj", "lkj"])");
+
+ this->AssertFillNullArraySlices(FillNullForward, first_input_array,
+ expected_slice_length_2, expected_slice_length_3,
+ expected_slice_length_4);
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullForwardRandomSliced) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/666);
+ int64_t len_random = 64;
+ int64_t len_null = 64;
+ int64_t slices = 15;
+ std::vector<std::shared_ptr<Array>> vector_values;
+ std::vector<std::shared_ptr<Array>> vector_filled;
+ for (int i = 0; i < slices; i++) {
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/5, /*max=*/200, /*null_count=*/0);
+ if (array_random) {
Review comment:
Ok I deleted this random tests for slicing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r763225230
##########
File path: cpp/src/arrow/compute/registry.cc
##########
@@ -175,6 +175,8 @@ static std::unique_ptr<FunctionRegistry> CreateBuiltInRegistry() {
RegisterVectorHash(registry.get());
RegisterVectorNested(registry.get());
RegisterVectorReplace(registry.get());
+ RegisterVectorFillForwardNullValues(registry.get());
+ RegisterVectorFillBackwardNullValues(registry.get());
Review comment:
nit: just call these from RegisterVectorReplace instead of adding them here and below
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
Review comment:
Null arrays can never contain anything but null, so you can just no-op here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -326,6 +326,12 @@ struct ReplaceWithMask<Type, enable_if_null<Type>> {
*output = array;
return Status::OK();
}
+ static void CopyData(const DataType&, uint8_t* out, const int64_t out_offset,
+ const ArrayData& in, const int64_t in_offset,
+ const int64_t length) {}
+
+ static void CopyData(const DataType&, uint8_t* out, const int64_t out_offset,
+ const Scalar& in, const int64_t in_offset, const int64_t length) {}
Review comment:
This should be unnecessary if the NullType case is just a no-op.
##########
File path: cpp/src/arrow/util/bitmap_ops.cc
##########
@@ -84,15 +84,26 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
return count;
}
-enum class TransferMode : bool { Copy, Invert };
+enum class TransferMode : char { Copy, Invert, Revert };
+
+uint8_t revert_uint8(uint8_t num) {
Review comment:
Ditto comment about function naming here.
##########
File path: cpp/src/arrow/util/bitmap_ops.cc
##########
@@ -84,15 +84,26 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
return count;
}
-enum class TransferMode : bool { Copy, Invert };
+enum class TransferMode : char { Copy, Invert, Revert };
+
+uint8_t revert_uint8(uint8_t num) {
+ num = ((num & 0xf0) >> 4) | ((num & 0x0f) << 4);
+ num = ((num & 0xcc) >> 2) | ((num & 0x33) << 2);
+ num = ((num & 0xaa) >> 1) | ((num & 0x55) << 1);
+ return num;
+}
+
+uint8_t get_reverted_block(uint8_t block_left, uint8_t block_right, uint8_t length) {
+ return revert_uint8(block_left >> (length) | block_right << (8 - length));
+}
template <TransferMode mode>
void TransferBitmap(const uint8_t* data, int64_t offset, int64_t length,
Review comment:
Given that the bit-reverse mode has basically 0 in common with the other modes, maybe we can just keep it in an entirely separate function, i.e. use RevertBlockOffsets directly?
##########
File path: cpp/src/arrow/util/bitmap_ops.cc
##########
@@ -84,15 +84,26 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
return count;
}
-enum class TransferMode : bool { Copy, Invert };
+enum class TransferMode : char { Copy, Invert, Revert };
+
+uint8_t revert_uint8(uint8_t num) {
Review comment:
(This extends to e.g. variable names, parameter names, etc. above.)
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
Review comment:
We can use the ArrayData overload of CopyData to avoid allocating a scalar in the first place.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reverted_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reverted.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reverted.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reverted.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reverted.begin(); it != offsets_reverted.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reverted.rbegin(); it != offsets_reverted.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNull {
+ static Status ExecForwardFillNull(KernelContext* ctx, const ArrayData& array,
+ ArrayData* output) {
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status ExecBackwardFillNull(KernelContext* ctx, const ArrayData& array,
+ ArrayData* output) {
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto memory_pool = ctx->memory_pool();
+ auto reverted_bitmap = arrow::internal::CopyBitmap(
+ memory_pool, array.buffers[0]->data(), array.offset, array.length);
+ arrow::internal::RevertBitmap(array.buffers[0]->data(), array.offset, array.length,
+ reverted_bitmap.ValueOrDie()->mutable_data(),
+ output->offset);
Review comment:
Why not use the overload of RevertBitmap that returns a buffer?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reverted_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reverted.push_back(std::make_pair(offset0, offset1 - offset0));
+ current_value_offset = array_value_index;
+ has_fill_value = true;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value) {
+ const offset_type offset0 = offsets[current_value_offset];
+ const offset_type offset1 = offsets[current_value_offset + 1];
+ offsets_reverted.push_back(std::make_pair(offset0, offset1 - offset0));
+ } else {
+ offsets_reverted.push_back(std::make_pair(-1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reverted.begin(); it != offsets_reverted.end(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ } else {
+ for (auto it = offsets_reverted.rbegin(); it != offsets_reverted.rend(); ++it) {
+ if (it->first == -1U && it->second == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(builder.Append(data + it->first, it->second));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNull {
+ static Status ExecForwardFillNull(KernelContext* ctx, const ArrayData& array,
+ ArrayData* output) {
+ if (array.MayHaveNulls()) {
+ int8_t direction = 1;
+ return FillNullExecutor<Type>::ExecFillNull(ctx, array, array.buffers[0]->data(),
+ output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status ExecBackwardFillNull(KernelContext* ctx, const ArrayData& array,
+ ArrayData* output) {
+ if (array.MayHaveNulls()) {
+ int8_t direction = -1;
+ auto memory_pool = ctx->memory_pool();
+ auto reverted_bitmap = arrow::internal::CopyBitmap(
+ memory_pool, array.buffers[0]->data(), array.offset, array.length);
+ arrow::internal::RevertBitmap(array.buffers[0]->data(), array.offset, array.length,
+ reverted_bitmap.ValueOrDie()->mutable_data(),
+ output->offset);
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, reverted_bitmap.ValueOrDie()->data(), output, direction);
+ } else {
+ *output = array;
+ }
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ const ArrayData& array = *batch[0].array();
+ ArrayData* output = out->array().get();
+ output->length = array.length;
+
+ return FillNull<Type>::ExecForwardFillNull(ctx, array, output);
Review comment:
Since everything is templated anyways, why not just inline FillNull into FillForwardFunctor/FillBackwardFunctor?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -500,6 +731,104 @@ void RegisterVectorReplace(FunctionRegistry* registry) {
// TODO(ARROW-9431): "replace_with_indices"
}
+
+const FunctionDoc fill_forward_null_doc(
+ "Replace null values using forward mode",
+ ("Given an array, propagate last valid observation forward to next valid\n"
+ "or nothing if all previous values are null. "),
+ {"values"});
+
+void RegisterVectorFillForwardNullValues(FunctionRegistry* registry) {
Review comment:
We could consolidate these two registration functions:
```cpp
{
auto func = std::make_shared(...);
RegisterVectorFill<FillForwardFunctor>(func.get());
registry->AddFunction(std::move(func));
}
{
auto func = std::make_shared(...);
RegisterVectorFill<FillBackwardFunctor>(func.get());
registry->AddFunction(std::move(func));
}
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -410,6 +416,12 @@ struct ReplaceWithMask<Type, enable_if_base_binary<Type>> {
output->type = array.type;
return Status::OK();
}
+ static void CopyData(const DataType&, uint8_t* out, const int64_t out_offset,
+ const ArrayData& in, const int64_t in_offset,
+ const int64_t length) {}
+
+ static void CopyData(const DataType&, uint8_t* out, const int64_t out_offset,
+ const Scalar& in, const int64_t in_offset, const int64_t length) {}
Review comment:
Why are these needed? It seems this template shouldn't get instantiated below.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t current_value_offset = 0;
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ bool has_fill_value = false;
+ const uint8_t* data = array.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+
+ std::vector<std::pair<uint64_t, uint64_t>> offsets_reverted;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
Review comment:
Instead of passing through the array twice/allocating an intermediate, why not append to the builder directly? We can also use VisitArrayValuesInline to get access to string_views instead of having to manipulate offsets manually.
##########
File path: cpp/src/arrow/util/bitmap_ops.cc
##########
@@ -84,15 +84,26 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
return count;
}
-enum class TransferMode : bool { Copy, Invert };
+enum class TransferMode : char { Copy, Invert, Revert };
+
+uint8_t revert_uint8(uint8_t num) {
+ num = ((num & 0xf0) >> 4) | ((num & 0x0f) << 4);
+ num = ((num & 0xcc) >> 2) | ((num & 0x33) << 2);
+ num = ((num & 0xaa) >> 1) | ((num & 0x55) << 1);
+ return num;
+}
+
+uint8_t get_reverted_block(uint8_t block_left, uint8_t block_right, uint8_t length) {
Review comment:
Could a comment be added here to describe what this does/is it possible to name this in a way that is more clear about its function?
##########
File path: cpp/src/arrow/util/bitmap_ops.h
##########
@@ -81,6 +81,40 @@ ARROW_EXPORT
Result<std::shared_ptr<Buffer>> InvertBitmap(MemoryPool* pool, const uint8_t* bitmap,
int64_t offset, int64_t length);
+/// Revert a bit range of an existing bitmap into an existing bitmap
+///
+/// \param[in] bitmap source data
+/// \param[in] offset bit offset into the source data
+/// \param[in] length number of bits to revert
+/// \param[in] dest_offset bit offset into the destination
+/// \param[out] dest the destination buffer, must have at least space for
+/// (offset + length) bits
+ARROW_EXPORT
+void RevertBitmap(const uint8_t* bitmap, int64_t offset, int64_t length, uint8_t* dest,
+ int64_t dest_offset);
+
+/// Revert a bit range of an existing bitmap
+///
+/// \param[in] pool memory pool to allocate memory from
+/// \param[in] bitmap source data
+/// \param[in] offset bit offset into the source data
+/// \param[in] length number of bits to revert
+///
+/// \return Status message
+ARROW_EXPORT
+Result<std::shared_ptr<Buffer>> RevertBitmap(MemoryPool* pool, const uint8_t* bitmap,
+ int64_t offset, int64_t length);
+
+/// Revert a bit range of an existing bitmap into an existing bitmap
+///
+/// \param[in] bitmap source data
+/// \param[in] offset bit offset into the source data
+/// \param[in] length number of bits to revert
+/// \param[in] dest_offset bit offset into the destination
+/// \param[out] dest the destination buffer, must have at least space for
+/// (offset + length) bits
+void RevertBlockOffsets(const uint8_t* data, int64_t offset, int64_t length,
Review comment:
Does this need to be exposed publically? It seems redundant with ReverseBitmap.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
+
+ auto array_scalars = arrow::MakeArray(array.Copy());
+
+ int64_t write_offset = incrementer == 1 ? 0 : array.length - 1;
+ int64_t bitmap_offset = 0;
+ int64_t current_value_offset = 0;
+ bool has_fill_value = false;
+ arrow::internal::OptionalBinaryBitBlockCounter counter(
+ null_bitmap, array.offset, null_bitmap, array.offset, array.length);
+
+ while (write_offset < array.length && write_offset >= 0) {
+ BitBlockCount block = counter.NextAndBlock();
+ if (block.AllSet()) {
+ current_value_offset = write_offset + incrementer * (block.length - 1);
+ } else {
+ if (block.popcount) {
+ for (int64_t i = 0; i != block.length; i++) {
+ uint64_t write_value_offset = write_offset + incrementer * bitmap_offset;
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_value_offset,
+ array, current_value_offset, /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ current_value_offset = write_value_offset;
+ }
+ bitmap_offset += 1;
+ }
+ } else {
+ auto in = *(array_scalars->GetScalar(current_value_offset));
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, write_offset, *in,
+ current_value_offset, block.length);
+ bitmap_offset += block.length;
+ }
+ write_offset += block.length * incrementer;
+ }
+ }
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ uint8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ fillNullInDirectionImpl<Type>(array, reverted_bitmap, output, direction);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reverted_bitmap, ArrayData* output,
+ int8_t direction) {
+ auto array_scalars = arrow::MakeArray(array.Copy());
Review comment:
This is unused.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(array.buffers[0]->data(), array.offset, array.length,
+ out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*array.type, out_values, 0, array, 0, array.length);
Review comment:
When we have constants as parameters, try to prefix them with a comment indicating the parameter name: `/*out_offset=*/0`, `/*in_offset=*/0`
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t incrementer) {
Review comment:
nit: incrementer -> direction?
##########
File path: cpp/src/arrow/util/bitmap_ops.cc
##########
@@ -84,15 +84,26 @@ int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) {
return count;
}
-enum class TransferMode : bool { Copy, Invert };
+enum class TransferMode : char { Copy, Invert, Revert };
+
+uint8_t revert_uint8(uint8_t num) {
Review comment:
Also, in general, instead of "revert", can we call it "reverse" or possibly "flip"?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +812,284 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullForwardNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullForwardBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBackwardNumeric : public TestReplaceKernel<T> {
Review comment:
Why have separate classes for forward/backward?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -444,6 +456,225 @@ struct ReplaceWithMaskFunctor {
}
};
+template <typename Type>
+void fillNullInDirectionImpl(const ArrayData& array, const uint8_t* null_bitmap,
Review comment:
Note: in general, function/method/class names should use UpperCamelCase naming according to our [style guide](https://arrow.apache.org/docs/developers/cpp/development.html#code-style-linting-and-ci).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] github-actions[bot] commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
github-actions[bot] commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-985771524
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r771393052
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -25,6 +25,8 @@
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/make_unique.h"
+#include <arrow/array/concatenate.h>
+#include <arrow/testing/generator.h>
Review comment:
nit: use `#include ""` for consistency/use `""` when including Arrow headers inside the codebase
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
Review comment:
Can we pass this by const reference?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
Review comment:
We should declare something like this above:
```cpp
const ArrayData* source_chunk = &last_valid_value_chunk;
```
then here we can just assign to the pointer.
All together, something like this will let you avoid copying the ArrayData around (which is still somewhat expensive since it contains shared_ptrs):
```diff
diff --git a/cpp/src/arrow/compute/kernels/vector_replace.cc b/cpp/src/arrow/compute/kernels/vector_replace.cc
index 6651e4859..92846fc25 100644
--- a/cpp/src/arrow/compute/kernels/vector_replace.cc
+++ b/cpp/src/arrow/compute/kernels/vector_replace.cc
@@ -452,9 +452,9 @@ struct ReplaceWithMaskFunctor {
// This is for fixed-size types only
template <typename Type>
-void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
ArrayData* output, int8_t direction,
- ArrayData last_valid_value_chunk,
+ const ArrayData& last_valid_value_chunk,
int64_t* last_valid_value_offset) {
uint8_t* out_bitmap = output->buffers[0]->mutable_data();
uint8_t* out_values = output->buffers[1]->mutable_data();
@@ -464,6 +464,7 @@ void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_
/*out_offset=*/output->offset, current_chunk,
/*in_offset=*/0, current_chunk.length);
+ const ArrayData* source_chunk = &last_valid_value_chunk;
bool has_fill_value = *last_valid_value_offset != -1;
int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
int64_t bitmap_offset = 0;
@@ -477,7 +478,7 @@ void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_
*last_valid_value_offset =
write_offset + direction * (block.length - 1 + bitmap_offset);
has_fill_value = true;
- last_valid_value_chunk = current_chunk;
+ source_chunk = ¤t_chunk;
} else {
uint64_t block_start_offset = write_offset + direction * bitmap_offset;
uint64_t write_value_offset = block_start_offset;
@@ -485,8 +486,8 @@ void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_
auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
if (!current_bit) {
if (has_fill_value) {
- ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
- write_value_offset, last_valid_value_chunk,
+ ReplaceWithMask<Type>::CopyData(*source_chunk->type, out_values,
+ write_value_offset, *source_chunk,
*last_valid_value_offset,
/*length=*/1);
bit_util::SetBitTo(out_bitmap, write_value_offset, true);
@@ -494,7 +495,7 @@ void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_
} else {
has_fill_value = true;
*last_valid_value_offset = write_value_offset;
- last_valid_value_chunk = current_chunk;
+ source_chunk = ¤t_chunk;
}
}
}
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
Review comment:
nit: don't leave behind commented code
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
Review comment:
I'm not sure this is worth factoring out into a separate function instead of just inlining it here and below.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[0],
+ ctx->AllocateBitmap(chunk->length()));
Review comment:
Instead of doing this, can we push the allocation into the kernel implementation itself?
It would have to be something like
```cpp
auto* output = out->mutable_array();
auto bit_width = checked_cast<const FixedWidthType&>(*output.type).bit_width();
auto data_bytes = bit_util::BytesForBits(bit_width * chunk->length);
ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(chunk->length));
ARROW_ASSIGN_OR_RAISE(output->buffers[1], ctx->Allocate(data_bytes));
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillForwardNull, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillForwardNull, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillForwardNull,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillForwardNull, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillForwardNull,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillBackwardNull, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillBackwardNull, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillBackwardNull,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
Review comment:
Ping here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
Review comment:
Note that if block.popcount is 0, it may still be worth duplicating this loop and removing the bitmap check and `has_fill_value` check. I was just saying that it is not worth allocating a Scalar for that case.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
Review comment:
nit: why the blank line here?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
Review comment:
If we're going to use negative numbers, then just make the offsets `int64_t`. Arrow generally uses signed offsets/lengths anyways.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
Review comment:
Can we leave a comment about what the tuple members mean?
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
Review comment:
```suggestion
std::make_tuple(/*current_chunk=*/true, offset0, offset1 - offset0));
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
Review comment:
```suggestion
void FillNullInDirectionImpl(const ArrayData& current_chunk, const uint8_t* null_bitmap,
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
Review comment:
again, please pass as `const ArrayData&`
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
Review comment:
```suggestion
const ArrayData& last_valid_value_chunk,
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
Review comment:
```suggestion
offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/false, -1U, -1U));
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[0],
+ ctx->AllocateBitmap(chunk->length()));
Review comment:
The reason why I prefer not to do this here is because in the case of strings, it is redundant with what the kernel itself does.
As mentioned previously, the kernel registration below will have to be adjusted:
```
kernel.mem_allocation = MemAllocation::type::COMPUTED_NO_PREALLOCATE;
```
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -798,5 +828,688 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
Review comment:
Ping here.
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[0],
+ ctx->AllocateBitmap(chunk->length()));
Review comment:
Alternatively, you can try a check like the following right here:
```cpp
if (is_fixed_width(out->type()->id())) {
// ...snippet from above
}
// otherwise, the kernel implementation itself will allocate
```
But we shouldn't need a builder just to allocate the output.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] lidavidm commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
lidavidm commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777552716
##########
File path: cpp/src/arrow/compute/kernels/vector_replace.cc
##########
@@ -442,23 +442,409 @@ struct ReplaceWithMaskFunctor {
}
return ReplaceWithMask<Type>::ExecArrayMask(ctx, array, mask, replacements, output);
}
+
+ static std::shared_ptr<KernelSignature> GetSignature(detail::GetTypeId get_id){
+ return KernelSignature::Make(
+ {InputType::Array(get_id.id), InputType(boolean()), InputType(get_id.id)},
+ OutputType(FirstType));
+ }
};
-} // namespace
+// This is for fixed-size types only
+template <typename Type>
+void FillNullInDirectionImpl(const ArrayData current_chunk, const uint8_t* null_bitmap,
+ ArrayData* output, int8_t direction,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ uint8_t* out_bitmap = output->buffers[0]->mutable_data();
+ uint8_t* out_values = output->buffers[1]->mutable_data();
+ arrow::internal::CopyBitmap(current_chunk.buffers[0]->data(), current_chunk.offset,
+ current_chunk.length, out_bitmap, output->offset);
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ /*out_offset=*/output->offset, current_chunk,
+ /*in_offset=*/0, current_chunk.length);
-const FunctionDoc replace_with_mask_doc(
- "Replace items selected with a mask",
- ("Given an array and a boolean mask (either scalar or of equal length),\n"
- "along with replacement values (either scalar or array),\n"
- "each element of the array for which the corresponding mask element is\n"
- "true will be replaced by the next value from the replacements,\n"
- "or with null if the mask is null.\n"
- "Hence, for replacement arrays, len(replacements) == sum(mask == true)."),
- {"values", "mask", "replacements"});
+ bool has_fill_value = *last_valid_value_offset != -1;
+ int64_t write_offset = direction == 1 ? 0 : current_chunk.length - 1;
+ int64_t bitmap_offset = 0;
-void RegisterVectorReplace(FunctionRegistry* registry) {
- auto func = std::make_shared<VectorFunction>("replace_with_mask", Arity::Ternary(),
- &replace_with_mask_doc);
+ arrow::internal::OptionalBitBlockCounter counter(null_bitmap, output->offset,
+ current_chunk.length);
+
+ while (bitmap_offset < current_chunk.length) {
+ BitBlockCount block = counter.NextBlock();
+ if (block.AllSet()) {
+ *last_valid_value_offset =
+ write_offset + direction * (block.length - 1 + bitmap_offset);
+ has_fill_value = true;
+ last_valid_value_chunk = current_chunk;
+ } else {
+ uint64_t block_start_offset = write_offset + direction * bitmap_offset;
+ uint64_t write_value_offset = block_start_offset;
+ for (int64_t i = 0; i < block.length; i++, write_value_offset += direction) {
+ auto current_bit = bit_util::GetBit(null_bitmap, bitmap_offset + i);
+ if (!current_bit) {
+ if (has_fill_value) {
+ ReplaceWithMask<Type>::CopyData(*current_chunk.type, out_values,
+ write_value_offset, last_valid_value_chunk,
+ *last_valid_value_offset,
+ /*length=*/1);
+ bit_util::SetBitTo(out_bitmap, write_value_offset, true);
+ }
+ } else {
+ has_fill_value = true;
+ *last_valid_value_offset = write_value_offset;
+ last_valid_value_chunk = current_chunk;
+ }
+ }
+ }
+ bitmap_offset += block.length;
+ }
+ output->null_count = -1;
+ output->GetNullCount();
+}
+
+static int64_t LastElementOffset(const ArrayData& array, int8_t direction) {
+ int64_t write_offset = direction != 1 ? 0 : array.length - 1;
+ return write_offset;
+}
+
+template <typename Type, typename Enable = void>
+struct FillNullExecutor {};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_boolean<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<
+ Type, enable_if_t<is_number_type<Type>::value ||
+ std::is_same<Type, MonthDayNanoIntervalType>::value>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_fixed_size_binary<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ FillNullInDirectionImpl<Type>(array, reversed_bitmap, output, direction,
+ last_valid_value_chunk, last_valid_value_offset);
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_base_binary<Type>> {
+ using offset_type = typename Type::offset_type;
+ using BuilderType = typename TypeTraits<Type>::BuilderType;
+
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ BuilderType builder(array.type, ctx->memory_pool());
+ RETURN_NOT_OK(builder.Reserve(array.length));
+ RETURN_NOT_OK(builder.ReserveData(array.buffers[2]->size()));
+ int64_t array_value_index = direction == 1 ? 0 : array.length - 1;
+ const uint8_t* data = array.buffers[2]->data();
+ const uint8_t* data_prev = last_valid_value_chunk.buffers[2]->data();
+ const offset_type* offsets = array.GetValues<offset_type>(1);
+ const offset_type* offsets_prev = last_valid_value_chunk.GetValues<offset_type>(1);
+
+ bool has_fill_value_last_chunk = *last_valid_value_offset != -1;
+ bool has_fill_value_current_chunk = false;
+ std::vector<std::tuple<bool, uint64_t, uint64_t>> offsets_reversed;
+ RETURN_NOT_OK(VisitNullBitmapInline<>(
+ reversed_bitmap, array.offset, array.length, array.GetNullCount(),
+ [&]() {
+ const offset_type offset0 = offsets[array_value_index];
+ const offset_type offset1 = offsets[array_value_index + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ *last_valid_value_offset = array_value_index;
+ has_fill_value_current_chunk = true;
+ has_fill_value_last_chunk = false;
+ array_value_index += direction;
+ return Status::OK();
+ },
+ [&]() {
+ if (has_fill_value_current_chunk || has_fill_value_last_chunk) {
+ if (!has_fill_value_last_chunk) {
+ const offset_type offset0 = offsets[*last_valid_value_offset];
+ const offset_type offset1 = offsets[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/1, offset0, offset1 - offset0));
+ } else {
+ const offset_type offset0 = offsets_prev[*last_valid_value_offset];
+ const offset_type offset1 = offsets_prev[*last_valid_value_offset + 1];
+ offsets_reversed.push_back(
+ std::make_tuple(/*current_chunk=*/0, offset0, offset1 - offset0));
+ }
+ } else {
+ offsets_reversed.push_back(std::make_tuple(/*current_chunk=*/0, -1U, -1U));
+ }
+ array_value_index += direction;
+ return Status::OK();
+ }));
+
+ if (direction == 1) {
+ for (auto it = offsets_reversed.begin(); it != offsets_reversed.end(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ } else {
+ for (auto it = offsets_reversed.rbegin(); it != offsets_reversed.rend(); ++it) {
+ if (std::get<1>(*it) == -1U && std::get<2>(*it) == -1U) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else if (std::get<0>(*it)) {
+ RETURN_NOT_OK(builder.Append(data + std::get<1>(*it), std::get<2>(*it)));
+ } else {
+ RETURN_NOT_OK(builder.Append(data_prev + std::get<1>(*it), std::get<2>(*it)));
+ }
+ }
+ }
+
+ std::shared_ptr<Array> temp_output;
+ RETURN_NOT_OK(builder.Finish(&temp_output));
+ *output = *temp_output->data();
+ // Builder type != logical type due to GenerateTypeAgnosticVarBinaryBase
+ output->type = array.type;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullExecutor<Type, enable_if_null<Type>> {
+ static Status ExecFillNull(KernelContext* ctx, const ArrayData& array,
+ const uint8_t* reversed_bitmap, ArrayData* output,
+ int8_t direction, ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ *output = array;
+ return Status::OK();
+ }
+};
+
+template <typename Type>
+struct FillNullForwardFunctor {
+ static Status Exec(KernelContext* ctx, const ExecBatch& batch, Datum* out) {
+ switch (batch[0].kind()) {
+ case Datum::ARRAY: {
+ auto array_input = *batch[0].array();
+ int64_t last_valid_value_offset = -1;
+ return FillNullForwardArray(ctx, array_input, out, array_input,
+ &last_valid_value_offset);
+ }
+ case Datum::CHUNKED_ARRAY: {
+ return FillNullForwardChunkedArray(ctx, batch[0].chunked_array(), out);
+ }
+ default:
+ break;
+ }
+ return Status::NotImplemented(
+ "Unsupported types for drop_null operation: "
+ "values=",
+ batch[0].ToString());
+ }
+
+ static Status FillNullForwardArray(KernelContext* ctx, ArrayData& array, Datum* out,
+ ArrayData last_valid_value_chunk,
+ int64_t* last_valid_value_offset) {
+ ArrayData* output = out->array().get();
+ /*if (!output->buffers[0]) {
+ ARROW_ASSIGN_OR_RAISE(output->buffers[0], ctx->AllocateBitmap(array.length));
+ }*/
+ output->length = array.length;
+ int8_t direction = 1;
+
+ if (array.MayHaveNulls()) {
+ return FillNullExecutor<Type>::ExecFillNull(
+ ctx, array, array.buffers[0]->data(), output, direction, last_valid_value_chunk,
+ last_valid_value_offset);
+ } else {
+ if (array.length > 0) {
+ *last_valid_value_offset = LastElementOffset(array, direction);
+ }
+ *output = array;
+ }
+ return Status::OK();
+ }
+
+ static Status FillNullForwardChunkedArray(KernelContext* ctx,
+ const std::shared_ptr<ChunkedArray>& values,
+ Datum* out) {
+ if (values->null_count() == 0) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ if (values->null_count() == values->length()) {
+ *out = Datum(values);
+ return Status::OK();
+ }
+ std::vector<std::shared_ptr<Array>> new_chunks;
+
+ if (values->length() > 0) {
+ ArrayData array_with_current = *values->chunk(/*first_chunk=*/0)->data();
+ int64_t last_valid_value_offset = -1;
+ for (const auto& chunk : values->chunks()) {
+ auto buffer_size = chunk->length() * bit_width(values->type()->id()) / 8;
+
+ std::unique_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(ctx->memory_pool(), values->type(), &builder));
+ RETURN_NOT_OK(builder->Reserve(chunk->length()));
+ ARROW_ASSIGN_OR_RAISE(auto array_output, builder->Finish());
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[1],
+ ctx->Allocate(buffer_size));
+ ARROW_ASSIGN_OR_RAISE(array_output->data()->buffers[0],
+ ctx->AllocateBitmap(chunk->length()));
Review comment:
LGTM, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r778329628
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
@lidavidm the types were added and the condition was removed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] ursabot commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
ursabot commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1005743192
Benchmark runs are scheduled for baseline = 67a29fdffec3c2646b29aa07b49729305aac0d38 and contender = ec38aebb36e99e54e69089cbc6a623a616575dde. ec38aebb36e99e54e69089cbc6a623a616575dde is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Scheduled] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/a9958e4a8da04ccf834dc20ef43bbb5e...1b5e3a4ab1db415ca90ec502fccedb4f/)
[Scheduled] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/9f3bf8138e5148209c3760429894d52b...ffe8032605144c5388649c3edf75a52a/)
[Scheduled] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/85986fd142c94225a708903d62926e83...924ad696374942ce813258bde7c2334e/)
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python. Runs only benchmarks with cloud = True
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on a change in pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on a change in pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#discussion_r777816740
##########
File path: cpp/src/arrow/compute/kernels/vector_replace_test.cc
##########
@@ -793,5 +851,772 @@ TYPED_TEST(TestReplaceBinary, ReplaceWithMaskRandom) {
}
}
+template <typename T>
+class TestFillNullNumeric : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullDecimal : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+template <typename T>
+class TestFillNullBinary : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+template <typename T>
+class TestFillNullNullType : public TestReplaceKernel<T> {
+ protected:
+ std::shared_ptr<DataType> type() override { return default_type_instance<T>(); }
+};
+
+TYPED_TEST_SUITE(TestFillNullNumeric, NumericBasedTypes);
+TYPED_TEST_SUITE(TestFillNullDecimal, DecimalArrowTypes);
+TYPED_TEST_SUITE(TestFillNullBinary, BaseBinaryArrowTypes);
+TYPED_TEST_SUITE(TestFillNullNullType, ::testing::Types<NullType>);
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array("[]"), this->array("[]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"),
+ this->array("[null, null, null, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"),
+ this->array("[null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"),
+ this->array("[null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"),
+ this->array("[null, null, null, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"),
+ this->array("[null, null, 4, 4, 5, 5]"));
+
+ this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"),
+ this->array("[1,4,4]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 ,4, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 6]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,4, 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 6]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+ this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"),
+ this->array("[1, 4 ,4, 5, 5, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00"])"),
+ this->array(R"([null, null, null, "30.00"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "30.00", null])"),
+ this->array(R"([null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "30.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "30.00", null])"),
+ this->array(R"([null, null, null, "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"([null, null, "30.00",null, "5.00", null])"),
+ this->array(R"([null, null, "30.00", "30.00", "5.00", "5.00"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["10.00","30.00",null])"),
+ this->array(R"(["10.00","30.00","30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, null, null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "30.00", "30.00", "30.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["10.00", "30.00", null, "5.00", null, null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, null, "5.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "5.00", "5.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00", null, "5.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "30.00" ,"30.00", "5.00", "5.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesForward) {
+ this->AssertFillNullArray(FillNullForward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, null, "ccc"])"),
+ this->array(R"([null, null, null, "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, "ccc", null])"),
+ this->array(R"([null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward, this->array(R"([null, null, "ccc", null])"),
+ this->array(R"([null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, null, "ccc", null])"),
+ this->array(R"([null, null, null, "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"([null, null, "ccc",null, "xyz", null])"),
+ this->array(R"([null, null, "ccc", "ccc", "xyz", "xyz"])"));
+
+ this->AssertFillNullArray(FillNullForward, this->array(R"(["aaa","ccc",null])"),
+ this->array(R"(["aaa","ccc","ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, null, null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "ccc", "ccc", "ccc"])"));
+ this->AssertFillNullArray(FillNullForward,
+ this->array(R"(["aaa", "ccc", null, "xyz", null, null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "qwert"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, null, "xyz"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "xyz", "xyz"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", null])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "qwert"])"));
+ this->AssertFillNullArray(
+ FillNullForward, this->array(R"(["aaa", "ccc", null, "xyz", null, "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+ this->AssertFillNullArray(
+ FillNullForward,
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"),
+ this->array(R"(["aaa", "ccc" ,"ccc", "xyz", "xyz", "qwert", "uy"])"));
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"),
+ this->array("[null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"),
+ this->array("[4, 4,null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"),
+ this->array("[4, 4, 4, 4]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"),
+ this->array("[4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"),
+ this->array("[4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"),
+ this->array("[4, 4, 4, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[null, null, 4,null, 5, null]"),
+ this->array("[4, 4, 4, 5, 5, null]"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"),
+ this->array("[1, 4, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, null, null, null]"),
+ this->array("[1, 4 ,null, null, null, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"),
+ this->array("[1, 4 , 5, 5, null, null]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 6]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 6]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, null, 5]"),
+ this->array("[1, 4 ,5 , 5, 5, 5, 5]"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array("[1, 4, null, 5, null, 6, null]"),
+ this->array("[1, 4 ,5 , 5, 6, 6, null]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+ this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"),
+ this->array("[1, 4 ,5, 5, 6, 6, 7]"));
+}
+
+TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "40.00", null, null, null])"),
+ this->array(R"(["40.00", "40.00",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00"])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "40.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "40.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"([null, null, "40.00",null, "50.00", null])"),
+ this->array(R"(["40.00", "40.00", "40.00", "50.00", "50.00", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["10.00", "40.00", null])"),
+ this->array(R"(["10.00", "40.00", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, null, null, null])"),
+ this->array(R"(["10.00", "40.00" ,null, null, null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["10.00", "40.00", null, "50.00", null, null])"),
+ this->array(R"(["10.00", "40.00" , "50.00", "50.00", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "6.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "6.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, null, "50.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "50.00", "50.00", "50.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", null])"),
+ this->array(R"(["10.00", "40.00" ,"50.00" , "50.00", "6.00", "6.00", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00", null, "50.00", null, "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"),
+ this->array(R"(["10.00", "40.00" ,"50.00", "50.00", "6.00", "6.00", "7.00"])"));
+}
+
+TYPED_TEST(TestFillNullBinary, FillNullValuesBackward) {
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([])"), this->array(R"([])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, null])"),
+ this->array(R"([null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, "afd", null, null, null])"),
+ this->array(R"(["afd", "afd",null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, null, "afd"])"),
+ this->array(R"(["afd", "afd", "afd", "afd"])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, "afd", null])"),
+ this->array(R"(["afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward, this->array(R"([null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, null, "afd", null])"),
+ this->array(R"(["afd", "afd", "afd", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"([null, null, "afd",null, "qwe", null])"),
+ this->array(R"(["afd", "afd", "afd", "qwe", "qwe", null])"));
+
+ this->AssertFillNullArray(FillNullBackward, this->array(R"(["tyu", "afd", null])"),
+ this->array(R"(["tyu", "afd", null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, null, null, null])"),
+ this->array(R"(["tyu", "afd" ,null, null, null, null])"));
+ this->AssertFillNullArray(FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null])"),
+ this->array(R"(["tyu", "afd" , "qwe", "qwe", null, null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, null, "oiutyu"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "oiutyu"])"));
+ this->AssertFillNullArray(
+ FillNullBackward, this->array(R"(["tyu", "afd", null, "qwe", null, null, "qwe"])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "qwe", "qwe", "qwe"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", null])"),
+ this->array(R"(["tyu", "afd" ,"qwe" , "qwe", "oiutyu", "oiutyu", null])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd", null, "qwe", null, "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+ this->AssertFillNullArray(
+ FillNullBackward,
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"),
+ this->array(R"(["tyu", "afd" ,"qwe", "qwe", "oiutyu", "oiutyu", "aaaagggbbb"])"));
+}
+
+// For Test Blocks
+TYPED_TEST(TestFillNullNumeric, FillNullForwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*nulls=*/0);
+
+ if (array_random) {
+ auto x_ptr = array_random->data()->template GetValues<CType>(1);
+ ASSERT_OK_AND_ASSIGN(auto array_null,
+ MakeArrayOfNull(array_random->type(), len_null));
+ auto array_null_filled =
+ ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1]);
+ {
+ ASSERT_OK_AND_ASSIGN(auto value_array,
+ Concatenate({array_random, array_null, array_random}));
+ ASSERT_OK_AND_ASSIGN(auto result_array,
+ Concatenate({array_random, array_null_filled, array_random}));
+ this->AssertFillNullArray(FillNullForward, value_array, result_array);
+ }
+ }
+}
+
+TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) {
+ using CType = typename TypeTraits<TypeParam>::CType;
+ random::RandomArrayGenerator rand(/*seed=*/1000);
+ int64_t len_null = 500;
+ int64_t len_random = 1000;
+ std::shared_ptr<Array> array_random =
+ rand.Numeric<TypeParam>(len_random, /*min=*/0, /*max=*/200, /*null_count=*/0);
+
+ if (array_random) {
Review comment:
It work for generate a random_array, but currently there is not support for DateTypes for generate a ConstantArray which is needed for this test.
`
auto constant_array = ConstantArrayGenerator::Numeric<TypeParam>(len_null, x_ptr[len_random - 1])
Concatenate({array_random, constant_array, array_random}))
`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] AlvinJ15 commented on pull request #11853: ARROW-1699: [C++] forward, backward fill kernel functions
Posted by GitBox <gi...@apache.org>.
AlvinJ15 commented on pull request #11853:
URL: https://github.com/apache/arrow/pull/11853#issuecomment-1002310526
@lidavidm I resolved all comments. The AppVeyor gave me errors on pipeline, how Can I solve that error?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org